From 81dc810d58a26ddfbfac3063ee3868d9c8431489 Mon Sep 17 00:00:00 2001 From: chatton Date: Wed, 25 Feb 2026 12:16:03 +0000 Subject: [PATCH 1/5] refactor: move spamoor benchmark into testify suite in test/e2e/benchmark - Create test/e2e/benchmark/ subpackage with SpamoorSuite (testify/suite) - Move spamoor smoke test into suite as TestSpamoorSmoke - Split helpers into focused files: traces.go, output.go, metrics.go - Introduce resultWriter for defer-based benchmark JSON output - Export shared symbols from evm_test_common.go for cross-package use - Restructure CI to fan-out benchmark jobs and fan-in publishing - Run benchmarks on PRs only when benchmark-related files change --- .github/workflows/benchmark.yml | 111 ++++++++- test/e2e/benchmark/metrics.go | 48 ++++ test/e2e/benchmark/output.go | 77 +++++++ test/e2e/benchmark/spamoor_smoke_test.go | 114 +++++++++ test/e2e/benchmark/suite_test.go | 144 ++++++++++++ test/e2e/benchmark/traces.go | 67 ++++++ test/e2e/evm_contract_bench_test.go | 6 +- test/e2e/evm_force_inclusion_e2e_test.go | 6 +- test/e2e/evm_full_node_e2e_test.go | 10 +- test/e2e/evm_spamoor_smoke_test.go | 281 ----------------------- test/e2e/evm_test_common.go | 75 +++--- test/e2e/failover_e2e_test.go | 4 +- 12 files changed, 605 insertions(+), 338 deletions(-) create mode 100644 test/e2e/benchmark/metrics.go create mode 100644 test/e2e/benchmark/output.go create mode 100644 test/e2e/benchmark/spamoor_smoke_test.go create mode 100644 test/e2e/benchmark/suite_test.go create mode 100644 test/e2e/benchmark/traces.go delete mode 100644 test/e2e/evm_spamoor_smoke_test.go diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 15cb1a4ff..2f8fcdb1c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -5,6 +5,15 @@ permissions: {} push: branches: - main + pull_request: + branches: + - main + paths: + - 'test/e2e/benchmark/**' + - 'test/e2e/evm_contract_bench_test.go' + - 'test/e2e/evm_test_common.go' + - 'test/e2e/sut_helper.go' + - '.github/workflows/benchmark.yml' workflow_dispatch: jobs: @@ -12,9 +21,6 @@ jobs: name: EVM Contract Benchmark runs-on: ubuntu-latest timeout-minutes: 30 - permissions: - contents: write - issues: write steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Go @@ -29,30 +35,113 @@ jobs: run: | cd test/e2e && go test -tags evm -bench=. -benchmem -run='^$' \ -timeout=10m --evm-binary=../../build/evm | tee output.txt - - name: Store benchmark result + - name: Run Block Executor benchmarks + run: | + go test -bench=BenchmarkProduceBlock -benchmem -run='^$' \ + ./block/internal/executing/... > block_executor_output.txt + - name: Upload benchmark results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: evm-benchmark-results + path: | + test/e2e/output.txt + block_executor_output.txt + + spamoor-benchmark: + name: Spamoor Trace Benchmark + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Set up Go + uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 + with: + go-version-file: ./go.mod + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + - name: Build binaries + run: make build-evm build-da + - name: Run Spamoor smoke test + run: | + cd test/e2e && BENCH_JSON_OUTPUT=benchmark/spamoor_bench.json go test -tags evm \ + -run='^TestSpamoorSuite$/^TestSpamoorSmoke$' -v -timeout=15m \ + --evm-binary=../../build/evm ./benchmark/ + - name: Upload benchmark results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: spamoor-benchmark-results + path: test/e2e/benchmark/spamoor_bench.json + + # single job to push all results to gh-pages sequentially, avoiding race conditions + publish-benchmarks: + name: Publish Benchmark Results + needs: [evm-benchmark, spamoor-benchmark] + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Download EVM benchmark results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: evm-benchmark-results + - name: Download Spamoor benchmark results + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: spamoor-benchmark-results + path: test/e2e/benchmark/ + + # only update the benchmark baseline on push/dispatch, not on PRs + - name: Store EVM Contract Roundtrip result + if: always() uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7 with: name: EVM Contract Roundtrip tool: 'go' output-file-path: test/e2e/output.txt - auto-push: true + auto-push: ${{ github.event_name != 'pull_request' }} + save-data-file: ${{ github.event_name != 'pull_request' }} github-token: ${{ secrets.GITHUB_TOKEN }} alert-threshold: '150%' fail-on-alert: true comment-on-alert: true - - name: Run Block Executor benchmarks - run: | - go test -bench=BenchmarkProduceBlock -benchmem -run='^$' \ - ./block/internal/executing/... > block_executor_output.txt - - name: Store Block Executor benchmark result + # delete local gh-pages so the next benchmark action step fetches fresh from remote + - name: Reset local gh-pages branch + if: always() + run: git branch -D gh-pages || true + + - name: Store Block Executor result + if: always() uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7 with: name: Block Executor Benchmark tool: 'go' output-file-path: block_executor_output.txt - auto-push: true + auto-push: ${{ github.event_name != 'pull_request' }} + save-data-file: ${{ github.event_name != 'pull_request' }} github-token: ${{ secrets.GITHUB_TOKEN }} alert-threshold: '150%' fail-on-alert: true comment-on-alert: true + + # delete local gh-pages so the next benchmark action step fetches fresh from remote + - name: Reset local gh-pages branch + if: always() + run: git branch -D gh-pages || true + + - name: Store Spamoor Trace result + if: always() + uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7 + with: + name: Spamoor Trace Benchmarks + tool: 'customSmallerIsBetter' + output-file-path: test/e2e/benchmark/spamoor_bench.json + auto-push: ${{ github.event_name != 'pull_request' }} + save-data-file: ${{ github.event_name != 'pull_request' }} + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: '150%' + fail-on-alert: false + comment-on-alert: true diff --git a/test/e2e/benchmark/metrics.go b/test/e2e/benchmark/metrics.go new file mode 100644 index 000000000..fcc51af86 --- /dev/null +++ b/test/e2e/benchmark/metrics.go @@ -0,0 +1,48 @@ +//go:build evm + +package benchmark + +import ( + "fmt" + "net/http" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" +) + +// requireHTTP polls a URL until it returns a 2xx status code or the timeout expires. +func requireHTTP(t testing.TB, url string, timeout time.Duration) { + t.Helper() + client := &http.Client{Timeout: 200 * time.Millisecond} + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + resp, err := client.Get(url) + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return + } + lastErr = fmt.Errorf("status %d", resp.StatusCode) + } else { + lastErr = err + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("daemon not ready at %s: %v", url, lastErr) +} + +// sumCounter sums all counter values in a prometheus MetricFamily. +func sumCounter(f *dto.MetricFamily) float64 { + if f == nil || f.GetType() != dto.MetricType_COUNTER { + return 0 + } + var sum float64 + for _, m := range f.GetMetric() { + if m.GetCounter() != nil && m.GetCounter().Value != nil { + sum += m.GetCounter().GetValue() + } + } + return sum +} diff --git a/test/e2e/benchmark/output.go b/test/e2e/benchmark/output.go new file mode 100644 index 000000000..0d05ebd4e --- /dev/null +++ b/test/e2e/benchmark/output.go @@ -0,0 +1,77 @@ +//go:build evm + +package benchmark + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "testing" + + e2e "github.com/evstack/ev-node/test/e2e" + "github.com/stretchr/testify/require" +) + +// entry matches the customSmallerIsBetter format for github-action-benchmark. +type entry struct { + Name string `json:"name"` + Unit string `json:"unit"` + Value float64 `json:"value"` +} + +// resultWriter accumulates benchmark entries and writes them to a JSON file +// when flush is called. Create one early in a test and defer flush so results +// are written regardless of where the test exits. +type resultWriter struct { + t testing.TB + label string + entries []entry +} + +func newResultWriter(t testing.TB, label string) *resultWriter { + return &resultWriter{t: t, label: label} +} + +// addSpans aggregates trace spans into per-operation avg duration entries. +func (w *resultWriter) addSpans(spans []e2e.TraceSpan) { + m := e2e.AggregateSpanStats(spans) + if len(m) == 0 { + return + } + + names := make([]string, 0, len(m)) + for name := range m { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + s := m[name] + avg := float64(s.Total.Microseconds()) / float64(s.Count) + w.entries = append(w.entries, entry{ + Name: fmt.Sprintf("%s - %s (avg)", w.label, name), + Unit: "us", + Value: avg, + }) + } +} + +// addEntry appends a custom entry to the results. +func (w *resultWriter) addEntry(e entry) { + w.entries = append(w.entries, e) +} + +// flush writes accumulated entries to the path in BENCH_JSON_OUTPUT. +// It is a no-op when the env var is unset or no entries were added. +func (w *resultWriter) flush() { + outputPath := os.Getenv("BENCH_JSON_OUTPUT") + if outputPath == "" || len(w.entries) == 0 { + return + } + + data, err := json.MarshalIndent(w.entries, "", " ") + require.NoError(w.t, err, "failed to marshal benchmark JSON") + require.NoError(w.t, os.WriteFile(outputPath, data, 0644), "failed to write benchmark JSON to %s", outputPath) + w.t.Logf("wrote %d benchmark entries to %s", len(w.entries), outputPath) +} diff --git a/test/e2e/benchmark/spamoor_smoke_test.go b/test/e2e/benchmark/spamoor_smoke_test.go new file mode 100644 index 000000000..1f56f2687 --- /dev/null +++ b/test/e2e/benchmark/spamoor_smoke_test.go @@ -0,0 +1,114 @@ +//go:build evm + +package benchmark + +import ( + "time" + + "github.com/celestiaorg/tastora/framework/docker/evstack/spamoor" + e2e "github.com/evstack/ev-node/test/e2e" +) + +// TestSpamoorSmoke spins up reth + sequencer and a Spamoor node, starts a few +// basic spammers, waits briefly, then validates trace spans and prints a concise +// metrics summary. +func (s *SpamoorSuite) TestSpamoorSmoke() { + t := s.T() + w := newResultWriter(t, "SpamoorSmoke") + defer w.flush() + + e := s.setupEnv(config{ + rethTag: "pr-140", + serviceName: "ev-node-smoke", + }) + api := e.spamoorAPI + + eoatx := map[string]any{ + "throughput": 100, + "total_count": 3000, + "max_pending": 4000, + "max_wallets": 300, + "amount": 100, + "random_amount": true, + "random_target": true, + "base_fee": 20, + "tip_fee": 2, + "refill_amount": "1000000000000000000", + "refill_balance": "500000000000000000", + "refill_interval": 600, + } + + gasburner := map[string]any{ + "throughput": 25, + "total_count": 2000, + "max_pending": 8000, + "max_wallets": 500, + "gas_units_to_burn": 3000000, + "base_fee": 20, + "tip_fee": 5, + "rebroadcast": 5, + "refill_amount": "5000000000000000000", + "refill_balance": "2000000000000000000", + "refill_interval": 300, + } + + var ids []int + id, err := api.CreateSpammer("smoke-eoatx", spamoor.ScenarioEOATX, eoatx, true) + s.Require().NoError(err, "failed to create eoatx spammer") + ids = append(ids, id) + id, err = api.CreateSpammer("smoke-gasburner", spamoor.ScenarioGasBurnerTX, gasburner, true) + s.Require().NoError(err, "failed to create gasburner spammer") + ids = append(ids, id) + + for _, id := range ids { + idToDelete := id + t.Cleanup(func() { _ = api.DeleteSpammer(idToDelete) }) + } + + // allow spamoor enough time to generate transaction throughput + // so that the expected tracing spans appear in Jaeger. + time.Sleep(60 * time.Second) + + // fetch parsed metrics and print a concise summary. + metrics, err := api.GetMetrics() + s.Require().NoError(err, "failed to get metrics") + sent := sumCounter(metrics["spamoor_transactions_sent_total"]) + fail := sumCounter(metrics["spamoor_transactions_failed_total"]) + + // collect traces + evNodeSpans := s.collectServiceTraces(e, "ev-node-smoke") + evRethSpans := s.collectServiceTraces(e, "ev-reth") + e2e.PrintTraceReport(t, "ev-node-smoke", evNodeSpans) + e2e.PrintTraceReport(t, "ev-reth", evRethSpans) + + w.addSpans(append(evNodeSpans, evRethSpans...)) + + // assert expected ev-node span names + assertSpanNames(t, evNodeSpans, []string{ + "BlockExecutor.ProduceBlock", + "BlockExecutor.ApplyBlock", + "BlockExecutor.CreateBlock", + "BlockExecutor.RetrieveBatch", + "Executor.ExecuteTxs", + "Executor.SetFinal", + "Engine.ForkchoiceUpdated", + "Engine.NewPayload", + "Engine.GetPayload", + "Eth.GetBlockByNumber", + "Sequencer.GetNextBatch", + "DASubmitter.SubmitHeaders", + "DASubmitter.SubmitData", + "DA.Submit", + }, "ev-node-smoke") + + // assert expected ev-reth span names + assertSpanNames(t, evRethSpans, []string{ + "build_payload", + "execute_tx", + "try_build", + "validate_transaction", + }, "ev-reth") + + s.Require().Greater(sent, float64(0), "at least one transaction should have been sent") + s.Require().Zero(fail, "no transactions should have failed") +} diff --git a/test/e2e/benchmark/suite_test.go b/test/e2e/benchmark/suite_test.go new file mode 100644 index 000000000..a43f25890 --- /dev/null +++ b/test/e2e/benchmark/suite_test.go @@ -0,0 +1,144 @@ +//go:build evm + +package benchmark + +import ( + "context" + "path/filepath" + "testing" + "time" + + tastoradocker "github.com/celestiaorg/tastora/framework/docker" + "github.com/celestiaorg/tastora/framework/docker/evstack/reth" + "github.com/celestiaorg/tastora/framework/docker/evstack/spamoor" + "github.com/celestiaorg/tastora/framework/docker/jaeger" + tastoratypes "github.com/celestiaorg/tastora/framework/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/stretchr/testify/suite" + "go.uber.org/zap/zaptest" + + e2e "github.com/evstack/ev-node/test/e2e" +) + +// SpamoorSuite groups benchmarks that use Spamoor for load generation +// and Jaeger for distributed tracing. Docker client and network are shared +// across all tests in the suite. +type SpamoorSuite struct { + suite.Suite + dockerCli tastoratypes.TastoraDockerClient + networkID string +} + +func TestSpamoorSuite(t *testing.T) { + suite.Run(t, new(SpamoorSuite)) +} + +func (s *SpamoorSuite) SetupTest() { + s.dockerCli, s.networkID = tastoradocker.Setup(s.T()) +} + +// env holds a fully-wired environment created by setupEnv. +type env struct { + jaeger *jaeger.Node + evmEnv *e2e.EVMEnv + sut *e2e.SystemUnderTest + spamoorAPI *spamoor.API + ethClient *ethclient.Client +} + +// config parameterizes the per-test environment setup. +type config struct { + rethTag string + serviceName string +} + +// setupEnv creates a Jaeger + reth + sequencer + Spamoor environment for +// a single test. Each call spins up isolated infrastructure so tests +// can't interfere with each other. +func (s *SpamoorSuite) setupEnv(cfg config) *env { + t := s.T() + ctx := t.Context() + sut := e2e.NewSystemUnderTest(t) + + // jaeger + jcfg := jaeger.Config{Logger: zaptest.NewLogger(t), DockerClient: s.dockerCli, DockerNetworkID: s.networkID} + jg, err := jaeger.New(ctx, jcfg, t.Name(), 0) + s.Require().NoError(err, "failed to create jaeger node") + t.Cleanup(func() { _ = jg.Remove(t.Context()) }) + s.Require().NoError(jg.Start(ctx), "failed to start jaeger node") + + // reth + local DA with OTLP tracing to Jaeger + evmEnv := e2e.SetupCommonEVMEnv(t, sut, s.dockerCli, s.networkID, + e2e.WithRethOpts(func(b *reth.NodeBuilder) { + b.WithTag(cfg.rethTag).WithEnv( + "OTEL_EXPORTER_OTLP_ENDPOINT="+jg.Internal.IngestHTTPEndpoint()+"/v1/traces", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="+jg.Internal.IngestHTTPEndpoint()+"/v1/traces", + "OTEL_EXPORTER_OTLP_PROTOCOL=http", + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http", + "RUST_LOG=debug", + "OTEL_SDK_DISABLED=false", + ) + }), + ) + + // sequencer with tracing + sequencerHome := filepath.Join(t.TempDir(), "sequencer") + otlpHTTP := jg.External.IngestHTTPEndpoint() + e2e.SetupSequencerNode(t, sut, sequencerHome, evmEnv.SequencerJWT, evmEnv.GenesisHash, evmEnv.Endpoints, + "--evnode.instrumentation.tracing=true", + "--evnode.instrumentation.tracing_endpoint", otlpHTTP, + "--evnode.instrumentation.tracing_sample_rate", "1.0", + "--evnode.instrumentation.tracing_service_name", cfg.serviceName, + ) + t.Log("sequencer node is up") + + // eth client + ethClient, err := ethclient.Dial(evmEnv.Endpoints.GetSequencerEthURL()) + s.Require().NoError(err, "failed to dial sequencer eth endpoint") + t.Cleanup(func() { ethClient.Close() }) + + // spamoor + ni, err := evmEnv.RethNode.GetNetworkInfo(ctx) + s.Require().NoError(err, "failed to get reth network info") + internalRPC := "http://" + ni.Internal.RPCAddress() + + spBuilder := spamoor.NewNodeBuilder(t.Name()). + WithDockerClient(evmEnv.RethNode.DockerClient). + WithDockerNetworkID(evmEnv.RethNode.NetworkID). + WithLogger(evmEnv.RethNode.Logger). + WithRPCHosts(internalRPC). + WithPrivateKey(e2e.TestPrivateKey) + + spNode, err := spBuilder.Build(ctx) + s.Require().NoError(err, "failed to build spamoor node") + t.Cleanup(func() { _ = spNode.Remove(t.Context()) }) + s.Require().NoError(spNode.Start(ctx), "failed to start spamoor node") + + spInfo, err := spNode.GetNetworkInfo(ctx) + s.Require().NoError(err, "failed to get spamoor network info") + apiAddr := "http://127.0.0.1:" + spInfo.External.Ports.HTTP + requireHTTP(t, apiAddr+"/api/spammers", 30*time.Second) + + return &env{ + jaeger: jg, + evmEnv: evmEnv, + sut: sut, + spamoorAPI: spNode.API(), + ethClient: ethClient, + } +} + +// collectServiceTraces fetches traces from Jaeger for the given service and returns the spans. +func (s *SpamoorSuite) collectServiceTraces(e *env, serviceName string) []e2e.TraceSpan { + ctx, cancel := context.WithTimeout(s.T().Context(), 3*time.Minute) + defer cancel() + + ok, err := e.jaeger.External.WaitForTraces(ctx, serviceName, 1, 2*time.Second) + s.Require().NoError(err, "error waiting for %s traces; UI: %s", serviceName, e.jaeger.External.QueryURL()) + s.Require().True(ok, "expected at least one trace from %s; UI: %s", serviceName, e.jaeger.External.QueryURL()) + + traces, err := e.jaeger.External.Traces(ctx, serviceName, 10000) + s.Require().NoError(err, "failed to fetch %s traces", serviceName) + + return toTraceSpans(extractSpansFromTraces(traces)) +} diff --git a/test/e2e/benchmark/traces.go b/test/e2e/benchmark/traces.go new file mode 100644 index 000000000..76f160dfa --- /dev/null +++ b/test/e2e/benchmark/traces.go @@ -0,0 +1,67 @@ +//go:build evm + +package benchmark + +import ( + "testing" + "time" + + e2e "github.com/evstack/ev-node/test/e2e" + "github.com/stretchr/testify/require" +) + +// jaegerSpan holds the fields we extract from Jaeger's untyped JSON response. +type jaegerSpan struct { + operationName string + duration float64 // microseconds +} + +func (j jaegerSpan) SpanName() string { return j.operationName } +func (j jaegerSpan) SpanDuration() time.Duration { return time.Duration(j.duration) * time.Microsecond } + +// extractSpansFromTraces walks Jaeger's []any response and pulls out span operation names and durations. +func extractSpansFromTraces(traces []any) []jaegerSpan { + var out []jaegerSpan + for _, t := range traces { + traceMap, ok := t.(map[string]any) + if !ok { + continue + } + spans, ok := traceMap["spans"].([]any) + if !ok { + continue + } + for _, s := range spans { + spanMap, ok := s.(map[string]any) + if !ok { + continue + } + name, _ := spanMap["operationName"].(string) + dur, _ := spanMap["duration"].(float64) + if name != "" { + out = append(out, jaegerSpan{operationName: name, duration: dur}) + } + } + } + return out +} + +func toTraceSpans(spans []jaegerSpan) []e2e.TraceSpan { + out := make([]e2e.TraceSpan, len(spans)) + for i, s := range spans { + out[i] = s + } + return out +} + +// assertSpanNames verifies that all expected span names appear in the trace data. +func assertSpanNames(t testing.TB, spans []e2e.TraceSpan, expected []string, label string) { + t.Helper() + opNames := make(map[string]struct{}, len(spans)) + for _, span := range spans { + opNames[span.SpanName()] = struct{}{} + } + for _, name := range expected { + require.Contains(t, opNames, name, "expected span %q not found in %s traces", name, label) + } +} diff --git a/test/e2e/evm_contract_bench_test.go b/test/e2e/evm_contract_bench_test.go index 7fd67dcc5..006a8ea1c 100644 --- a/test/e2e/evm_contract_bench_test.go +++ b/test/e2e/evm_contract_bench_test.go @@ -202,7 +202,7 @@ func (c *otlpCollector) getSpans() []*tracepb.Span { return cp } -// otlpSpanAdapter wraps an OTLP protobuf span to implement traceSpan. +// otlpSpanAdapter wraps an OTLP protobuf span to implement TraceSpan. type otlpSpanAdapter struct { span *tracepb.Span } @@ -215,11 +215,11 @@ func (a otlpSpanAdapter) SpanDuration() time.Duration { func printCollectedTraceReport(b testing.TB, collector *otlpCollector) { b.Helper() raw := collector.getSpans() - spans := make([]traceSpan, len(raw)) + spans := make([]TraceSpan, len(raw)) for i, s := range raw { spans[i] = otlpSpanAdapter{span: s} } - printTraceReport(b, "ev-node", spans) + PrintTraceReport(b, "ev-node", spans) } // waitForReceipt polls for a transaction receipt until it is available. diff --git a/test/e2e/evm_force_inclusion_e2e_test.go b/test/e2e/evm_force_inclusion_e2e_test.go index 00046e3b2..bc04eba6a 100644 --- a/test/e2e/evm_force_inclusion_e2e_test.go +++ b/test/e2e/evm_force_inclusion_e2e_test.go @@ -79,7 +79,7 @@ func setupSequencerWithForceInclusion(t *testing.T, sut *SystemUnderTest, nodeHo // Use common setup (no full node needed initially) dcli, netID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dcli, netID) + env := SetupCommonEVMEnv(t, sut, dcli, netID) // Create passphrase file passphraseFile := createPassphraseFile(t, nodeHome) @@ -195,7 +195,7 @@ func TestEvmFullNodeForceInclusionE2E(t *testing.T) { // We manually setup sequencer here because we need the force inclusion flag, // and we need to capture variables for full node setup. dockerClient, networkID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) passphraseFile := createPassphraseFile(t, sequencerHome) jwtSecretFile := createJWTSecretFile(t, sequencerHome, env.SequencerJWT) @@ -288,7 +288,7 @@ func setupMaliciousSequencer(t *testing.T, sut *SystemUnderTest, nodeHome string // Use common setup with full node support dockerClient, networkID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) // Use env fields inline below to reduce local vars passphraseFile := createPassphraseFile(t, nodeHome) diff --git a/test/e2e/evm_full_node_e2e_test.go b/test/e2e/evm_full_node_e2e_test.go index 23a3dbcf8..98d07fe81 100644 --- a/test/e2e/evm_full_node_e2e_test.go +++ b/test/e2e/evm_full_node_e2e_test.go @@ -215,10 +215,10 @@ func setupSequencerWithFullNode(t *testing.T, sut *SystemUnderTest, sequencerHom // Common setup for both sequencer and full node dcli, netID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dcli, netID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dcli, netID, WithFullNode()) // Setup sequencer - setupSequencerNode(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints) + SetupSequencerNode(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints) t.Log("Sequencer node is up") // Get P2P address and setup full node @@ -648,7 +648,7 @@ func setupSequencerWithFullNodeLazy(t *testing.T, sut *SystemUnderTest, sequence // Common setup for both sequencer and full node dcli, netID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dcli, netID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dcli, netID, WithFullNode()) t.Logf("Generated test endpoints - Rollkit RPC: %s, P2P: %s, Full Node RPC: %s, P2P: %s, DA Port: %s", env.Endpoints.RollkitRPCPort, env.Endpoints.RollkitP2PPort, env.Endpoints.FullNodeRPCPort, env.Endpoints.FullNodeP2PPort, env.Endpoints.DAPort) @@ -1042,14 +1042,14 @@ func testSequencerFullNodeRestart(t *testing.T, initialLazyMode, restartLazyMode t.Logf("Test mode: initial_lazy=%t, restart_lazy=%t", initialLazyMode, restartLazyMode) dcli, netID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dcli, netID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dcli, netID, WithFullNode()) // Setup sequencer based on initial mode if initialLazyMode { setupSequencerNodeLazy(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints) t.Log("Sequencer node (lazy mode) is up") } else { - setupSequencerNode(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints) + SetupSequencerNode(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints) t.Log("Sequencer node is up") } diff --git a/test/e2e/evm_spamoor_smoke_test.go b/test/e2e/evm_spamoor_smoke_test.go deleted file mode 100644 index ca172948a..000000000 --- a/test/e2e/evm_spamoor_smoke_test.go +++ /dev/null @@ -1,281 +0,0 @@ -//go:build evm - -package e2e - -import ( - "context" - "fmt" - "net/http" - "path/filepath" - "testing" - "time" - - tastoradocker "github.com/celestiaorg/tastora/framework/docker" - reth "github.com/celestiaorg/tastora/framework/docker/evstack/reth" - spamoor "github.com/celestiaorg/tastora/framework/docker/evstack/spamoor" - jaeger "github.com/celestiaorg/tastora/framework/docker/jaeger" - dto "github.com/prometheus/client_model/go" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" -) - -// TestSpamoorSmoke spins up reth + sequencer and a Spamoor node, starts a few -// basic spammers, waits briefly, then prints a concise metrics summary. -func TestSpamoorSmoke(t *testing.T) { - t.Parallel() - - sut := NewSystemUnderTest(t) - // Prepare a shared docker client and network for Jaeger and reth. - ctx := t.Context() - dcli, netID := tastoradocker.Setup(t) - jcfg := jaeger.Config{Logger: zaptest.NewLogger(t), DockerClient: dcli, DockerNetworkID: netID} - jg, err := jaeger.New(ctx, jcfg, t.Name(), 0) - require.NoError(t, err, "failed to create jaeger node") - t.Cleanup(func() { _ = jg.Remove(t.Context()) }) - require.NoError(t, jg.Start(ctx), "failed to start jaeger node") - - // Bring up reth + local DA on the same docker network as Jaeger so reth can export traces. - env := setupCommonEVMEnv(t, sut, dcli, netID, - WithRethOpts(func(b *reth.NodeBuilder) { - b.WithEnv( - "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="+jg.Internal.IngestHTTPEndpoint()+"/v1/traces", - "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http", - "RUST_LOG=info", - "OTEL_SDK_DISABLED=false", - ) - }), - ) - sequencerHome := filepath.Join(t.TempDir(), "sequencer") - - // ev-node runs on the host, so use Jaeger's external OTLP/HTTP endpoint. - otlpHTTP := jg.External.IngestHTTPEndpoint() - - // Start sequencer with tracing to Jaeger collector. - setupSequencerNode(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints, - "--evnode.instrumentation.tracing=true", - "--evnode.instrumentation.tracing_endpoint", otlpHTTP, - "--evnode.instrumentation.tracing_sample_rate", "1.0", - "--evnode.instrumentation.tracing_service_name", "ev-node-smoke", - ) - t.Log("Sequencer node is up") - - // Start Spamoor within the same Docker network, targeting reth internal RPC. - ni, err := env.RethNode.GetNetworkInfo(ctx) - require.NoError(t, err, "failed to get network info") - - internalRPC := "http://" + ni.Internal.RPCAddress() - - spBuilder := spamoor.NewNodeBuilder(t.Name()). - WithDockerClient(env.RethNode.DockerClient). - WithDockerNetworkID(env.RethNode.NetworkID). - WithLogger(env.RethNode.Logger). - WithRPCHosts(internalRPC). - WithPrivateKey(TestPrivateKey) - - spNode, err := spBuilder.Build(ctx) - require.NoError(t, err, "failed to build sp node") - - t.Cleanup(func() { _ = spNode.Remove(t.Context()) }) - require.NoError(t, spNode.Start(ctx), "failed to start spamoor node") - - // Wait for daemon readiness. - spInfo, err := spNode.GetNetworkInfo(ctx) - require.NoError(t, err, "failed to get network info") - - apiAddr := "http://127.0.0.1:" + spInfo.External.Ports.HTTP - requireHTTP(t, apiAddr+"/api/spammers", 30*time.Second) - api := spNode.API() - - // Basic scenarios (structs that YAML-marshal into the daemon config). - eoatx := map[string]any{ - "throughput": 100, - "total_count": 3000, - "max_pending": 4000, - "max_wallets": 300, - "amount": 100, - "random_amount": true, - "random_target": true, - "base_fee": 20, // gwei - "tip_fee": 2, // gwei - "refill_amount": "1000000000000000000", - "refill_balance": "500000000000000000", - "refill_interval": 600, - } - - gasburner := map[string]any{ - "throughput": 25, - "total_count": 2000, - "max_pending": 8000, - "max_wallets": 500, - "gas_units_to_burn": 3000000, - "base_fee": 20, - "tip_fee": 5, - "rebroadcast": 5, - "refill_amount": "5000000000000000000", - "refill_balance": "2000000000000000000", - "refill_interval": 300, - } - - var ids []int - id, err := api.CreateSpammer("smoke-eoatx", spamoor.ScenarioEOATX, eoatx, true) - require.NoError(t, err, "failed to create eoatx spammer") - ids = append(ids, id) - id, err = api.CreateSpammer("smoke-gasburner", spamoor.ScenarioGasBurnerTX, gasburner, true) - require.NoError(t, err, "failed to create gasburner spammer") - ids = append(ids, id) - - for _, id := range ids { - idToDelete := id - t.Cleanup(func() { _ = api.DeleteSpammer(idToDelete) }) - } - - // allow spamoor enough time to generate transaction throughput - // so that the expected tracing spans appear in Jaeger. - time.Sleep(60 * time.Second) - - // Fetch parsed metrics and print a concise summary. - metrics, err := api.GetMetrics() - require.NoError(t, err, "failed to get metrics") - sent := sumCounter(metrics["spamoor_transactions_sent_total"]) - fail := sumCounter(metrics["spamoor_transactions_failed_total"]) - - // Verify Jaeger received traces from ev-node. - // Service name is set above via --evnode.instrumentation.tracing_service_name "ev-node-smoke". - traceCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) - defer cancel() - ok, err := jg.External.WaitForTraces(traceCtx, "ev-node-smoke", 1, 2*time.Second) - require.NoError(t, err, "error while waiting for Jaeger traces; UI: %s", jg.External.QueryURL()) - require.True(t, ok, "expected at least one trace in Jaeger; UI: %s", jg.External.QueryURL()) - - // Also wait for traces from ev-reth and print a small sample. - ok, err = jg.External.WaitForTraces(traceCtx, "ev-reth", 1, 2*time.Second) - require.NoError(t, err, "error while waiting for ev-reth traces; UI: %s", jg.External.QueryURL()) - require.True(t, ok, "expected at least one trace from ev-reth; UI: %s", jg.External.QueryURL()) - - // fetch traces and print reports for both services. - // use a large limit to fetch all traces from the test run. - evNodeTraces, err := jg.External.Traces(traceCtx, "ev-node-smoke", 10000) - require.NoError(t, err, "failed to fetch ev-node-smoke traces from Jaeger") - evNodeSpans := extractSpansFromTraces(evNodeTraces) - printTraceReport(t, "ev-node-smoke", toTraceSpans(evNodeSpans)) - - evRethTraces, err := jg.External.Traces(traceCtx, "ev-reth", 10000) - require.NoError(t, err, "failed to fetch ev-reth traces from Jaeger") - evRethSpans := extractSpansFromTraces(evRethTraces) - printTraceReport(t, "ev-reth", toTraceSpans(evRethSpans)) - - // assert expected ev-node span names are present. - // these spans reliably appear during block production with transactions flowing. - expectedSpans := []string{ - "BlockExecutor.ProduceBlock", - "BlockExecutor.ApplyBlock", - "BlockExecutor.CreateBlock", - "BlockExecutor.RetrieveBatch", - "Executor.ExecuteTxs", - "Executor.SetFinal", - "Engine.ForkchoiceUpdated", - "Engine.NewPayload", - "Engine.GetPayload", - "Eth.GetBlockByNumber", - "Sequencer.GetNextBatch", - "DASubmitter.SubmitHeaders", - "DASubmitter.SubmitData", - "DA.Submit", - } - opNames := make(map[string]struct{}, len(evNodeSpans)) - for _, s := range evNodeSpans { - opNames[s.operationName] = struct{}{} - } - for _, name := range expectedSpans { - require.Contains(t, opNames, name, "expected span %q not found in ev-node-smoke traces", name) - } - - // ev-reth span names are internal to the Rust OTLP exporter and may change - // across versions, so we only assert that spans were collected at all. - // TODO: check for more specific spans once implemented. - require.NotEmpty(t, evRethSpans, "expected at least one span from ev-reth") - - require.Greater(t, sent, float64(0), "at least one transaction should have been sent") - require.Zero(t, fail, "no transactions should have failed") -} - -// --- helpers --- - -func requireHTTP(t *testing.T, url string, timeout time.Duration) { - t.Helper() - client := &http.Client{Timeout: 200 * time.Millisecond} - deadline := time.Now().Add(timeout) - var lastErr error - for time.Now().Before(deadline) { - resp, err := client.Get(url) - if err == nil { - _ = resp.Body.Close() - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return - } - lastErr = fmt.Errorf("status %d", resp.StatusCode) - } else { - lastErr = err - } - time.Sleep(100 * time.Millisecond) - } - t.Fatalf("daemon not ready at %s: %v", url, lastErr) -} - -// Metric family helpers. -func sumCounter(f *dto.MetricFamily) float64 { - if f == nil || f.GetType() != dto.MetricType_COUNTER { - return 0 - } - var sum float64 - for _, m := range f.GetMetric() { - if m.GetCounter() != nil && m.GetCounter().Value != nil { - sum += m.GetCounter().GetValue() - } - } - return sum -} - -// jaegerSpan holds the fields we extract from Jaeger's untyped JSON response. -type jaegerSpan struct { - operationName string - duration float64 // microseconds -} - -func (j jaegerSpan) SpanName() string { return j.operationName } -func (j jaegerSpan) SpanDuration() time.Duration { return time.Duration(j.duration) * time.Microsecond } - -// extractSpansFromTraces walks Jaeger's []any response and pulls out span operation names and durations. -func extractSpansFromTraces(traces []any) []jaegerSpan { - var out []jaegerSpan - for _, t := range traces { - traceMap, ok := t.(map[string]any) - if !ok { - continue - } - spans, ok := traceMap["spans"].([]any) - if !ok { - continue - } - for _, s := range spans { - spanMap, ok := s.(map[string]any) - if !ok { - continue - } - name, _ := spanMap["operationName"].(string) - dur, _ := spanMap["duration"].(float64) - if name != "" { - out = append(out, jaegerSpan{operationName: name, duration: dur}) - } - } - } - return out -} - -func toTraceSpans(spans []jaegerSpan) []traceSpan { - out := make([]traceSpan, len(spans)) - for i, s := range spans { - out[i] = s - } - return out -} diff --git a/test/e2e/evm_test_common.go b/test/e2e/evm_test_common.go index 5ddaf935b..161ff8934 100644 --- a/test/e2e/evm_test_common.go +++ b/test/e2e/evm_test_common.go @@ -315,7 +315,7 @@ func getNodeP2PAddress(t testing.TB, sut *SystemUnderTest, nodeHome string, rpcP // - jwtSecret: JWT secret for authenticating with EVM engine // - genesisHash: Hash of the genesis block for chain validation // - endpoints: TestEndpoints struct containing unique port assignments -func setupSequencerNode(t testing.TB, sut *SystemUnderTest, sequencerHome, jwtSecret, genesisHash string, endpoints *TestEndpoints, extraArgs ...string) { +func SetupSequencerNode(t testing.TB, sut *SystemUnderTest, sequencerHome, jwtSecret, genesisHash string, endpoints *TestEndpoints, extraArgs ...string) { t.Helper() // Create passphrase file @@ -531,7 +531,7 @@ func WithRethOpts(opts ...evmtest.RethNodeOpt) SetupOpt { // setupCommonEVMEnv creates and initializes ev-reth instances, while also initializing the local ev-node instance // managed by sut. If a full node is also required, we can use the WithFullNode() additional option. -func setupCommonEVMEnv(t testing.TB, sut *SystemUnderTest, client tastoratypes.TastoraDockerClient, networkID string, opts ...SetupOpt) *EVMEnv { +func SetupCommonEVMEnv(t testing.TB, sut *SystemUnderTest, client tastoratypes.TastoraDockerClient, networkID string, opts ...SetupOpt) *EVMEnv { t.Helper() // Configuration via functional options @@ -665,9 +665,9 @@ func setupSequencerOnlyTest(t testing.TB, sut *SystemUnderTest, nodeHome string, t.Helper() // Use common setup (no full node needed) - env := setupCommonEVMEnv(t, sut, client, networkID) + env := SetupCommonEVMEnv(t, sut, client, networkID) // Initialize and start sequencer node - setupSequencerNode(t, sut, nodeHome, env.SequencerJWT, env.GenesisHash, env.Endpoints, extraArgs...) + SetupSequencerNode(t, sut, nodeHome, env.SequencerJWT, env.GenesisHash, env.Endpoints, extraArgs...) t.Log("Sequencer node is up") return env.GenesisHash, env.Endpoints.GetSequencerEthURL() @@ -849,27 +849,23 @@ func verifyNoBlockProduction(t testing.TB, client *ethclient.Client, duration ti t.Logf("✅ %s maintained height %d for %v (no new blocks produced)", nodeName, initialHeight, duration) } -// traceSpan is a common interface for span data from different sources (OTLP collector, Jaeger API). -type traceSpan interface { +// TraceSpan is a common interface for span data from different sources (OTLP collector, Jaeger API). +type TraceSpan interface { SpanName() string SpanDuration() time.Duration } -// printTraceReport aggregates spans by operation name and prints a timing breakdown. -func printTraceReport(t testing.TB, label string, spans []traceSpan) { - t.Helper() - if len(spans) == 0 { - t.Logf("WARNING: no spans found for %s", label) - return - } +// SpanStats holds aggregated timing statistics for a single span operation. +type SpanStats struct { + Count int + Total time.Duration + Min time.Duration + Max time.Duration +} - type stats struct { - count int - total time.Duration - min time.Duration - max time.Duration - } - m := make(map[string]*stats) +// AggregateSpanStats groups spans by operation name and computes count, total, min, max. +func AggregateSpanStats(spans []TraceSpan) map[string]*SpanStats { + m := make(map[string]*SpanStats) for _, span := range spans { d := span.SpanDuration() if d <= 0 { @@ -878,45 +874,57 @@ func printTraceReport(t testing.TB, label string, spans []traceSpan) { name := span.SpanName() s, ok := m[name] if !ok { - s = &stats{min: d, max: d} + s = &SpanStats{Min: d, Max: d} m[name] = s } - s.count++ - s.total += d - if d < s.min { - s.min = d + s.Count++ + s.Total += d + if d < s.Min { + s.Min = d } - if d > s.max { - s.max = d + if d > s.Max { + s.Max = d } } + return m +} + +// PrintTraceReport aggregates spans by operation name and prints a timing breakdown. +func PrintTraceReport(t testing.TB, label string, spans []TraceSpan) { + t.Helper() + if len(spans) == 0 { + t.Logf("WARNING: no spans found for %s", label) + return + } + + m := AggregateSpanStats(spans) names := make([]string, 0, len(m)) for name := range m { names = append(names, name) } sort.Slice(names, func(i, j int) bool { - return m[names[i]].total > m[names[j]].total + return m[names[i]].Total > m[names[j]].Total }) var overallTotal time.Duration for _, s := range m { - overallTotal += s.total + overallTotal += s.Total } t.Logf("\n--- %s Trace Breakdown (%d spans) ---", label, len(spans)) t.Logf("%-40s %6s %12s %12s %12s %7s", "OPERATION", "COUNT", "AVG", "MIN", "MAX", "% TOTAL") for _, name := range names { s := m[name] - avg := s.total / time.Duration(s.count) - pct := float64(s.total) / float64(overallTotal) * 100 - t.Logf("%-40s %6d %12s %12s %12s %6.1f%%", name, s.count, avg, s.min, s.max, pct) + avg := s.Total / time.Duration(s.Count) + pct := float64(s.Total) / float64(overallTotal) * 100 + t.Logf("%-40s %6d %12s %12s %12s %6.1f%%", name, s.Count, avg, s.Min, s.Max, pct) } t.Logf("\n--- %s Time Distribution ---", label) for _, name := range names { s := m[name] - pct := float64(s.total) / float64(overallTotal) * 100 + pct := float64(s.Total) / float64(overallTotal) * 100 bar := "" for range int(pct / 2) { bar += "█" @@ -924,3 +932,4 @@ func printTraceReport(t testing.TB, label string, spans []traceSpan) { t.Logf("%-40s %5.1f%% %s", name, pct, bar) } } + diff --git a/test/e2e/failover_e2e_test.go b/test/e2e/failover_e2e_test.go index 905b280cf..26333a2dd 100644 --- a/test/e2e/failover_e2e_test.go +++ b/test/e2e/failover_e2e_test.go @@ -56,7 +56,7 @@ func TestLeaseFailoverE2E(t *testing.T) { // Get JWT secrets and setup common components first dockerClient, networkID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) // Use a fresh reth node on the same Docker network as used by the env setup. rethFn := evmtest.SetupTestRethNode(t, dockerClient, networkID) jwtSecret3 := rethFn.JWTSecretHex() @@ -257,7 +257,7 @@ func TestHASequencerRollingRestartE2E(t *testing.T) { // Get Docker and common environment dockerClient, networkID := tastoradocker.Setup(t) - env := setupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) + env := SetupCommonEVMEnv(t, sut, dockerClient, networkID, WithFullNode()) rethFn := evmtest.SetupTestRethNode(t, dockerClient, networkID) jwtSecret3 := rethFn.JWTSecretHex() fnInfo, err := rethFn.GetNetworkInfo(context.Background()) From 18fc15a6a4a72c8c2f153dff9f2a749728f0010c Mon Sep 17 00:00:00 2001 From: chatton Date: Wed, 25 Feb 2026 12:40:33 +0000 Subject: [PATCH 2/5] fix: correct BENCH_JSON_OUTPUT path for spamoor benchmark go test sets the working directory to the package under test, so the env var should be relative to test/e2e/benchmark/, not test/e2e/. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2f8fcdb1c..bb2bd296d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -63,7 +63,7 @@ jobs: run: make build-evm build-da - name: Run Spamoor smoke test run: | - cd test/e2e && BENCH_JSON_OUTPUT=benchmark/spamoor_bench.json go test -tags evm \ + cd test/e2e && BENCH_JSON_OUTPUT=spamoor_bench.json go test -tags evm \ -run='^TestSpamoorSuite$/^TestSpamoorSmoke$' -v -timeout=15m \ --evm-binary=../../build/evm ./benchmark/ - name: Upload benchmark results From fccd9db21a00243d01b821ae8adb7f7ddc75c5f9 Mon Sep 17 00:00:00 2001 From: chatton Date: Wed, 25 Feb 2026 12:50:50 +0000 Subject: [PATCH 3/5] fix: place package pattern before test binary flags in benchmark CI go test treats all arguments after an unknown flag (--evm-binary) as test binary args, so ./benchmark/ was never recognized as a package pattern. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index bb2bd296d..c4be10227 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -65,7 +65,7 @@ jobs: run: | cd test/e2e && BENCH_JSON_OUTPUT=spamoor_bench.json go test -tags evm \ -run='^TestSpamoorSuite$/^TestSpamoorSmoke$' -v -timeout=15m \ - --evm-binary=../../build/evm ./benchmark/ + ./benchmark/ --evm-binary=../../build/evm - name: Upload benchmark results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: From ae525cae6fe048844f4fa5eb01180c0a534a7f5c Mon Sep 17 00:00:00 2001 From: chatton Date: Wed, 25 Feb 2026 13:48:41 +0000 Subject: [PATCH 4/5] fix: adjust evm-binary path for benchmark subpackage working directory go test sets the cwd to the package directory (test/e2e/benchmark/), so the binary path needs an extra parent traversal. --- .github/workflows/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c4be10227..b3a157a0b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -65,7 +65,7 @@ jobs: run: | cd test/e2e && BENCH_JSON_OUTPUT=spamoor_bench.json go test -tags evm \ -run='^TestSpamoorSuite$/^TestSpamoorSmoke$' -v -timeout=15m \ - ./benchmark/ --evm-binary=../../build/evm + ./benchmark/ --evm-binary=../../../build/evm - name: Upload benchmark results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: From 85c9d2d693fa32a7195598562d3cfaeadae90e35 Mon Sep 17 00:00:00 2001 From: chatton Date: Wed, 25 Feb 2026 14:45:37 +0000 Subject: [PATCH 5/5] fix: exclude benchmark subpackage from make test-e2e The benchmark package doesn't define the --binary flag that test-e2e passes. It has its own CI workflow so it doesn't need to run here. --- scripts/test.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.mk b/scripts/test.mk index f9f76c1ab..524f8a4cd 100644 --- a/scripts/test.mk +++ b/scripts/test.mk @@ -24,7 +24,7 @@ test-integration: ## test-e2e: Running e2e tests test-e2e: build build-da build-evm docker-build-if-local @echo "--> Running e2e tests" - @cd test/e2e && go test -mod=readonly -failfast -timeout=15m -tags='e2e evm' ./... --binary=../../build/testapp --evm-binary=../../build/evm + @cd test/e2e && go test -mod=readonly -failfast -timeout=15m -tags='e2e evm' $$(go list -tags='e2e evm' ./... | grep -v /benchmark) --binary=../../build/testapp --evm-binary=../../build/evm .PHONY: test-e2e ## test-integration-cover: generate code coverage report for integration tests.