chore: gather spans from both ev-node and ev-reth in spamoor test (#3099)

* wip: passing test, but hacky

* chore: refactor test setup to allow for injection of docker client

* chore: fix test compilation errors, removed useless comments

* chore: addresing PR feedback

* chore: create common trace printing code

* chore: adding assertions on addtional spans

* deps: tidy all

* chore: address PR feedback

* chore: removed unnessesdary test

* chore: removed unused imports

* chore: removed non-existant trace

Author: chatton

test/e2e/evm_contract_bench_test.go

@@ -11,7 +11,6 @@ import (
 	"net"
 	"net/http"
 	"path/filepath"
-	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -203,81 +202,24 @@ func (c *otlpCollector) getSpans() []*tracepb.Span {
 	return cp
 }
 
-// printCollectedTraceReport aggregates collected spans by operation name and
-// prints a timing breakdown.
-func printCollectedTraceReport(b testing.TB, collector *otlpCollector) {
-	b.Helper()
-
-	spans := collector.getSpans()
-	if len(spans) == 0 {
-		b.Logf("WARNING: no spans collected from ev-node")
-		return
-	}
-
-	type stats struct {
-		count int
-		total time.Duration
-		min   time.Duration
-		max   time.Duration
-	}
-	m := make(map[string]*stats)
-
-	for _, span := range spans {
-		// Duration: end - start in nanoseconds.
-		d := time.Duration(span.GetEndTimeUnixNano()-span.GetStartTimeUnixNano()) * time.Nanosecond
-		if d <= 0 {
-			continue
-		}
-		name := span.GetName()
-		s, ok := m[name]
-		if !ok {
-			s = &stats{min: d, max: d}
-			m[name] = s
-		}
-		s.count++
-		s.total += d
-		if d < s.min {
-			s.min = d
-		}
-		if d > s.max {
-			s.max = d
-		}
-	}
-
-	// Sort by total time descending.
-	names := make([]string, 0, len(m))
-	for name := range m {
-		names = append(names, name)
-	}
-	sort.Slice(names, func(i, j int) bool {
-		return m[names[i]].total > m[names[j]].total
-	})
-
-	// Calculate overall total for percentages.
-	var overallTotal time.Duration
-	for _, s := range m {
-		overallTotal += s.total
-	}
+// otlpSpanAdapter wraps an OTLP protobuf span to implement traceSpan.
+type otlpSpanAdapter struct {
+	span *tracepb.Span
+}
 
-	b.Logf("\n--- ev-node Trace Breakdown (%d spans collected) ---", len(spans))
-	b.Logf("%-40s %6s %12s %12s %12s %7s", "OPERATION", "COUNT", "AVG", "MIN", "MAX", "% TOTAL")
-	for _, name := range names {
-		s := m[name]
-		avg := s.total / time.Duration(s.count)
-		pct := float64(s.total) / float64(overallTotal) * 100
-		b.Logf("%-40s %6d %12s %12s %12s %6.1f%%", name, s.count, avg, s.min, s.max, pct)
-	}
+func (a otlpSpanAdapter) SpanName() string { return a.span.GetName() }
+func (a otlpSpanAdapter) SpanDuration() time.Duration {
+	return time.Duration(a.span.GetEndTimeUnixNano()-a.span.GetStartTimeUnixNano()) * time.Nanosecond
+}
 
-	b.Logf("\n--- Time Distribution ---")
-	for _, name := range names {
-		s := m[name]
-		pct := float64(s.total) / float64(overallTotal) * 100
-		bar := ""
-		for range int(pct / 2) {
-			bar += "█"
-		}
-		b.Logf("%-40s %5.1f%% %s", name, pct, bar)
+func printCollectedTraceReport(b testing.TB, collector *otlpCollector) {
+	b.Helper()
+	raw := collector.getSpans()
+	spans := make([]traceSpan, len(raw))
+	for i, s := range raw {
+		spans[i] = otlpSpanAdapter{span: s}
 	}
+	printTraceReport(b, "ev-node", spans)
 }
 
 // waitForReceipt polls for a transaction receipt until it is available.

test/e2e/evm_spamoor_smoke_test.go

@@ -11,9 +11,12 @@ import (
 	"time"
 
 	tastoradocker "github.com/celestiaorg/tastora/framework/docker"
+	reth "github.com/celestiaorg/tastora/framework/docker/evstack/reth"
 	spamoor "github.com/celestiaorg/tastora/framework/docker/evstack/spamoor"
+	jaeger "github.com/celestiaorg/tastora/framework/docker/jaeger"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
 )
 
 // TestSpamoorSmoke spins up reth + sequencer and a Spamoor node, starts a few
@@ -22,28 +25,42 @@ func TestSpamoorSmoke(t *testing.T) {
 	t.Parallel()
 
 	sut := NewSystemUnderTest(t)
-	// Bring up reth + local DA and start sequencer with default settings.
+	// Prepare a shared docker client and network for Jaeger and reth.
+	ctx := t.Context()
 	dcli, netID := tastoradocker.Setup(t)
-	env := setupCommonEVMEnv(t, sut, dcli, netID)
+	jcfg := jaeger.Config{Logger: zaptest.NewLogger(t), DockerClient: dcli, DockerNetworkID: netID}
+	jg, err := jaeger.New(ctx, jcfg, t.Name(), 0)
+	require.NoError(t, err, "failed to create jaeger node")
+	t.Cleanup(func() { _ = jg.Remove(t.Context()) })
+	require.NoError(t, jg.Start(ctx), "failed to start jaeger node")
+
+	// Bring up reth + local DA on the same docker network as Jaeger so reth can export traces.
+	env := setupCommonEVMEnv(t, sut, dcli, netID,
+		WithRethOpts(func(b *reth.NodeBuilder) {
+			b.WithEnv(
+				"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="+jg.Internal.IngestHTTPEndpoint()+"/v1/traces",
+				"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http",
+				"RUST_LOG=info",
+				"OTEL_SDK_DISABLED=false",
+			)
+		}),
+	)
 	sequencerHome := filepath.Join(t.TempDir(), "sequencer")
 
-	// In-process OTLP/HTTP collector to capture ev-node spans.
-	collector := newOTLPCollector(t)
-	t.Cleanup(func() {
-		_ = collector.close()
-	})
+	// ev-node runs on the host, so use Jaeger's external OTLP/HTTP endpoint.
+	otlpHTTP := jg.External.IngestHTTPEndpoint()
 
-	// Start sequencer with tracing to our collector.
+	// Start sequencer with tracing to Jaeger collector.
 	setupSequencerNode(t, sut, sequencerHome, env.SequencerJWT, env.GenesisHash, env.Endpoints,
 		"--evnode.instrumentation.tracing=true",
-		"--evnode.instrumentation.tracing_endpoint", collector.endpoint(),
+		"--evnode.instrumentation.tracing_endpoint", otlpHTTP,
 		"--evnode.instrumentation.tracing_sample_rate", "1.0",
 		"--evnode.instrumentation.tracing_service_name", "ev-node-smoke",
 	)
 	t.Log("Sequencer node is up")
 
 	// Start Spamoor within the same Docker network, targeting reth internal RPC.
-	ni, err := env.RethNode.GetNetworkInfo(context.Background())
+	ni, err := env.RethNode.GetNetworkInfo(ctx)
 	require.NoError(t, err, "failed to get network info")
 
 	internalRPC := "http://" + ni.Internal.RPCAddress()
@@ -55,7 +72,6 @@ func TestSpamoorSmoke(t *testing.T) {
 		WithRPCHosts(internalRPC).
 		WithPrivateKey(TestPrivateKey)
 
-	ctx := t.Context()
 	spNode, err := spBuilder.Build(ctx)
 	require.NoError(t, err, "failed to build sp node")
 
@@ -113,7 +129,8 @@ func TestSpamoorSmoke(t *testing.T) {
 		t.Cleanup(func() { _ = api.DeleteSpammer(idToDelete) })
 	}
 
-	// Allow additional time to accumulate activity.
+	// allow spamoor enough time to generate transaction throughput
+	// so that the expected tracing spans appear in Jaeger.
 	time.Sleep(60 * time.Second)
 
 	// Fetch parsed metrics and print a concise summary.
@@ -122,8 +139,61 @@ func TestSpamoorSmoke(t *testing.T) {
 	sent := sumCounter(metrics["spamoor_transactions_sent_total"])
 	fail := sumCounter(metrics["spamoor_transactions_failed_total"])
 
-	time.Sleep(2 * time.Second)
-	printCollectedTraceReport(t, collector)
+	// Verify Jaeger received traces from ev-node.
+	// Service name is set above via --evnode.instrumentation.tracing_service_name "ev-node-smoke".
+	traceCtx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+	ok, err := jg.External.WaitForTraces(traceCtx, "ev-node-smoke", 1, 2*time.Second)
+	require.NoError(t, err, "error while waiting for Jaeger traces; UI: %s", jg.External.QueryURL())
+	require.True(t, ok, "expected at least one trace in Jaeger; UI: %s", jg.External.QueryURL())
+
+	// Also wait for traces from ev-reth and print a small sample.
+	ok, err = jg.External.WaitForTraces(traceCtx, "ev-reth", 1, 2*time.Second)
+	require.NoError(t, err, "error while waiting for ev-reth traces; UI: %s", jg.External.QueryURL())
+	require.True(t, ok, "expected at least one trace from ev-reth; UI: %s", jg.External.QueryURL())
+
+	// fetch traces and print reports for both services.
+	// use a large limit to fetch all traces from the test run.
+	evNodeTraces, err := jg.External.Traces(traceCtx, "ev-node-smoke", 10000)
+	require.NoError(t, err, "failed to fetch ev-node-smoke traces from Jaeger")
+	evNodeSpans := extractSpansFromTraces(evNodeTraces)
+	printTraceReport(t, "ev-node-smoke", toTraceSpans(evNodeSpans))
+
+	evRethTraces, err := jg.External.Traces(traceCtx, "ev-reth", 10000)
+	require.NoError(t, err, "failed to fetch ev-reth traces from Jaeger")
+	evRethSpans := extractSpansFromTraces(evRethTraces)
+	printTraceReport(t, "ev-reth", toTraceSpans(evRethSpans))
+
+	// assert expected ev-node span names are present.
+	// these spans reliably appear during block production with transactions flowing.
+	expectedSpans := []string{
+		"BlockExecutor.ProduceBlock",
+		"BlockExecutor.ApplyBlock",
+		"BlockExecutor.CreateBlock",
+		"BlockExecutor.RetrieveBatch",
+		"Executor.ExecuteTxs",
+		"Executor.SetFinal",
+		"Engine.ForkchoiceUpdated",
+		"Engine.NewPayload",
+		"Engine.GetPayload",
+		"Eth.GetBlockByNumber",
+		"Sequencer.GetNextBatch",
+		"DASubmitter.SubmitHeaders",
+		"DASubmitter.SubmitData",
+		"DA.Submit",
+	}
+	opNames := make(map[string]struct{}, len(evNodeSpans))
+	for _, s := range evNodeSpans {
+		opNames[s.operationName] = struct{}{}
+	}
+	for _, name := range expectedSpans {
+		require.Contains(t, opNames, name, "expected span %q not found in ev-node-smoke traces", name)
+	}
+
+	// ev-reth span names are internal to the Rust OTLP exporter and may change
+	// across versions, so we only assert that spans were collected at all.
+	// TODO: check for more specific spans once implemented.
+	require.NotEmpty(t, evRethSpans, "expected at least one span from ev-reth")
 
 	require.Greater(t, sent, float64(0), "at least one transaction should have been sent")
 	require.Zero(t, fail, "no transactions should have failed")
@@ -165,15 +235,47 @@ func sumCounter(f *dto.MetricFamily) float64 {
 	}
 	return sum
 }
-func sumGauge(f *dto.MetricFamily) float64 {
-	if f == nil || f.GetType() != dto.MetricType_GAUGE {
-		return 0
-	}
-	var sum float64
-	for _, m := range f.GetMetric() {
-		if m.GetGauge() != nil && m.GetGauge().Value != nil {
-			sum += m.GetGauge().GetValue()
+
+// jaegerSpan holds the fields we extract from Jaeger's untyped JSON response.
+type jaegerSpan struct {
+	operationName string
+	duration      float64 // microseconds
+}
+
+func (j jaegerSpan) SpanName() string            { return j.operationName }
+func (j jaegerSpan) SpanDuration() time.Duration { return time.Duration(j.duration) * time.Microsecond }
+
+// extractSpansFromTraces walks Jaeger's []any response and pulls out span operation names and durations.
+func extractSpansFromTraces(traces []any) []jaegerSpan {
+	var out []jaegerSpan
+	for _, t := range traces {
+		traceMap, ok := t.(map[string]any)
+		if !ok {
+			continue
+		}
+		spans, ok := traceMap["spans"].([]any)
+		if !ok {
+			continue
+		}
+		for _, s := range spans {
+			spanMap, ok := s.(map[string]any)
+			if !ok {
+				continue
+			}
+			name, _ := spanMap["operationName"].(string)
+			dur, _ := spanMap["duration"].(float64)
+			if name != "" {
+				out = append(out, jaegerSpan{operationName: name, duration: dur})
+			}
 		}
 	}
-	return sum
+	return out
+}
+
+func toTraceSpans(spans []jaegerSpan) []traceSpan {
+	out := make([]traceSpan, len(spans))
+	for i, s := range spans {
+		out[i] = s
+	}
+	return out
 }