Fixes and enable rust for all e2e tests

Author: jy-tan

.github/workflows/e2e.yml

@@ -103,6 +103,7 @@ jobs:
         env:
           DOCKER_DEFAULT_PLATFORM: linux/amd64
           TUSK_CLI_VERSION: ${{ steps.tusk-version.outputs.version }}
+          TUSK_USE_RUST_CORE: "1"
         run: |
           chmod +x ./drift/instrumentation/${{ matrix.library }}/e2e-tests/run.sh
           cd ./drift/instrumentation/${{ matrix.library }}/e2e-tests && ./run.sh 8000
@@ -180,6 +181,7 @@ jobs:
         env:
           DOCKER_DEFAULT_PLATFORM: linux/amd64
           TUSK_CLI_VERSION: ${{ steps.tusk-version.outputs.version }}
+          TUSK_USE_RUST_CORE: "1"
         run: |
           chmod +x ./drift/stack-tests/${{ matrix.test }}/run.sh
           cd ./drift/stack-tests/${{ matrix.test }} && ./run.sh 8000

drift/core/json_schema_helper.py

@@ -160,13 +160,23 @@ def generate_schema_and_hash(data: Any, schema_merges: SchemaMerges | None = Non
 
             decoded = JsonSchemaHelper._decode_with_merges(normalized, schema_merges)
             rust_decoded_hash = deterministic_hash_jsonable(decoded)
-            decoded_value_hash = rust_decoded_hash or JsonSchemaHelper.generate_deterministic_hash(decoded)
+            if rust_decoded_hash is not None:
+                decoded_value_hash = rust_decoded_hash
+            else:
+                sorted_decoded = JsonSchemaHelper._sort_object_keys(decoded)
+                payload = json.dumps(sorted_decoded, ensure_ascii=False, separators=(",", ":"))
+                decoded_value_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest()
 
         schema = JsonSchemaHelper.generate_schema(decoded, schema_merges)
 
         schema_primitive = schema.to_primitive()
         rust_schema_hash = deterministic_hash_jsonable(schema_primitive)
-        decoded_schema_hash = rust_schema_hash or JsonSchemaHelper.generate_deterministic_hash(schema_primitive)
+        if rust_schema_hash is not None:
+            decoded_schema_hash = rust_schema_hash
+        else:
+            sorted_schema = JsonSchemaHelper._sort_object_keys(schema_primitive)
+            payload = json.dumps(sorted_schema, ensure_ascii=False, separators=(",", ":"))
+            decoded_schema_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest()
         return SchemaComputationResult(
             schema=schema,
             decoded_value_hash=decoded_value_hash,

drift/core/span_serialization.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from datetime import datetime, timedelta, timezone
 from typing import Any
 
@@ -178,4 +179,7 @@ def _metadata_to_dict(metadata: Any) -> dict[str, Any]:
     if metadata is None:
         return {}
 
+    if isinstance(metadata, Mapping):
+        return {str(key): value for key, value in metadata.items() if value is not None}
+
     return {key: value for key, value in metadata.__dict__.items() if value is not None}

drift/core/tracing/otel_converter.py

@@ -455,16 +455,11 @@ def otel_span_to_clean_span_data(
         output_value_proto_struct_bytes=output_proto_bytes,
     )
 
-    # Avoid expensive schema object tree conversion when we already have prebuilt
-    # full Span bytes from Rust; those schema objects would not be used on hot path.
-    from ..json_schema_helper import JsonSchema
-
-    if span_proto_bytes is not None:
-        input_schema_obj = JsonSchema()
-        output_schema_obj = JsonSchema()
-    else:
-        input_schema_obj = dict_to_json_schema(input_schema)
-        output_schema_obj = dict_to_json_schema(output_schema)
+    # Keep schema objects populated for all export paths (API + filesystem).
+    # Even when Rust prebuilds Span bytes for API export, other sinks still
+    # serialize from `CleanSpanData` fields.
+    input_schema_obj = dict_to_json_schema(input_schema)
+    output_schema_obj = dict_to_json_schema(output_schema)
 
     # Build CleanSpanData (note: no input_schema_merges or output_schema_merges fields)
     return CleanSpanData(

drift/instrumentation/aiohttp/e2e-tests/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - PORT=8000
       - TUSK_ANALYTICS_DISABLED=1
       - PYTHONUNBUFFERED=1
+      - TUSK_USE_RUST_CORE=${TUSK_USE_RUST_CORE:-0}
       - BENCHMARKS=${BENCHMARKS:-}
       - BENCHMARK_DURATION=${BENCHMARK_DURATION:-10}
       - BENCHMARK_WARMUP=${BENCHMARK_WARMUP:-3}

drift/instrumentation/grpc/e2e-tests/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - PORT=8000
       - TUSK_ANALYTICS_DISABLED=1
       - PYTHONUNBUFFERED=1
+      - TUSK_USE_RUST_CORE=${TUSK_USE_RUST_CORE:-0}
       - BENCHMARKS=${BENCHMARKS:-}
       - BENCHMARK_DURATION=${BENCHMARK_DURATION:-10}
       - BENCHMARK_WARMUP=${BENCHMARK_WARMUP:-3}

drift/instrumentation/httpx/e2e-tests/docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - PORT=8000
       - TUSK_ANALYTICS_DISABLED=1
       - PYTHONUNBUFFERED=1
+      - TUSK_USE_RUST_CORE=${TUSK_USE_RUST_CORE:-0}
       - BENCHMARKS=${BENCHMARKS:-}
       - BENCHMARK_DURATION=${BENCHMARK_DURATION:-10}
       - BENCHMARK_WARMUP=${BENCHMARK_WARMUP:-3}

drift/instrumentation/psycopg/e2e-tests/docker-compose.yml

@@ -29,6 +29,7 @@ services:
       - POSTGRES_PASSWORD=testpass
       - TUSK_ANALYTICS_DISABLED=1
       - PYTHONUNBUFFERED=1
+      - TUSK_USE_RUST_CORE=${TUSK_USE_RUST_CORE:-0}
       - BENCHMARKS=${BENCHMARKS:-}
       - BENCHMARK_DURATION=${BENCHMARK_DURATION:-10}
       - BENCHMARK_WARMUP=${BENCHMARK_WARMUP:-3}

drift/instrumentation/psycopg2/e2e-tests/docker-compose.yml

@@ -29,6 +29,7 @@ services:
       - POSTGRES_PASSWORD=testpass
       - TUSK_ANALYTICS_DISABLED=1
       - PYTHONUNBUFFERED=1
+      - TUSK_USE_RUST_CORE=${TUSK_USE_RUST_CORE:-0}
       - BENCHMARKS=${BENCHMARKS:-}
       - BENCHMARK_DURATION=${BENCHMARK_DURATION:-10}
       - BENCHMARK_WARMUP=${BENCHMARK_WARMUP:-3}

drift/instrumentation/redis/e2e-tests/docker-compose.yml

@@ -22,6 +22,7 @@ services:
       - REDIS_PORT=6379
       - TUSK_ANALYTICS_DISABLED=1
       - PYTHONUNBUFFERED=1
+      - TUSK_USE_RUST_CORE=${TUSK_USE_RUST_CORE:-0}
       - BENCHMARKS=${BENCHMARKS:-}
       - BENCHMARK_DURATION=${BENCHMARK_DURATION:-10}
       - BENCHMARK_WARMUP=${BENCHMARK_WARMUP:-3}