open-telemetry · lmolkova · May 18, 2026 · May 14, 2026 · May 14, 2026 · May 14, 2026
@@ -69,6 +69,12 @@ prefer opt-in or additive. Breaking changes need explicit justification in the P
   (`from opentelemetry.test_util_genai.fixtures import *` and
   `from opentelemetry.test_util_genai.vcr import fixture_vcr, scrub_response_headers`). Do not
   re-implement in-memory provider/exporter setup or the VCR pretty-print serializer locally.
+- Conformance: packages ship `tests/conformance/<scenario>.py` modules (each
+  defining a subclass of
+  `opentelemetry.test_util_genai.conformance.Scenario` that sets
+  `expected_spans`, `expected_metrics`, and implements `run(...)`) and a
+  `tests/test_conformance.py` that runs them via
+  `opentelemetry.test_util_genai.conformance.run_conformance`.
 
 ## 6. Examples
 

@@ -13,6 +13,18 @@
     // requirements.latest.txt files are skipped without this.
     "managerFilePatterns": ["/(^|/)requirements\\.latest\\.txt$/"]
   },
+  // Manage WEAVER_VERSION and SEMCONV_GENAI_REF in versions.env via their
+  // `# renovate:` annotations.
+  "customManagers": [
+    {
+      "customType": "regex",
+      "managerFilePatterns": ["/^versions\\.env$/"],
+      "matchStrings": [
+        "# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+) versioning=(?<versioning>\\S+)\\s+WEAVER_VERSION=(?<currentValue>\\S+)",
+        "# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+) packageName=(?<packageName>\\S+) versioning=(?<versioning>\\S+)\\s+SEMCONV_GENAI_REF=(?<currentValue>\\S+)"
+      ]
+    }
+  ],
   "packageRules": [
     {
       "groupName": "all patch versions",

@@ -88,6 +88,7 @@ def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:
                     "python_version": aliased_python_version,
                     "tox_env": tox_env,
                     "os": operating_system,
+                    "needs_weaver": tox_env.endswith("-conformance"),
                 }
             )
 

@@ -45,6 +45,47 @@ jobs:
       - name: Configure git to support long filenames
         run: git config --system core.longpaths true
       {%- endif %}
+      {%- if job_data.needs_weaver %}
+
+      - name: Read pinned weaver version
+        id: versions
+        {%- if job_data.os == "windows-latest" %}
+        shell: pwsh
+        run: |
+          Select-String -Path versions.env -Pattern '^WEAVER_VERSION=' | ForEach-Object { $_.Line } | Add-Content $env:GITHUB_OUTPUT
+        {%- else %}
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+        {%- endif %}
+
+      - name: Install weaver ${% raw %}{{ steps.versions.outputs.WEAVER_VERSION }}{% endraw %}
+        env:
+          WEAVER_VERSION: ${% raw %}{{ steps.versions.outputs.WEAVER_VERSION }}{% endraw %}
+        {%- if job_data.os == "windows-latest" %}
+        shell: pwsh
+        run: |
+          $url = "https://github.com/open-telemetry/weaver/releases/download/$env:WEAVER_VERSION/weaver-x86_64-pc-windows-msvc.zip"
+          Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\weaver.zip"
+          Expand-Archive -Path "$env:RUNNER_TEMP\weaver.zip" -DestinationPath "$env:RUNNER_TEMP\weaver"
+          "$env:RUNNER_TEMP\weaver" | Add-Content $env:GITHUB_PATH
+          & "$env:RUNNER_TEMP\weaver\weaver.exe" --version
+        {%- elif job_data.os == "macos-latest" %}
+        run: |
+          WEAVER_ASSET="weaver-aarch64-apple-darwin"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+        {%- else %}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+        {%- endif %}
+      {%- endif %}
 
       - name: Run tests
         run: tox -e {{ job_data.tox_env }} -- -ra

@@ -252,6 +252,76 @@ jobs:
       - name: Run tests
         run: tox -e pypy3-test-instrumentation-openai-v2-latest -- -ra
 
+  py312-test-instrumentation-openai-v2-conformance_ubuntu-latest:
+    name: instrumentation-openai-v2-conformance 3.12 Ubuntu
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install tox
+        run: pip install tox-uv
+
+      - name: Read pinned weaver version
+        id: versions
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+
+      - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
+        env:
+          WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+
+      - name: Run tests
+        run: tox -e py312-test-instrumentation-openai-v2-conformance -- -ra
+
+  py313-test-instrumentation-openai-v2-conformance_ubuntu-latest:
+    name: instrumentation-openai-v2-conformance 3.13 Ubuntu
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.13"
+
+      - name: Install tox
+        run: pip install tox-uv
+
+      - name: Read pinned weaver version
+        id: versions
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+
+      - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
+        env:
+          WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+
+      - name: Run tests
+        run: tox -e py313-test-instrumentation-openai-v2-conformance -- -ra
+
   py310-test-instrumentation-openai_agents-v2-oldest_ubuntu-latest:
     name: instrumentation-openai_agents-v2-oldest 3.10 Ubuntu
     runs-on: ubuntu-latest
@@ -822,6 +892,76 @@ jobs:
       - name: Run tests
         run: tox -e py314-test-instrumentation-anthropic-latest -- -ra
 
+  py312-test-instrumentation-anthropic-conformance_ubuntu-latest:
+    name: instrumentation-anthropic-conformance 3.12 Ubuntu
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install tox
+        run: pip install tox-uv
+
+      - name: Read pinned weaver version
+        id: versions
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+
+      - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
+        env:
+          WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+
+      - name: Run tests
+        run: tox -e py312-test-instrumentation-anthropic-conformance -- -ra
+
+  py313-test-instrumentation-anthropic-conformance_ubuntu-latest:
+    name: instrumentation-anthropic-conformance 3.13 Ubuntu
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.13"
+
+      - name: Install tox
+        run: pip install tox-uv
+
+      - name: Read pinned weaver version
+        id: versions
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+
+      - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
+        env:
+          WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+
+      - name: Run tests
+        run: tox -e py313-test-instrumentation-anthropic-conformance -- -ra
+
   py310-test-instrumentation-claude-agent-sdk-oldest_ubuntu-latest:
     name: instrumentation-claude-agent-sdk-oldest 3.10 Ubuntu
     runs-on: ubuntu-latest
@@ -974,6 +1114,76 @@ jobs:
       - name: Run tests
         run: tox -e py313-test-instrumentation-claude-agent-sdk-latest -- -ra
 
+  py312-test-instrumentation-langchain-conformance_ubuntu-latest:
+    name: instrumentation-langchain-conformance 3.12 Ubuntu
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.12"
+
+      - name: Install tox
+        run: pip install tox-uv
+
+      - name: Read pinned weaver version
+        id: versions
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+
+      - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
+        env:
+          WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+
+      - name: Run tests
+        run: tox -e py312-test-instrumentation-langchain-conformance -- -ra
+
+  py313-test-instrumentation-langchain-conformance_ubuntu-latest:
+    name: instrumentation-langchain-conformance 3.13 Ubuntu
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repo @ SHA - ${{ github.sha }}
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Set up Python 3.13
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.13"
+
+      - name: Install tox
+        run: pip install tox-uv
+
+      - name: Read pinned weaver version
+        id: versions
+        run: |
+          # shellcheck disable=SC2002
+          cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
+
+      - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
+        env:
+          WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
+        run: |
+          WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
+          WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
+          curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
+          sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
+          weaver --version
+
+      - name: Run tests
+        run: tox -e py313-test-instrumentation-langchain-conformance -- -ra
+
   py310-test-util-genai_ubuntu-latest:
     name: util-genai 3.10 Ubuntu
     runs-on: ubuntu-latest

@@ -66,3 +66,10 @@ target
 opentelemetry-admin-jobs.txt
 
 .claude/settings.local.json
+
+# Generated by test_util_genai._setup_weaver
+policies/_schemas.rego
+.build/
+
+# Conformance test weaver live-check reports
+weaver_reports/
@@ -109,6 +109,21 @@ Apply to packages under `instrumentation/`.
   `from opentelemetry.test_util_genai.vcr import fixture_vcr, scrub_response_headers`) rather
   than re-implementing provider/exporter/VCR plumbing.
 
+### Conformance tests
+
+Packages with substantive instrumentation ship `tests/conformance/<scenario>.py`
+scenarios and a `tests/test_conformance.py` that validates emitted telemetry
+against the [GenAI semantic conventions](https://github.com/open-telemetry/semantic-conventions-genai)
+via Weaver live-check. Each scenario module defines a subclass of
+`opentelemetry.test_util_genai.conformance.Scenario` that sets
+`expected_spans`, `expected_metrics`, and implements
+`run(*, tracer_provider, meter_provider, logger_provider, vcr)`.
+
+`tests/test_conformance.py` must set `pytestmark = pytest.mark.conformance` at
+module level.
+
+Run via `tox -e py312-test-instrumentation-<pkg>-conformance`.
+
 The parallel PR-review rules live in
 [`.github/instructions/instrumentation.instructions.md`](.github/instructions/instrumentation.instructions.md)
 and should be kept in sync with this section.

@@ -66,6 +66,27 @@ Run type checking across the workspace:
 uv run tox -e typecheck
 ```
 
+#### Managing cassettes (test recordings)
+
+GenAI tests replay recorded HTTP interactions (cassettes) stored under each
+package's `tests/cassettes/`.
+
+- **Run**: nothing extra — cassettes replay automatically when present. Tests
+  that need a cassette skip if it is missing and no real API key is set.
+- **Record**: delete the target `tests/cassettes/<test_name>.yaml`, export a
+  real provider API key (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`), and
+  rerun the test. `pytest-vcr` writes the cassette on the live call.
+- **Sanitize**: every package's `vcr_config()` in `tests/conftest.py` must
+  scrub auth via `filter_headers` and strip identifying response headers via
+  `scrub_response_headers(...)` from `opentelemetry.test_util_genai.vcr`.
+  Diff each new cassette before committing — leaked API keys, org ids, or
+  `Set-Cookie` values block the PR.
+- **AI-generated cassettes**: if you lack provider access, you may
+  synthesize a cassette from the provider's API reference via AI. Make sure
+  to mention it in the PR and open a follow-up issue to re-record it in CI
+  against the real provider.
+- **CI**: replay-only; recording in CI is a future improvement.
+
 ### 4. Update the changelog
 
 This repo uses [towncrier](https://towncrier.readthedocs.io/) to manage
@@ -136,3 +157,4 @@ For more information about the maintainer role, see the [community repository](h
 - [Leighton Chen](https://github.com/lzchen), Microsoft
 
 For more information about the approver role, see the [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#approver).
+
@@ -0,0 +1,14 @@
+# Conformance tests use `opentelemetry.test.weaver_live_check`, not yet on
+# PyPI. Pin the whole stack from git so versions stay coherent. Mirrors
+# pyproject.toml's [tool.uv.sources]. Drop once the file ships to PyPI.
+#
+# TODO: switch to PyPI versions once the test utils are released
+
+opentelemetry-api @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-api
+opentelemetry-sdk @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-sdk
+opentelemetry-semantic-conventions @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-semantic-conventions
+opentelemetry-test-utils @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=tests/opentelemetry-test-utils
+opentelemetry-instrumentation @ git+https://github.com/open-telemetry/opentelemetry-python-contrib@d2f396de68a969dfb74b8afc247e1d0dc6739b67#subdirectory=opentelemetry-instrumentation
+opentelemetry-exporter-otlp-proto-grpc @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=exporter/opentelemetry-exporter-otlp-proto-grpc
+opentelemetry-exporter-otlp-proto-common @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=exporter/opentelemetry-exporter-otlp-proto-common
+opentelemetry-proto @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-proto