diff --git a/.github/instructions/instrumentation.instructions.md b/.github/instructions/instrumentation.instructions.md index 717170ee..4163838b 100644 --- a/.github/instructions/instrumentation.instructions.md +++ b/.github/instructions/instrumentation.instructions.md @@ -69,6 +69,12 @@ prefer opt-in or additive. Breaking changes need explicit justification in the P (`from opentelemetry.test_util_genai.fixtures import *` and `from opentelemetry.test_util_genai.vcr import fixture_vcr, scrub_response_headers`). Do not re-implement in-memory provider/exporter setup or the VCR pretty-print serializer locally. +- Conformance: packages ship `tests/conformance/.py` modules (each + defining a subclass of + `opentelemetry.test_util_genai.conformance.Scenario` that sets + `expected_spans`, `expected_metrics`, and implements `run(...)`) and a + `tests/test_conformance.py` that runs them via + `opentelemetry.test_util_genai.conformance.run_conformance`. ## 6. Examples diff --git a/.github/renovate.json5 b/.github/renovate.json5 index af03085a..4a0cf6a7 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -13,6 +13,18 @@ // requirements.latest.txt files are skipped without this. "managerFilePatterns": ["/(^|/)requirements\\.latest\\.txt$/"] }, + // Manage WEAVER_VERSION and SEMCONV_GENAI_REF in versions.env via their + // `# renovate:` annotations. + "customManagers": [ + { + "customType": "regex", + "managerFilePatterns": ["/^versions\\.env$/"], + "matchStrings": [ + "# renovate: datasource=(?\\S+) depName=(?\\S+) versioning=(?\\S+)\\s+WEAVER_VERSION=(?\\S+)", + "# renovate: datasource=(?\\S+) depName=(?\\S+) packageName=(?\\S+) versioning=(?\\S+)\\s+SEMCONV_GENAI_REF=(?\\S+)" + ] + } + ], "packageRules": [ { "groupName": "all patch versions", diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py index 054508fa..5843196f 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py @@ -88,6 +88,7 @@ def get_test_job_datas(tox_envs: list, operating_systems: list) -> list: "python_version": aliased_python_version, "tox_env": tox_env, "os": operating_system, + "needs_weaver": tox_env.endswith("-conformance"), } ) diff --git a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 index 96ad40c7..0efbf746 100644 --- a/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 +++ b/.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2 @@ -45,6 +45,47 @@ jobs: - name: Configure git to support long filenames run: git config --system core.longpaths true {%- endif %} + {%- if job_data.needs_weaver %} + + - name: Read pinned weaver version + id: versions + {%- if job_data.os == "windows-latest" %} + shell: pwsh + run: | + Select-String -Path versions.env -Pattern '^WEAVER_VERSION=' | ForEach-Object { $_.Line } | Add-Content $env:GITHUB_OUTPUT + {%- else %} + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + {%- endif %} + + - name: Install weaver ${% raw %}{{ steps.versions.outputs.WEAVER_VERSION }}{% endraw %} + env: + WEAVER_VERSION: ${% raw %}{{ steps.versions.outputs.WEAVER_VERSION }}{% endraw %} + {%- if job_data.os == "windows-latest" %} + shell: pwsh + run: | + $url = "https://github.com/open-telemetry/weaver/releases/download/$env:WEAVER_VERSION/weaver-x86_64-pc-windows-msvc.zip" + Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\weaver.zip" + Expand-Archive -Path "$env:RUNNER_TEMP\weaver.zip" -DestinationPath "$env:RUNNER_TEMP\weaver" + "$env:RUNNER_TEMP\weaver" | Add-Content $env:GITHUB_PATH + & "$env:RUNNER_TEMP\weaver\weaver.exe" --version + {%- elif job_data.os == "macos-latest" %} + run: | + WEAVER_ASSET="weaver-aarch64-apple-darwin" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + {%- else %} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + {%- endif %} + {%- endif %} - name: Run tests run: tox -e {{ job_data.tox_env }} -- -ra diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d273940..d3f5bc7c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -252,6 +252,76 @@ jobs: - name: Run tests run: tox -e pypy3-test-instrumentation-openai-v2-latest -- -ra + py312-test-instrumentation-openai-v2-conformance_ubuntu-latest: + name: instrumentation-openai-v2-conformance 3.12 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.12 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox-uv + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Run tests + run: tox -e py312-test-instrumentation-openai-v2-conformance -- -ra + + py313-test-instrumentation-openai-v2-conformance_ubuntu-latest: + name: instrumentation-openai-v2-conformance 3.13 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.13 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Run tests + run: tox -e py313-test-instrumentation-openai-v2-conformance -- -ra + py310-test-instrumentation-openai_agents-v2-oldest_ubuntu-latest: name: instrumentation-openai_agents-v2-oldest 3.10 Ubuntu runs-on: ubuntu-latest @@ -822,6 +892,76 @@ jobs: - name: Run tests run: tox -e py314-test-instrumentation-anthropic-latest -- -ra + py312-test-instrumentation-anthropic-conformance_ubuntu-latest: + name: instrumentation-anthropic-conformance 3.12 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.12 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox-uv + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Run tests + run: tox -e py312-test-instrumentation-anthropic-conformance -- -ra + + py313-test-instrumentation-anthropic-conformance_ubuntu-latest: + name: instrumentation-anthropic-conformance 3.13 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.13 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Run tests + run: tox -e py313-test-instrumentation-anthropic-conformance -- -ra + py310-test-instrumentation-claude-agent-sdk-oldest_ubuntu-latest: name: instrumentation-claude-agent-sdk-oldest 3.10 Ubuntu runs-on: ubuntu-latest @@ -974,6 +1114,76 @@ jobs: - name: Run tests run: tox -e py313-test-instrumentation-claude-agent-sdk-latest -- -ra + py312-test-instrumentation-langchain-conformance_ubuntu-latest: + name: instrumentation-langchain-conformance 3.12 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.12 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox-uv + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Run tests + run: tox -e py312-test-instrumentation-langchain-conformance -- -ra + + py313-test-instrumentation-langchain-conformance_ubuntu-latest: + name: instrumentation-langchain-conformance 3.13 Ubuntu + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python 3.13 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Install tox + run: pip install tox-uv + + - name: Read pinned weaver version + id: versions + run: | + # shellcheck disable=SC2002 + cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT" + + - name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }} + env: + WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }} + run: | + WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu" + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver" + sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver + weaver --version + + - name: Run tests + run: tox -e py313-test-instrumentation-langchain-conformance -- -ra + py310-test-util-genai_ubuntu-latest: name: util-genai 3.10 Ubuntu runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 6c3e08c1..fa7bfddc 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,10 @@ target opentelemetry-admin-jobs.txt .claude/settings.local.json + +# Generated by test_util_genai._setup_weaver +policies/_schemas.rego +.build/ + +# Conformance test weaver live-check reports +weaver_reports/ diff --git a/AGENTS.md b/AGENTS.md index 4530f07f..fcc477ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,6 +109,21 @@ Apply to packages under `instrumentation/`. `from opentelemetry.test_util_genai.vcr import fixture_vcr, scrub_response_headers`) rather than re-implementing provider/exporter/VCR plumbing. +### Conformance tests + +Packages with substantive instrumentation ship `tests/conformance/.py` +scenarios and a `tests/test_conformance.py` that validates emitted telemetry +against the [GenAI semantic conventions](https://github.com/open-telemetry/semantic-conventions-genai) +via Weaver live-check. Each scenario module defines a subclass of +`opentelemetry.test_util_genai.conformance.Scenario` that sets +`expected_spans`, `expected_metrics`, and implements +`run(*, tracer_provider, meter_provider, logger_provider, vcr)`. + +`tests/test_conformance.py` must set `pytestmark = pytest.mark.conformance` at +module level. + +Run via `tox -e py312-test-instrumentation--conformance`. + The parallel PR-review rules live in [`.github/instructions/instrumentation.instructions.md`](.github/instructions/instrumentation.instructions.md) and should be kept in sync with this section. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90576015..c6b54437 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,6 +66,27 @@ Run type checking across the workspace: uv run tox -e typecheck ``` +#### Managing cassettes (test recordings) + +GenAI tests replay recorded HTTP interactions (cassettes) stored under each +package's `tests/cassettes/`. + +- **Run**: nothing extra — cassettes replay automatically when present. Tests + that need a cassette skip if it is missing and no real API key is set. +- **Record**: delete the target `tests/cassettes/.yaml`, export a + real provider API key (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`), and + rerun the test. `pytest-vcr` writes the cassette on the live call. +- **Sanitize**: every package's `vcr_config()` in `tests/conftest.py` must + scrub auth via `filter_headers` and strip identifying response headers via + `scrub_response_headers(...)` from `opentelemetry.test_util_genai.vcr`. + Diff each new cassette before committing — leaked API keys, org ids, or + `Set-Cookie` values block the PR. +- **AI-generated cassettes**: if you lack provider access, you may + synthesize a cassette from the provider's API reference via AI. Make sure + to mention it in the PR and open a follow-up issue to re-record it in CI + against the real provider. +- **CI**: replay-only; recording in CI is a future improvement. + ### 4. Update the changelog This repo uses [towncrier](https://towncrier.readthedocs.io/) to manage @@ -136,3 +157,4 @@ For more information about the maintainer role, see the [community repository](h - [Leighton Chen](https://github.com/lzchen), Microsoft For more information about the approver role, see the [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#approver). + diff --git a/dev-requirements-conformance.txt b/dev-requirements-conformance.txt new file mode 100644 index 00000000..3045a308 --- /dev/null +++ b/dev-requirements-conformance.txt @@ -0,0 +1,14 @@ +# Conformance tests use `opentelemetry.test.weaver_live_check`, not yet on +# PyPI. Pin the whole stack from git so versions stay coherent. Mirrors +# pyproject.toml's [tool.uv.sources]. Drop once the file ships to PyPI. +# +# TODO: switch to PyPI versions once the test utils are released + +opentelemetry-api @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-api +opentelemetry-sdk @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-sdk +opentelemetry-semantic-conventions @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-semantic-conventions +opentelemetry-test-utils @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=tests/opentelemetry-test-utils +opentelemetry-instrumentation @ git+https://github.com/open-telemetry/opentelemetry-python-contrib@d2f396de68a969dfb74b8afc247e1d0dc6739b67#subdirectory=opentelemetry-instrumentation +opentelemetry-exporter-otlp-proto-grpc @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=exporter/opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-common @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=exporter/opentelemetry-exporter-otlp-proto-common +opentelemetry-proto @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-proto diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/pyproject.toml b/instrumentation/opentelemetry-instrumentation-anthropic/pyproject.toml index 9fe8e25c..c0f08cc0 100644 --- a/instrumentation/opentelemetry-instrumentation-anthropic/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-anthropic/pyproject.toml @@ -52,6 +52,9 @@ packages = ["src/opentelemetry"] [tool.pytest.ini_options] testpaths = ["tests"] +markers = [ + "conformance: GenAI semconv conformance scenario (run via the *-conformance tox envs)", +] [tool.towncrier] directory = ".changelog" diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/cassettes/inference_conformance.yaml b/instrumentation/opentelemetry-instrumentation-anthropic/tests/cassettes/inference_conformance.yaml new file mode 100644 index 00000000..f7c1cab0 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/cassettes/inference_conformance.yaml @@ -0,0 +1,140 @@ +interactions: +- request: + body: |- + { + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Say hello in one word." + } + ], + "model": "claude-sonnet-4-20250514" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '117' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_0176GK1qFwwpVM59jDYiKPjN", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "Hello!" + } + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 13, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 5, + "service_tier": "standard", + "inference_geo": "not_available" + } + } + headers: + CF-RAY: + - 9d6567b84d8509ae-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 03 Mar 2026 03:02:57 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-ratelimit-input-tokens-limit: + - '450000' + anthropic-ratelimit-input-tokens-remaining: + - '450000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-03T03:02:57Z' + anthropic-ratelimit-output-tokens-limit: + - '90000' + anthropic-ratelimit-output-tokens-remaining: + - '90000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-03T03:02:57Z' + anthropic-ratelimit-requests-limit: + - '1000' + anthropic-ratelimit-requests-remaining: + - '999' + anthropic-ratelimit-requests-reset: + - '2026-03-03T03:02:56Z' + anthropic-ratelimit-tokens-limit: + - '540000' + anthropic-ratelimit-tokens-remaining: + - '540000' + anthropic-ratelimit-tokens-reset: + - '2026-03-03T03:02:57Z' + cf-cache-status: + - DYNAMIC + content-length: + - '441' + request-id: + - req_011CYfQM6TP1iFTVWNrJQJcn + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '1056' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/cassettes/tool_calling_conformance.yaml b/instrumentation/opentelemetry-instrumentation-anthropic/tests/cassettes/tool_calling_conformance.yaml new file mode 100644 index 00000000..85d6bc4d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/cassettes/tool_calling_conformance.yaml @@ -0,0 +1,168 @@ +interactions: +- request: + body: |- + { + "max_tokens": 256, + "messages": [ + { + "role": "user", + "content": "What is the weather in SF?" + } + ], + "model": "claude-sonnet-4-20250514", + "tool_choice": { + "type": "tool", + "name": "get_weather" + }, + "tools": [ + { + "name": "get_weather", + "description": "Get weather by city", + "input_schema": { + "type": "object", + "properties": { + "city": { + "type": "string" + } + }, + "required": [ + "city" + ] + } + } + ] + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '334' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.75.0 + x-api-key: + - test_anthropic_api_key + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 0.75.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: |- + { + "model": "claude-sonnet-4-20250514", + "id": "msg_01VkeD2PaERGgjekk84zPSwr", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01UAcA2APHENDob6qJ8kmoaC", + "name": "get_weather", + "input": { + "city": "San Francisco" + }, + "caller": { + "type": "direct" + } + } + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": { + "input_tokens": 386, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "cache_creation": { + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0 + }, + "output_tokens": 34, + "service_tier": "standard", + "inference_geo": "not_available" + } + } + headers: + CF-RAY: + - 9d656813eddfae20-EWR + Connection: + - keep-alive + Content-Security-Policy: + - default-src 'none'; frame-ancestors 'none' + Content-Type: + - application/json + Date: + - Tue, 03 Mar 2026 03:03:11 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + anthropic-ratelimit-input-tokens-limit: + - '450000' + anthropic-ratelimit-input-tokens-remaining: + - '450000' + anthropic-ratelimit-input-tokens-reset: + - '2026-03-03T03:03:11Z' + anthropic-ratelimit-output-tokens-limit: + - '90000' + anthropic-ratelimit-output-tokens-remaining: + - '90000' + anthropic-ratelimit-output-tokens-reset: + - '2026-03-03T03:03:11Z' + anthropic-ratelimit-requests-limit: + - '1000' + anthropic-ratelimit-requests-remaining: + - '999' + anthropic-ratelimit-requests-reset: + - '2026-03-03T03:03:10Z' + anthropic-ratelimit-tokens-limit: + - '540000' + anthropic-ratelimit-tokens-remaining: + - '540000' + anthropic-ratelimit-tokens-reset: + - '2026-03-03T03:03:11Z' + cf-cache-status: + - DYNAMIC + content-length: + - '550' + request-id: + - req_011CYfQNBARQegAkczG6kweQ + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + vary: + - Accept-Encoding + x-envoy-upstream-service-time: + - '1256' + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/__init__.py new file mode 100644 index 00000000..e57cf4ab --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/__init__.py @@ -0,0 +1,2 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/inference.py new file mode 100644 index 00000000..1c5e59ad --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/inference.py @@ -0,0 +1,61 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: anthropic chat (inference).""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from anthropic import Anthropic + +from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class InferenceScenario(Scenario): + expected_spans = ("chat",) + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("ANTHROPIC_API_KEY") + else {"ANTHROPIC_API_KEY": "test_anthropic_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + AnthropicInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("inference_conformance.yaml"): + Anthropic().messages.create( + model="claude-sonnet-4-20250514", + max_tokens=100, + messages=[ + { + "role": "user", + "content": "Say hello in one word.", + } + ], + ) diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/tool_calling.py b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/tool_calling.py new file mode 100644 index 00000000..75f42098 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conformance/tool_calling.py @@ -0,0 +1,73 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: anthropic chat with tool calls.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from anthropic import Anthropic + +from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class ToolCallingScenario(Scenario): + expected_spans = ("chat",) + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("ANTHROPIC_API_KEY") + else {"ANTHROPIC_API_KEY": "test_anthropic_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + AnthropicInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("tool_calling_conformance.yaml"): + Anthropic().messages.create( + model="claude-sonnet-4-20250514", + max_tokens=256, + messages=[ + { + "role": "user", + "content": "What is the weather in SF?", + } + ], + tools=[ + { + "name": "get_weather", + "description": "Get weather by city", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + tool_choice={"type": "tool", "name": "get_weather"}, + ) diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conftest.py index 6e074fdc..f7d2d342 100644 --- a/instrumentation/opentelemetry-instrumentation-anthropic/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/conftest.py @@ -13,7 +13,10 @@ from opentelemetry.test_util_genai.instrumentor import instrument from opentelemetry.test_util_genai.vcr import scrub_response_headers -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] @pytest.fixture(autouse=True) diff --git a/instrumentation/opentelemetry-instrumentation-anthropic/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-anthropic/tests/test_conformance.py new file mode 100644 index 00000000..5e6535e9 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-anthropic/tests/test_conformance.py @@ -0,0 +1,42 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for anthropic.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check isn't installed (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.inference import InferenceScenario +from .conformance.tool_calling import ToolCallingScenario + +pytestmark = pytest.mark.conformance + +_LEGACY_SYSTEM_SKIP = pytest.mark.skip( + reason="anthropic emits legacy gen_ai.system in experimental mode" +) + + +@pytest.mark.parametrize( + "scenario", + [ + pytest.param(InferenceScenario(), marks=_LEGACY_SYSTEM_SKIP), + pytest.param(ToolCallingScenario(), marks=_LEGACY_SYSTEM_SKIP), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-langchain/tests/cassettes/inference_conformance.yaml b/instrumentation/opentelemetry-instrumentation-langchain/tests/cassettes/inference_conformance.yaml new file mode 100644 index 00000000..5bd76600 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-langchain/tests/cassettes/inference_conformance.yaml @@ -0,0 +1,161 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "content": "You are a helpful assistant!", + "role": "system" + }, + { + "content": "What is the capital of France?", + "role": "user" + } + ], + "model": "gpt-3.5-turbo", + "frequency_penalty": 0.5, + "max_completion_tokens": 100, + "presence_penalty": 0.5, + "seed": 100, + "stop": [ + "\n", + "Human:", + "AI:" + ], + "stream": false, + "temperature": 0.1, + "top_p": 0.9 + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '316' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.21.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.21.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.5 + authorization: + - Bearer test_openai_api_key + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-DB5VKnX0DBvthDkYqZDgfZVAqhZUk", + "object": "chat.completion", + "created": 1771535138, + "model": "gpt-3.5-turbo-0125", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 24, + "completion_tokens": 7, + "total_tokens": 31, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null + } + headers: + CF-RAY: + - 9d08b930cee533ed-SJC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Thu, 19 Feb 2026 21:05:38 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '822' + openai-organization: test_openai_org_id + openai-processing-ms: + - '170' + openai-project: + - proj_GLiYlAc06hF0Fm06IMReZLy4 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=tYjXaOj8KokJGc56XWDShqpm2zPWeq3Q19yhnJcqKqU-1771535137.4012983-1.0.1.1-k50ojuWx4sSPmgdu8QLKD3OmGYDMjKIDSqQcANN1wo1SBWNjIE.yAsOYOgaalvDiPrXObJN80xhnlcC9mXOXnOfGb9HRyT5un2ASAG29wbujzo_b9a3zQg9hHU024Nkz; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 19 Feb 2026 + 21:35:38 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199982' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 5ms + x-request-id: + - req_5f037f577c544db292661f8bbf12d569 + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-langchain/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-langchain/tests/conformance/__init__.py new file mode 100644 index 00000000..e57cf4ab --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-langchain/tests/conformance/__init__.py @@ -0,0 +1,2 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 diff --git a/instrumentation/opentelemetry-instrumentation-langchain/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-langchain/tests/conformance/inference.py new file mode 100644 index 00000000..b42b324e --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-langchain/tests/conformance/inference.py @@ -0,0 +1,72 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: langchain chat (inference) via ChatOpenAI.""" + +from __future__ import annotations + +import os +from typing import Any +from unittest import mock + +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_openai import ChatOpenAI + +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class InferenceScenario(Scenario): + expected_spans = ("chat",) + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + key_override = ( + {} + if os.getenv("OPENAI_API_KEY") + else {"OPENAI_API_KEY": "test_openai_api_key"} + ) + with mock.patch.dict(os.environ, key_override): + with instrument( + LangChainInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + llm = ChatOpenAI( + model="gpt-3.5-turbo", + temperature=0.1, + max_tokens=100, + top_p=0.9, + frequency_penalty=0.5, + presence_penalty=0.5, + stop_sequences=["\n", "Human:", "AI:"], + seed=100, + ) + with vcr.use_cassette("inference_conformance.yaml"): + llm.invoke( + [ + SystemMessage( + content="You are a helpful assistant!" + ), + HumanMessage( + content="What is the capital of France?" + ), + ] + ) diff --git a/instrumentation/opentelemetry-instrumentation-langchain/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-langchain/tests/test_conformance.py new file mode 100644 index 00000000..77128a53 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-langchain/tests/test_conformance.py @@ -0,0 +1,36 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for langchain.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check isn't installed (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.inference import InferenceScenario + +pytestmark = pytest.mark.conformance + + +@pytest.mark.parametrize( + "scenario", + [ + InferenceScenario(), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/embedding_conformance.yaml b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/embedding_conformance.yaml new file mode 100644 index 00000000..d7177abf --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/embedding_conformance.yaml @@ -0,0 +1,1652 @@ +interactions: +- request: + body: |- + { + "input": "This is a test for async embeddings", + "model": "text-embedding-3-small" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '83' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.26.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 1.26.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.9.2 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: |- + { + "object": "list", + "data": [ + { + "object": "embedding", + "index": 0, + "embedding": [ + -0.0012588563, + -0.029314144, + -0.004568109, + -0.021721158, + -0.025343766, + -0.0004159207, + 0.04500728, + 0.03663181, + 0.009042029, + -0.0049231243, + 0.026416058, + -0.030574812, + 7.522572e-05, + -0.031241372, + -0.03683468, + 0.0045789764, + -0.017837722, + -0.0065569207, + 0.017272595, + -0.0049231243, + -0.008542109, + -0.047383714, + 0.015185973, + 0.042804737, + 0.0033654028, + -0.019634536, + 0.012171963, + 0.041471615, + 0.033994555, + -0.034690093, + 0.006172924, + -0.038370665, + 0.009599911, + -0.028618604, + -0.017069729, + 0.06578656, + -0.010759146, + 0.036747735, + -0.039819706, + -0.0077886074, + 0.0027314464, + -0.03750124, + -0.04993403, + -0.0045898445, + 0.013425386, + 0.05135409, + -0.03257449, + -0.01773629, + 0.018866543, + 0.034516208, + -0.048398044, + 0.012041549, + -0.0050209346, + 0.047876388, + -0.032081816, + -0.007237971, + -0.029792327, + 0.030429907, + -0.0102447355, + 0.0064736004, + 0.061787203, + -0.031212391, + -0.029256182, + -0.018881032, + 0.006412016, + -0.00872324, + -0.022416698, + 0.025285805, + -0.011730005, + -0.0064736004, + 0.05193371, + 0.023720838, + -0.009969417, + 0.0034740812, + -0.0029632933, + 0.021025617, + 0.012063284, + 0.016519092, + -0.016345207, + -0.034516208, + -0.030111117, + 0.027531821, + -0.029256182, + -0.019040428, + -0.01775078, + -0.013179048, + -0.07807445, + -0.009136218, + -0.0018765109, + 0.0011746306, + -0.020721318, + -0.028256342, + -0.07679929, + -0.029328635, + 0.029343124, + 0.011845929, + -0.001069575, + -0.04593467, + -0.03428436, + 0.021721158, + 0.01576559, + 0.02543071, + -0.017576894, + 0.008230565, + 0.0015187784, + -0.016475622, + 0.0012162906, + -0.00537595, + -0.04758658, + -0.003651589, + -0.05671555, + -0.014193378, + 0.0036751358, + 0.003187895, + 0.057150263, + -0.022576094, + -0.00452826, + -0.013323952, + -0.008563845, + -0.021156032, + -0.018576734, + -0.007031482, + 0.03628404, + -0.007491553, + -0.024590263, + -0.046166517, + -0.03553054, + -0.047064923, + -0.061787203, + 0.0005922964, + 0.02082275, + -0.028270833, + 0.0557302, + 0.039153147, + -0.06288847, + -0.011164878, + -0.077205025, + 0.027010165, + -0.0033291767, + 0.022981824, + -0.017910173, + -0.04660123, + 0.03654487, + -0.022720998, + -0.045673843, + -0.010708429, + -0.021996476, + 0.047905367, + 0.005970058, + -0.008643542, + -0.011338763, + -0.02548867, + -0.04494932, + 0.042456966, + -0.018953485, + -0.031734046, + -0.038370665, + 3.5575144e-05, + 0.010679448, + -0.0052346685, + -0.036515888, + 0.019938834, + 0.034139458, + -0.0284737, + 0.0017578705, + 0.03083564, + 0.01005636, + -0.005129613, + -0.022735488, + -0.062830515, + 0.004568109, + -0.03750124, + -0.054889757, + 0.000167206, + -0.012012568, + -0.014867184, + 0.028096948, + -0.06943815, + 0.012164718, + -0.011186614, + -0.014722279, + 0.024039626, + 0.022416698, + 0.00039848688, + -0.06074389, + 0.06584452, + 0.02246017, + 0.06404771, + 0.0010713863, + 0.0025285804, + -0.007006124, + 0.058599308, + 0.056918416, + -0.008607317, + 0.024213511, + -0.031212391, + 0.042891677, + 0.003068349, + 0.031618122, + -0.01223717, + 0.0075712507, + 0.03990665, + -0.0050173122, + 0.0057671918, + 0.05199167, + -0.0251409, + -0.023633895, + -0.042312063, + -0.009969417, + 0.0045572408, + 0.042456966, + -0.019388199, + 0.0028763507, + 0.010643222, + -0.015953965, + 0.007810343, + -0.034371305, + 0.013505083, + 0.037066527, + -0.018055078, + -0.030980544, + 0.043326393, + 0.054426063, + 0.0402834, + -0.026256664, + -0.0015142502, + -0.016026419, + -0.0013657232, + 0.017388519, + 0.00015962117, + -0.004473921, + -0.023648385, + -0.0016011928, + -0.01677992, + 0.03190793, + -0.015968457, + -0.001709871, + -0.0135558, + -0.02482211, + 0.0091144815, + -0.015838042, + -0.034690093, + 0.01391806, + -0.021953005, + -0.006654731, + 0.023083258, + -0.001220819, + -0.028401246, + 0.038167797, + -0.019069409, + -0.019344727, + 0.02218485, + -0.014570129, + 0.0020576413, + -0.04219614, + -0.023271633, + -0.030024175, + 0.03929805, + 0.0073176683, + 0.005618665, + -0.012910975, + 0.042341042, + -0.02146033, + 0.007448082, + 0.013968777, + -0.05158594, + 0.033009205, + -0.008317508, + -0.006031642, + -0.047180846, + -0.00988972, + -0.0060026613, + 0.057440072, + -0.036747735, + -0.057816826, + 0.006129453, + -0.036370985, + -0.007364762, + -0.007926267, + 0.0067597865, + 0.004408714, + -0.046977982, + 0.010983747, + 0.0342264, + -0.011309782, + 0.007940757, + 0.0013113841, + -0.019518612, + -0.0016138719, + 0.011940116, + -0.0052093104, + -0.05868625, + 0.051209185, + 0.029009845, + 0.0051513487, + 0.004386978, + 0.033443917, + -0.010563525, + 0.016533583, + -0.010208509, + 0.05222352, + -0.04350028, + 0.0322557, + 0.02822736, + -0.018634696, + 0.033617802, + -0.04865887, + 0.040718116, + 0.040399324, + 0.001111235, + -0.028430227, + 0.01275158, + -0.035646465, + 0.023039786, + -0.046717152, + 0.008795693, + 0.02447434, + -0.015635176, + 0.034371305, + -0.047702502, + -0.008998558, + -0.036399964, + 0.011621326, + -0.045499958, + -0.0063830353, + -0.03118341, + -0.00072678574, + -0.035095826, + 0.015301896, + 0.029140258, + -0.0068902005, + 0.028444719, + 0.03663181, + -0.002318469, + 0.023691857, + -0.026633414, + -0.00048769361, + -0.023648385, + 0.0496732, + -0.0018620205, + -0.04121079, + -0.0017297954, + 0.04100792, + -0.025532141, + 0.005415799, + 0.052803133, + 0.027039146, + 0.030951563, + 0.031734046, + -0.017388519, + 0.013447121, + -0.0066112597, + -0.0068865777, + 0.003044802, + 0.057845805, + -0.05767192, + -0.020243134, + -0.0077596265, + -0.010222999, + -0.019634536, + 0.025749497, + -0.0053723278, + -0.009962172, + 0.066655986, + 0.013505083, + -0.031994876, + -0.007803098, + 0.021851571, + -0.002628202, + -0.015562724, + -0.016982786, + 0.049876068, + 0.004289168, + 0.009353574, + -0.02586542, + 0.02182259, + -0.026705867, + 0.03990665, + 0.04590569, + 0.054976698, + 0.03153118, + -0.032284684, + -0.039182127, + -0.07488655, + -0.02143135, + 0.005129613, + -0.014657072, + -0.01912737, + -0.026213191, + -0.011476423, + 0.0125849405, + -0.031994876, + -0.025169881, + -0.00569474, + 0.024300454, + 0.027604273, + 0.058077652, + -0.025720516, + 0.009382554, + -0.0035429106, + -0.0051404806, + -0.045876708, + -0.010273716, + 0.0016645883, + 0.0022170362, + 0.023097748, + 0.02819838, + -0.0016274566, + 0.025271313, + -0.0093898, + -0.005988171, + -0.0011338763, + 0.044021934, + -0.05630982, + -0.0009980286, + 0.019011447, + 0.012758825, + -0.043703143, + 0.010976503, + 0.06729357, + -0.036747735, + -0.0584544, + -0.024445359, + -0.013512328, + 0.024039626, + -0.010447602, + 0.075929865, + 0.0060859816, + 0.018315906, + -0.06132351, + -0.043674164, + -0.018996956, + 0.043413334, + 0.038428627, + -0.021054598, + -0.022938354, + 0.036689773, + 0.019272275, + -0.029212711, + -0.018489791, + -0.0052165557, + -0.007237971, + -0.0584544, + -0.010527299, + -0.009462252, + -0.011222839, + -0.026024817, + 0.0037566444, + -0.006346809, + -0.01291822, + -0.008143622, + -0.067931145, + 0.009599911, + -0.029256182, + -0.009027539, + -0.027415898, + 0.010143302, + 0.0031208768, + 0.06735153, + 0.007861059, + 0.019388199, + -0.050484665, + -0.06010631, + 0.035878308, + 0.06004835, + 0.017374028, + -0.0028093325, + 0.010947522, + 0.045528937, + 0.0018131153, + -0.008237811, + 0.04636938, + -0.0013901758, + 0.027357936, + 0.037530217, + -0.035443597, + 0.024010645, + -0.021532781, + -0.00030226135, + -0.022039948, + -0.012106756, + -0.021040108, + -0.039124165, + -0.036313023, + -0.00093463284, + 0.14397693, + -0.043065563, + -0.019359218, + 0.008773956, + -0.012505243, + -0.026503, + 0.017794251, + -0.012222679, + -0.018533263, + 0.034371305, + -0.047238808, + -0.01676543, + -0.05094836, + 0.03950092, + -0.027560802, + -0.012824032, + 0.0063721677, + -0.020170681, + 0.021547273, + 0.01776527, + -0.0010795372, + -0.041761424, + 0.009650628, + -0.014033983, + 0.0171132, + -0.014041228, + 0.01191838, + -0.020243134, + -0.010831598, + -0.012592185, + -0.004285545, + -0.036168117, + 0.054918736, + 0.005593307, + -0.04184837, + -0.009230405, + -0.01914186, + -0.0036443437, + -0.041529577, + 0.012512488, + 0.013193538, + -0.01375142, + -0.057208225, + -0.034139458, + -0.0052708946, + -0.018084059, + -0.015910495, + 0.028430227, + -0.026430547, + -0.027937554, + -0.00047275034, + -0.020735808, + 0.017243614, + 0.018402848, + -0.014772995, + 0.03883436, + 0.0044884114, + 0.034719076, + -0.02112705, + -0.022836922, + 0.043413334, + 0.023488991, + 0.015026578, + -0.020243134, + 0.0152439345, + 0.022720998, + -0.018996956, + 0.028430227, + -0.017388519, + 0.01225166, + 0.013722439, + 0.04500728, + -0.016895844, + -0.018156512, + -0.022228323, + 0.021445839, + 0.03686366, + 0.06706172, + 0.035008885, + -0.033067167, + 0.03689264, + 0.017808741, + -0.0019073031, + 0.044050913, + -0.0050245575, + -0.05801969, + 0.010317188, + 0.021590743, + 0.02143135, + 0.014533903, + 0.01982291, + -0.023517972, + 0.018852051, + -0.034342322, + -0.037385315, + -0.017069729, + -0.010817108, + 0.02279345, + 0.0118241925, + -0.00887539, + 0.054049313, + 0.004546373, + -0.019170841, + -0.010679448, + 0.032516528, + 0.02882147, + 0.0019471518, + 0.019764949, + -0.038109835, + 0.025995836, + -0.010715675, + 0.048484985, + 0.0021337161, + -0.03263245, + 0.010578016, + 0.011266311, + 0.0018746996, + -0.008607317, + -0.005437535, + 0.02987927, + 0.006506204, + -0.035066847, + 0.016519092, + 0.04019646, + -0.014041228, + -0.009143462, + -0.030806659, + 0.017939154, + -0.0146136, + -0.0031353673, + -0.007006124, + 0.008795693, + 0.0080277, + 0.028096948, + -0.020909693, + 0.014780241, + -0.03184997, + -0.00056014577, + 0.019735968, + -0.024068607, + 0.0015857967, + 0.0045318827, + 0.00562591, + 0.0029614822, + 0.025720516, + -0.015229444, + 0.0052346685, + 0.014910654, + -0.037153468, + -0.037877988, + -0.0160554, + -0.02754631, + 0.039442956, + 0.011382234, + -0.024126569, + -0.0106939385, + 0.0001393572, + 0.02315571, + 0.0080277, + -0.010998238, + -0.010933031, + 0.014316547, + 0.0033961951, + -0.004571731, + -0.013425386, + 0.0059773033, + -0.032313664, + 0.003635287, + -0.010462092, + 0.02586542, + -0.07088719, + -0.019837402, + -0.011650307, + -0.013381914, + -0.016649507, + 0.0073973658, + 0.00247243, + -0.013476102, + 0.0129544465, + 0.008216075, + 0.043094546, + -0.043703143, + 0.04636938, + -0.0017823231, + -0.008636298, + 0.010027379, + 0.012983427, + 0.04384805, + -0.02047498, + 0.023329595, + -0.039182127, + -0.029560482, + -0.016635016, + 0.01743199, + -0.017475462, + -0.0052093104, + 0.020706827, + -0.008360979, + -0.026314626, + 0.036805697, + -0.0044232043, + 0.03889232, + 0.0016365132, + 0.0021463952, + -0.01844632, + -0.046804097, + -0.028125929, + 0.052542306, + -0.0029940854, + -0.0065170717, + 0.035762385, + 0.010911295, + 0.02987927, + -0.018866543, + 0.036428947, + 0.007650948, + 0.017837722, + 0.008201584, + -0.025923382, + -0.004933992, + -0.02987927, + 0.015881514, + 0.029096788, + 0.030024175, + -0.005339724, + -0.016707469, + -0.035095826, + 0.011701024, + 0.026387077, + -0.007455327, + -0.01409919, + 0.0060461327, + -0.012548714, + -0.029966213, + -0.038022894, + -0.036660794, + 0.0026734846, + -0.0030647265, + -0.0086145615, + -0.0077234004, + -0.042312063, + -0.010063605, + 0.0073901205, + -0.013628251, + 0.003169782, + -0.010483827, + 0.00019460198, + -0.016649507, + -0.011259066, + -0.018736128, + 0.00089478417, + 0.001361195, + -0.010672203, + -0.040428307, + -0.011925626, + 0.01712769, + 0.009643382, + 0.011483667, + -0.0012733467, + -0.03019806, + 0.048484985, + 0.02751733, + 0.0006565977, + 0.0149541255, + -0.011498158, + -0.022923864, + 0.012700864, + -0.012027059, + 0.022880392, + -0.041819386, + -0.036370985, + 0.024054118, + -0.022445679, + -0.0124545265, + 0.013635497, + 0.02147482, + -0.007089444, + -0.012440036, + 0.017953645, + 0.015316387, + -0.0112445755, + -0.018562244, + -0.0052817627, + 0.045470975, + 0.024995996, + 0.0036207966, + -0.023720838, + 0.016519092, + 0.002394544, + 0.051701862, + -0.006810503, + -0.02214138, + 0.017200142, + -0.034081496, + 0.018576734, + -0.0155047625, + 0.021909533, + -0.00032105364, + -0.026126249, + -0.010606997, + -0.015490272, + -0.01744648, + 0.01673645, + -0.0028147665, + -0.043529257, + -0.057874784, + -0.0042638094, + 0.037820026, + -0.028575132, + 0.00652794, + -0.0021645082, + -0.044833396, + -0.0630044, + -0.018721638, + 0.029140258, + -0.023648385, + -0.0044232043, + 0.018518772, + -0.0029687274, + 0.01543231, + 0.00313899, + 0.020214153, + 0.038428627, + -0.041326713, + -0.035878308, + -0.029705387, + 0.03683468, + -0.009882474, + 0.024039626, + -0.01843183, + 0.0031317447, + -0.0113749895, + -0.02714058, + 0.008998558, + 0.03156016, + 0.012374829, + -0.036950603, + -0.0088246735, + 0.0077234004, + 0.008759466, + -0.010143302, + -0.007948002, + -0.018808581, + -0.0038218515, + 0.012679128, + 0.01257045, + -0.0077523813, + 0.0023818647, + 0.002454317, + -0.00063622056, + 0.025981344, + -0.017939154, + -0.003948643, + -0.028415738, + -0.010672203, + -0.024923543, + 0.054628927, + 0.010447602, + 0.017156672, + 0.013693458, + -0.003807361, + -0.030893601, + 0.012613921, + 0.0013177237, + 0.0074082334, + -0.023228163, + -0.00011943286, + 0.026358096, + 0.005850512, + 0.027024657, + 0.0057092304, + 0.023691857, + -0.026894242, + 0.014512167, + 0.008527619, + -0.00301401, + 0.0292272, + 0.005165839, + -0.0052056876, + 0.021025617, + -5.7905127e-05, + -0.02009823, + 0.025937874, + -0.008578336, + 0.061613318, + 0.017895684, + -0.017968135, + 0.007549515, + 0.013816627, + 0.035675444, + 0.011295292, + 0.019156352, + 0.050745495, + 0.015707629, + 0.021300934, + 0.017649347, + -0.023387557, + -0.010049115, + -0.03127035, + -0.052252498, + -0.031357296, + 0.027749177, + 0.043703143, + -0.03663181, + -0.0052455366, + 0.014809222, + 0.004223961, + -0.00049176905, + 0.014983106, + -0.03083564, + 0.045036264, + -0.013765911, + 0.02822736, + -0.007585741, + -0.02583644, + -0.011215595, + -0.030140098, + -0.014577375, + 0.008962332, + 0.036805697, + 0.0023365822, + -0.015359858, + 0.00755676, + 0.024923543, + -0.020953165, + -0.037414297, + -0.011505403, + 0.011845929, + 0.017692817, + 0.037964933, + 0.022213832, + -0.0044050915, + -0.0018837561, + 0.04836906, + -0.015577215, + -0.00789004, + -0.04663021, + 0.0025901648, + 0.0047999555, + -0.019562082, + -0.0153743485, + 0.011309782, + -0.016504603, + 0.03118341, + 0.0110562, + -0.0428627, + 0.008071171, + 6.022586e-05, + -0.008339244, + 0.0013747797, + -0.035675444, + 0.0030574813, + 0.022373227, + -0.041036904, + -0.014475942, + -0.021866063, + -0.028908413, + 0.02547418, + -0.02282243, + 0.020417018, + -0.014910654, + 0.0027785404, + 0.028082456, + 0.0051151225, + -0.021286445, + 0.022039948, + 0.04428276, + -0.0054592704, + -0.019054918, + -0.017475462, + -0.021909533, + 0.0218081, + 0.004785465, + 0.034979902, + -0.0042203385, + 0.0055462127, + 0.030053155, + 0.0023076013, + -0.040776078, + 0.01912737, + -0.026560962, + -0.023083258, + -0.030748697, + 0.023271633, + 0.008549355, + 0.017316066, + 0.022054438, + 0.016330717, + 0.0055751936, + 0.027604273, + 0.021286445, + 0.008933351, + 0.027705707, + 0.046977982, + -0.015330877, + 0.007984228, + 0.015591705, + 0.012657393, + -0.013200783, + 0.0046985224, + -0.008085661, + 0.008752221, + 0.0062055276, + 0.036428947, + -0.0012008946, + 0.016577054, + -0.015519253, + -0.036168117, + 0.0041189054, + -0.002626391, + -0.020706827, + -0.0050318027, + 0.0502818, + -0.02882147, + 0.010346169, + -0.0019471518, + -0.020257624, + -0.0055860616, + 0.004187735, + 0.0048035784, + -0.022692017, + -0.0020413396, + -0.01612785, + -0.008491393, + 0.029299654, + 0.014287566, + -0.02985029, + 0.008092906, + -0.060512044, + -0.028386757, + -0.02282243, + 0.0048289364, + -0.023054278, + 0.02415555, + -0.020561922, + -0.01205604, + -0.02543071, + 0.026705867, + 0.010947522, + -0.010317188, + -0.017808741, + -0.005165839, + -0.032806337, + 0.01193287, + 0.028676566, + -0.010353413, + 0.014005003, + -0.0075929863, + -0.025952363, + -0.026169721, + 0.045326073, + -0.0023039787, + 0.015881514, + 0.009628892, + -0.030893601, + -0.023851251, + -0.0017651158, + -0.0004333545, + 0.030342964, + 0.04494932, + -0.01777976, + 0.015403329, + 0.004571731, + 0.010085341, + 0.02511192, + -0.018924505, + 0.022010967, + -0.0013892702, + 0.0022061684, + 0.0018122096, + 0.023604915, + 0.0018457188, + 0.010338923, + 0.011664798, + 0.0064265067, + -0.03863149, + -0.00028822376, + 0.010215755, + -0.013063124, + -0.0027332578, + -0.009462252, + 0.010266471, + 0.025416218, + 0.008252301, + 0.029415578, + 0.0032023855, + -0.013294972, + 0.014142661, + -0.0006253527, + 0.027952043, + 0.013968777, + -0.012258906, + -0.025242332, + -0.0147440145, + 0.003050236, + -0.01343263, + 0.006248999, + -0.0076002316, + -0.004633316, + 0.021011127, + 0.007520534, + 0.008049435, + -0.0061040944, + 0.025575612, + -0.0024995995, + -0.017649347, + -0.017808741, + 0.028806979, + 0.0019036805, + -0.016533583, + 0.016794411, + -0.017997116, + -0.0153743485, + 0.018562244, + -0.007216235, + -0.02854615, + 0.01123733, + -0.007962492, + 0.020953165, + 0.0063612997, + 0.025952363, + -0.013947041, + -0.0009142557, + 0.0015830797, + 0.032226723, + -0.012773315, + 0.015707629, + 0.030371945, + -0.009925946, + -0.02586542, + -0.025198862, + 0.01875062, + -0.04500728, + -0.023706347, + -0.037617162, + 0.032081816, + 0.005821531, + 0.025648065, + 0.027386917, + -0.0136644775, + 0.008715995, + 0.028111437, + 0.005158594, + -0.020547433, + 0.013287726, + -0.006136698, + -0.037240412, + 0.0160554, + -0.026343606, + 0.004075434, + -0.04364518, + 0.0040066047, + -0.01473677, + 0.014925145, + -0.011541629, + 0.015953965, + -0.011816948, + 0.0036316644, + 0.0055208546, + 0.032980222, + -0.0036968715, + 0.025633574, + 0.10792474, + -0.0030701603, + 0.002318469, + 0.00044671286, + 0.016692977, + -0.021924024, + -0.027126089, + -0.0036425323, + -0.01914186, + -0.025937874, + -0.0026535604, + 0.032023855, + 0.010259226, + -0.016229283, + 0.0039341524, + 0.008433431, + 0.0010949332, + 0.002121037, + -0.001087688, + -0.00042044895, + 0.010201264, + 0.025387237, + -0.0027332578, + 0.016504603, + -0.053440712, + -0.009788287, + -0.025242332, + 0.010882314, + 0.014830957, + 0.043790087, + -0.013773155, + 0.03663181, + -0.03118341, + 0.0101143215, + 0.00050942926, + 0.023373067, + -0.053759504, + 0.008411696, + 0.023561442, + -0.013215274, + 0.005379573, + -0.009541949, + -0.010128812, + -0.017287085, + -0.009715835, + -0.04257289, + -0.0140847, + -0.020764789, + -0.0054665157, + -0.006690957, + 0.0024271475, + 0.027604273, + -0.028560642, + 0.024532301, + 0.042833716, + -0.030603793, + 0.008752221, + 0.02954599, + 0.05129613, + -0.023271633, + 0.017533423, + 0.025691535, + -0.0402834, + 0.023416538, + 0.005202065, + -0.022634055, + -0.02543071, + -0.0071365377, + -0.00060678687, + 0.03176303, + -0.015953965, + 0.021735648, + -0.014635337, + -0.050687533, + -0.000110319736, + 0.02080826, + 0.01914186, + -0.012302377, + -0.025343766, + 0.027357936, + -0.018518772, + 0.0054013086, + 0.033791687, + 0.033646785, + -0.016287245, + 0.0044340724, + 0.002318469, + -0.001212668, + -0.0008449733, + 0.04523913, + -0.035675444, + -0.021677686, + 0.011838683, + -0.02882147, + 0.002878162, + -0.00016143247, + 0.0021898665, + -0.03376271, + 0.012838523, + 0.010447602, + 0.013447121, + 0.03054583, + -0.007839324, + 0.016577054, + 0.016171321, + 0.015185973, + -0.054078292, + 0.04126875, + 0.014758505, + 0.036457926, + 0.011208349, + -0.028357776, + 0.013606516, + -0.012353093, + -0.016939316, + 0.012077775, + 3.0763866e-05, + 0.001561344, + 0.0073973658, + -0.028270833, + 0.0075277793, + 0.0025847307, + 0.0046079573, + -0.016359698, + 0.0045173923, + -0.043877028, + 0.0052093104, + -0.00045237318, + -0.014193378, + -0.021373387, + 0.037762064, + -0.0061765467, + 0.005745456, + -0.0018837561, + 0.039703783, + 0.008194339, + 0.014367264, + -0.027923062, + -0.01373693, + 0.02314122, + 0.018996956, + 0.024662714, + 0.012222679, + -0.0046152025, + 0.014381754, + -0.016997278, + 0.010396885, + 0.0035229863, + -0.03292226, + 0.008744976, + -0.0056150425, + -0.041181806, + 0.0016229284, + -0.029024335, + 0.026198702, + 0.0039595105, + 0.020286605, + 0.008252301, + 0.004473921, + -0.039442956, + -0.011295292, + -0.002979595, + -0.03990665, + 0.016533583, + 0.024532301, + 0.0045173923, + 0.013577535, + 0.016620526, + 0.014055719, + 0.02277896, + 0.028415738, + -0.025735008, + -0.0045789764, + -0.0008984068, + -0.00048135404, + -0.024083098, + 0.0006316923, + 0.020764789, + -0.05674453, + -0.0098245125, + 0.015403329, + -0.029705387, + -0.017634856, + 0.036023214, + -0.002251451, + -0.00789004, + 0.0067344285, + -0.016533583, + -0.004506524, + -0.00176421, + -0.009578176, + -0.00029682743, + -0.034835, + -0.0073720072, + -0.017707309, + -0.017953645, + -0.026285645, + -0.034168437, + 0.0020141702, + 0.036718756, + -0.0012244415, + -0.014120926, + 0.011382234, + -0.002466996, + 0.032052837, + 0.05129613, + -0.020243134, + 0.0147440145, + -0.013476102, + -0.019562082, + 0.039471935, + -0.0047999555, + 0.006321451, + 0.0026734846, + 0.055237528, + -0.008629052, + 0.0076944195, + -0.004137018, + -0.0033454786, + 0.015852533, + 0.0052056876, + 0.012193698, + 0.012729845, + -0.033733726, + -0.047412694, + 0.020895204, + 0.024083098, + 0.009512968, + -0.021648705, + 0.0016174945, + -0.060859814, + -0.047035944, + -0.0221124, + -0.0013412706, + -0.0034106853, + -0.02046049, + -0.017301576, + 0.019170841, + 0.026372585, + 9.8433055e-05, + -0.031936914, + -0.011215595, + 0.015041068, + 0.06004835, + -0.0091144815, + 0.005531722, + -0.0069517847, + 0.02175014, + -0.02918373, + -0.0032512906, + -0.00060905097, + 0.046079572, + -0.009433271, + -0.020692337, + -0.027444879, + -0.0026426925, + -0.0058613797, + -0.019866383, + -0.0096651185, + -0.012766071, + -0.020214153, + 0.002615523, + 0.049702182, + 0.008237811, + 0.007679929, + -0.038080856, + 0.013265991, + 0.011193858, + -0.0069083134, + 0.008049435, + 0.005165839, + -0.0035084959, + -0.030111117, + -0.0058831153, + -0.0035247975, + 0.012526979, + -0.010578016, + -0.014041228, + -0.0122444155, + -0.03721143, + -0.0057418337, + -0.0076002316, + 0.0023836761, + 0.0056440234, + 0.0015993814, + -0.006495336, + -0.0028799733, + -0.0044123367, + -0.01576559, + -0.021315426, + 0.01911288, + -0.022068929, + -0.037617162, + -0.001587608, + -0.0063794125, + 0.017605875, + 0.0054918737, + -0.019866383, + -0.041471615, + -0.007274197, + -0.032516528, + 0.026416058, + 0.020040268, + -0.011882154, + -0.0008101057, + 0.009911455, + -0.017272595, + 0.028053477, + -0.002573863, + 0.05468689, + -0.03689264, + 0.004408714, + -0.0013965154, + -0.021286445, + 0.017852213, + 0.019214313, + -0.04121079, + 0.029908251, + 0.003461402, + 0.038515568, + -0.009353574, + 0.029125769, + -0.0027459369, + 0.02985029, + -0.0030339342, + -0.018678168, + 0.040689133, + -0.006488091, + 0.029676406, + 0.03625506, + -0.009788287, + -0.019083899, + 0.012744335, + -0.01090405, + 0.044137858, + -0.005995416, + -0.016837882, + 0.022025457, + 0.012389319, + -0.046253458, + -0.018388359, + 0.017316066, + -0.012171963, + 0.00987523, + 0.0040500755, + 0.001626551, + -0.009440516, + -0.028879432, + -0.002937935, + -0.017316066, + -0.005951945, + 0.011983587, + 0.04094996, + -0.01975046, + -0.016649507, + 0.013715194, + 0.029343124, + -0.00313899, + 0.013381914, + -0.041181806, + -0.0321108, + -0.011317028, + 0.033414938, + -0.011230085, + -0.022605075, + -0.008049435, + -0.013483347, + -0.0008010492, + 0.0010125189, + -0.0443697, + -0.034690093, + 0.011990832, + 0.0013385536, + 0.0040283403, + -0.008629052, + -0.044137858, + -0.04828212, + -0.022213832, + -0.024677206, + -0.069206305, + 0.018214474, + -0.0010704807, + 0.0005184858, + 0.009773796, + 0.013519573, + 0.018721638, + 0.003521175, + -0.0237788, + -0.012831277 + ] + } + ], + "model": "text-embedding-3-small", + "usage": { + "prompt_tokens": 7, + "total_tokens": 7 + } + } + headers: + CF-RAY: + - 939c1708fdf87961-NRT + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 03 May 2025 01:47:23 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '33338' + openai-model: + - text-embedding-3-small + openai-organization: test_openai_org_id + openai-processing-ms: + - '89' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - envoy-router-6546dd5b5b-cgzc4 + x-envoy-upstream-service-time: + - '27' + x-ratelimit-limit-requests: + - '5000' + x-ratelimit-limit-tokens: + - '5000000' + x-ratelimit-remaining-requests: + - '4999' + x-ratelimit-remaining-tokens: + - '4999992' + x-ratelimit-reset-requests: + - 12ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3db89a0fc1f707874dd19823da0aab4b + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/inference_conformance.yaml b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/inference_conformance.yaml new file mode 100644 index 00000000..2abb443f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/inference_conformance.yaml @@ -0,0 +1,134 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "user", + "content": "Say this is a test" + } + ], + "model": "gpt-4o-mini", + "stream": false + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '106' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.54.3 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.54.3 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.6 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-ASYMQRl3A3DXL9FWCK9tnGRcKIO7q", + "object": "chat.completion", + "created": 1731368630, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "This is a test.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 5, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8e122593ff368bc8-SIN + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Mon, 11 Nov 2024 23:43:50 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '765' + openai-organization: test_openai_org_id + openai-processing-ms: + - '287' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '200000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '199977' + x-ratelimit-reset-requests: + - 8.64s + x-ratelimit-reset-tokens: + - 6ms + x-request-id: + - req_58cff97afd0e7c0bba910ccf0b044a6f + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/tool_calling_conformance.yaml b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/tool_calling_conformance.yaml new file mode 100644 index 00000000..ebebb206 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/tool_calling_conformance.yaml @@ -0,0 +1,342 @@ +interactions: +- request: + body: |- + { + "messages": [ + { + "role": "system", + "content": "You're a helpful assistant." + }, + { + "role": "user", + "content": "What's the weather in Seattle and San Francisco today?" + } + ], + "model": "gpt-4o-mini", + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. Boston, MA" + } + }, + "required": [ + "location" + ], + "additionalProperties": false + } + } + } + ] + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '543' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.26.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.26.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-ASv9ZqgNAOJAOLYMgdmxouatKXJlk", + "object": "chat.completion", + "created": 1731456245, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_O8NOz8VlxosSASEsOY7LDUcP", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"Seattle, WA\"}" + } + }, + { + "id": "call_3m7cyuckijnpiWr6tq0Tl8Mg", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"San Francisco, CA\"}" + } + } + ], + "refusal": null + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 75, + "completion_tokens": 51, + "total_tokens": 126, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_0ba0d124f1" + } + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8e1a8098ac5ae167-MRS + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Wed, 13 Nov 2024 00:04:06 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '1308' + openai-organization: test_openai_org_id + openai-processing-ms: + - '937' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999960' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3cd7152d2c8c10b4f354b27165f6c2b5 + status: + code: 200 + message: OK +- request: + body: |- + { + "messages": [ + { + "role": "system", + "content": "You're a helpful assistant." + }, + { + "role": "user", + "content": "What's the weather in Seattle and San Francisco today?" + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_O8NOz8VlxosSASEsOY7LDUcP", + "function": { + "arguments": "{\"location\": \"Seattle, WA\"}", + "name": "get_current_weather" + }, + "type": "function" + }, + { + "id": "call_3m7cyuckijnpiWr6tq0Tl8Mg", + "function": { + "arguments": "{\"location\": \"San Francisco, CA\"}", + "name": "get_current_weather" + }, + "type": "function" + } + ] + }, + { + "role": "tool", + "content": "50 degrees and raining", + "tool_call_id": "call_O8NOz8VlxosSASEsOY7LDUcP" + }, + { + "role": "tool", + "content": "70 degrees and sunny", + "tool_call_id": "call_3m7cyuckijnpiWr6tq0Tl8Mg" + } + ], + "model": "gpt-4o-mini" + } + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + authorization: + - Bearer test_openai_api_key + connection: + - keep-alive + content-length: + - '746' + content-type: + - application/json + cookie: + - test_cookie + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.26.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.26.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: |- + { + "id": "chatcmpl-ASv9aQnGndy04lqKoPRagym1eEaQK", + "object": "chat.completion", + "created": 1731456246, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Today, Seattle is experiencing 50 degrees and raining, while San Francisco has a pleasant 70 degrees and sunny weather.", + "refusal": null + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 99, + "completion_tokens": 24, + "total_tokens": 123, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "system_fingerprint": "fp_f59a81427f" + } + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8e1a80a39c71e167-MRS + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Wed, 13 Nov 2024 00:04:07 GMT + Server: + - cloudflare + Set-Cookie: test_set_cookie + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '871' + openai-organization: test_openai_org_id + openai-processing-ms: + - '477' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999948' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_193c74758ea30e77e55afe931e89fd6c + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/__init__.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/__init__.py new file mode 100644 index 00000000..e57cf4ab --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/__init__.py @@ -0,0 +1,2 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/embedding.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/embedding.py new file mode 100644 index 00000000..f18cae22 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/embedding.py @@ -0,0 +1,47 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: openai-v2 embeddings.""" + +from __future__ import annotations + +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class EmbeddingScenario(Scenario): + expected_spans = ("embeddings",) + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("embedding_conformance.yaml"): + OpenAI().embeddings.create( + input="The quick brown fox jumps over the lazy dog", + model="text-embedding-3-small", + ) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/inference.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/inference.py new file mode 100644 index 00000000..b5efb333 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/inference.py @@ -0,0 +1,50 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: openai-v2 chat completion (inference).""" + +from __future__ import annotations + +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + + +class InferenceScenario(Scenario): + expected_spans = ("chat",) + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("inference_conformance.yaml"): + OpenAI().chat.completions.create( + messages=[ + {"role": "user", "content": "Say this is a test"} + ], + model="gpt-4o-mini", + stream=False, + ) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/tool_calling.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/tool_calling.py new file mode 100644 index 00000000..174d2928 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conformance/tool_calling.py @@ -0,0 +1,128 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Conformance scenario: openai-v2 chat completion with tool calls.""" + +from __future__ import annotations + +import json +from typing import Any + +from openai import OpenAI + +from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.test.weaver_live_check import LiveCheckReport +from opentelemetry.test_util_genai.conformance import Scenario +from opentelemetry.test_util_genai.instrumentor import instrument + +DEFAULT_MODEL = "gpt-4o-mini" +WEATHER_TOOL_PROMPT = [ + {"role": "system", "content": "You're a helpful assistant."}, + { + "role": "user", + "content": "What's the weather in Seattle and San Francisco today?", + }, +] +# Tool outputs are pinned to the recorded cassette's second request body. +WEATHER_BY_LOCATION: dict[str, str] = { + "Seattle, WA": "50 degrees and raining", + "San Francisco, CA": "70 degrees and sunny", +} + + +def _get_current_weather_tool_definition() -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. Boston, MA", + }, + }, + "required": ["location"], + "additionalProperties": False, + }, + }, + } + + +def _execute_weather_tool(arguments: str) -> str: + location = json.loads(arguments)["location"] + return WEATHER_BY_LOCATION[location] + + +class ToolCallingScenario(Scenario): + expected_spans = ("chat",) + expected_metrics = ( + "gen_ai.client.operation.duration", + "gen_ai.client.token.usage", + ) + + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: + with instrument( + OpenAIInstrumentor(), + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + semconv="gen_ai_latest_experimental", + content_capture="SPAN_ONLY", + ): + with vcr.use_cassette("tool_calling_conformance.yaml"): + client = OpenAI() + messages: list[Any] = list(WEATHER_TOOL_PROMPT) + + first = client.chat.completions.create( + messages=messages, + model=DEFAULT_MODEL, + tool_choice="auto", + tools=[_get_current_weather_tool_definition()], + ) + + assistant_message = first.choices[0].message + messages.append( + assistant_message.model_dump(exclude_none=True) + ) + for tc in assistant_message.tool_calls or []: + messages.append( + { + "role": "tool", + "content": _execute_weather_tool( + tc.function.arguments + ), + "tool_call_id": tc.id, + } + ) + + client.chat.completions.create( + messages=messages, + model=DEFAULT_MODEL, + ) + + def validate(self, report: LiveCheckReport) -> None: + super().validate(report) + operations = [ + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == "gen_ai.operation.name" + ] + assert operations == ["chat", "chat"], ( + "Tool calling exercises two chat completions (initial request and " + f"follow-up with tool results); saw spans {operations}" + ) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py index 7c90a736..f8ec4714 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py @@ -13,9 +13,11 @@ from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.sampling import ALWAYS_OFF from opentelemetry.test_util_genai.instrumentor import instrument -from opentelemetry.test_util_genai.vcr import scrub_response_headers_overwrite -pytest_plugins = ["opentelemetry.test_util_genai.fixtures"] +pytest_plugins = [ + "opentelemetry.test_util_genai.fixtures", + "opentelemetry.test_util_genai.vcr", +] @pytest.fixture(autouse=True) @@ -36,6 +38,10 @@ def async_openai_client(): @pytest.fixture(scope="module") def vcr_config(): + from opentelemetry.test_util_genai.vcr import ( # noqa: PLC0415 + scrub_response_headers_overwrite, + ) + return { "filter_headers": [ ("cookie", "test_cookie"), diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_conformance.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_conformance.py new file mode 100644 index 00000000..c2c8abe3 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_conformance.py @@ -0,0 +1,50 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance tests for openai-v2. + +Each scenario runs the instrumentation against a recorded API call and +validates the emitted telemetry against the GenAI semantic conventions via +Weaver live-check. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +# Skip collection when weaver_live_check isn't installed (non-conformance envs). +pytest.importorskip("opentelemetry.test.weaver_live_check") + +from opentelemetry.test.weaver_live_check import WeaverLiveCheck # noqa: E402 +from opentelemetry.test_util_genai.conformance import ( # noqa: E402 + Scenario, + run_conformance, +) + +from .conformance.embedding import EmbeddingScenario +from .conformance.inference import InferenceScenario +from .conformance.tool_calling import ToolCallingScenario + +pytestmark = pytest.mark.conformance + + +@pytest.mark.parametrize( + "scenario", + [ + InferenceScenario(), + pytest.param( + EmbeddingScenario(), + marks=pytest.mark.skip( + reason="openai-v2 embeddings emit legacy gen_ai.system in experimental mode" + ), + ), + ToolCallingScenario(), + ], + ids=lambda s: type(s).__name__, +) +def test_conformance( + scenario: Scenario, vcr: Any, weaver_live_check: WeaverLiveCheck +) -> None: + run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) diff --git a/policies/genai_content_validation.rego b/policies/genai_content_validation.rego new file mode 100644 index 00000000..f5a4c89d --- /dev/null +++ b/policies/genai_content_validation.rego @@ -0,0 +1,53 @@ +# Validates the JSON payload of GenAI content attributes against the +# semconv JSON schemas. Schema constants (_schema_*) are defined in +# _schemas.rego, which is generated at test-run time from the semconv +# repository (docs/gen-ai/*.json) and placed alongside this file. +# Weaver only loads .rego files from --advice-policies, so schemas are +# inlined as Rego constants rather than loaded as OPA data documents. + +package live_check_advice + +import rego.v1 + +_genai_content_schemas := { + "gen_ai.input.messages": _schema_input_messages, + "gen_ai.output.messages": _schema_output_messages, + "gen_ai.system_instructions": _schema_system_instructions, + "gen_ai.tool.definitions": _schema_tool_definitions, + "gen_ai.retrieval.documents": _schema_retrieval_documents, +} + +deny contains result if { + input.sample.attribute + attr_name := input.sample.attribute.name + attr_value := input.sample.attribute.value + is_string(attr_value) + + schema := _genai_content_schemas[attr_name] + # Skip when the schema constant isn't present in the pinned semconv + # version yet (the script emits `null` stubs for forward-looking + # attributes like `gen_ai.tool.definitions` until upstream catches up). + schema != null + + parsed := json.unmarshal(attr_value) + + [matched, errors] := json.match_schema(parsed, schema) + not matched + + # PolicyFinding format per + # https://github.com/open-telemetry/weaver/blob/main/crates/weaver_live_check/README.md#policyfinding + # (id / level / context / message; signal_* omitted because the sample + # is attribute-level and weaver doesn't surface the parent span here). + result := { + "id": "genai_content_schema", + "level": "violation", + "context": { + "attribute": attr_name, + "errors": errors, + }, + "message": sprintf( + "Attribute '%v' value does not conform to the GenAI schema: %v", + [attr_name, errors], + ), + } +} diff --git a/policies/genai_span_validation.rego b/policies/genai_span_validation.rego new file mode 100644 index 00000000..e85c8453 --- /dev/null +++ b/policies/genai_span_validation.rego @@ -0,0 +1,246 @@ +# Validates GenAI span shape beyond what weaver's semconv-registry-driven +# checks already enforce. The registry validates per-attribute requirements +# (name, type, presence) for spans matching its definitions; this file adds +# cross-cutting span-level invariants the registry can't easily express. +# +# Two classes of rules, both keyed on `gen_ai.operation.name`: +# +# 1. Span name format → `violation` +# (`{operation_name} {request_model}` for inference / embeddings, +# `{operation_name} {agent_name}` for invoke_agent / create_agent, +# `{operation_name} {tool_name}` for execute_tool). +# +# 2. Per-operation expected attributes → `violation` +# Combines `Required` (always must be set) and the always-emit subset +# of `Recommended` (e.g. response model/id, token usage on inference) +# into one manifest per operation. Sourced from the rendered tables in +# semantic-conventions/docs/gen-ai/gen-ai-spans.md and +# gen-ai-agent-spans.md (the MD flattens the YAML inheritance chain +# via `extends:`, so it's the right place to source from). +# +# The "set when known" Recommended subset (sampling parameters like +# `frequency_penalty`, `max_tokens`; provider-side caches; conditionally- +# emitted things like `gen_ai.response.time_to_first_chunk` for streaming) +# is deliberately NOT flagged here — those depend on user input or on the +# request shape and would produce noisy false positives. Cross-attribute +# conditional rules (e.g. "if streaming, response.time_to_first_chunk +# SHOULD be set") would also belong here. +# +# Required attributes are also flagged by weaver's registry-driven +# validation. Listing them here too is intentional: rego rules give us +# stable advice ids to grep for in reports and let us tighten the check +# regardless of how the registry classifies the gap. +# +# Attribute access: weaver hands rego a span sample where `attributes` is a +# **list** of `{name, value, type}` objects, not a dict — `_attr(name)` +# walks that list and returns the value (or `null` if absent). + +package live_check_advice + +import rego.v1 + +# ─── Operation classification ─────────────────────────────────────────────── +# +# Mirrors the semconv `gen_ai.operation.name` enum +# (model/gen-ai/registry.yaml). When semconv adds a new operation, append it +# to the matching set below — or leave it out if the new operation has its +# own span definition with different conventions. + +_inference_ops := {"chat", "generate_content", "text_completion"} + +_embeddings_ops := {"embeddings"} + +_tool_ops := {"execute_tool"} + +_invoke_agent_ops := {"invoke_agent"} + +_create_agent_ops := {"create_agent"} + +# ─── Span name format (violation) ─────────────────────────────────────────── + +_span_name_keyed_attr["chat"] := "gen_ai.request.model" +_span_name_keyed_attr["generate_content"] := "gen_ai.request.model" +_span_name_keyed_attr["text_completion"] := "gen_ai.request.model" +_span_name_keyed_attr["embeddings"] := "gen_ai.request.model" +_span_name_keyed_attr["execute_tool"] := "gen_ai.tool.name" +_span_name_keyed_attr["invoke_agent"] := "gen_ai.agent.name" +_span_name_keyed_attr["create_agent"] := "gen_ai.agent.name" + +# Span name SHOULD be `{op}` (when the keyed attribute is absent) or +# `{op} {value}` (when present). Mirrors the "SHOULD append when known" +# guidance in semconv. +# +# Avoid `%v ` patterns in sprintf: weaver 0.22.1's OPA-based sprintf +# consumes a single space character immediately following any verb (`%v`, +# `%s`, `%d`) — interpreting it as Go's space-flag — so `%v %v` produces +# `` instead of ` `. We use `concat` for the literal-space +# joins below. +deny contains _span_finding( + "genai_span_name_format", + "violation", + input.sample.span, + { + "operation": op, + "keyed_attr": keyed_attr, + "expected_form": concat("", [op, " or '", op, " <", keyed_attr, ">'"]), + }, + concat("", [ + op, " span name should be '", + op, "' or '", + op, " ', got '", + input.sample.span.name, "'", + ]), +) if { + input.sample.span + op := _attr_value(input.sample.span, "gen_ai.operation.name") + keyed_attr := _span_name_keyed_attr[op] + not _valid_op_and_attr_span_name(input.sample.span, op, keyed_attr) +} + +# ─── Per-operation expected attributes (violation) ────────────────────────── + +_expected_for_op["chat"] := _inference_expected + +_expected_for_op["generate_content"] := _inference_expected + +_expected_for_op["text_completion"] := _inference_expected + +_expected_for_op["embeddings"] := _embeddings_expected + +_expected_for_op["execute_tool"] := _execute_tool_expected + +_expected_for_op["invoke_agent"] := _invoke_agent_expected + +_expected_for_op["create_agent"] := _create_agent_expected + +_expected_for_op["retrieval"] := _retrieval_expected + +# Inference (chat / generate_content / text_completion). +# Required: gen_ai.operation.name, gen_ai.provider.name. +# Always-emit Recommended: response model/id, finish reasons, token usage, +# server.address. Sampling parameters (frequency_penalty, max_tokens, …), +# cache counters, and `gen_ai.response.time_to_first_chunk` (streaming-only) +# are conditional and not flagged here. +_inference_expected := { + "gen_ai.operation.name", + "gen_ai.provider.name", + "gen_ai.response.model", + "gen_ai.response.id", + "gen_ai.response.finish_reasons", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + # "server.address", sometimes not available +} + +# Embeddings. +# Required: gen_ai.operation.name, gen_ai.provider.name. +# Always-emit Recommended: dimension.count, response.model, input tokens, +# server.address. (`gen_ai.request.encoding_formats` is conditional.) +_embeddings_expected := { + "gen_ai.operation.name", + "gen_ai.provider.name", + "gen_ai.embeddings.dimension.count", + "gen_ai.response.model", + "gen_ai.usage.input_tokens", + # "server.address", sometimes not available +} + +# Tool execution. +# Required: gen_ai.operation.name, gen_ai.tool.name. +# Recommended-when-available: gen_ai.tool.call.id, gen_ai.tool.type. (Tool +# description is genuinely optional per provider — not flagged.) +_execute_tool_expected := { + "gen_ai.operation.name", + "gen_ai.tool.name", + "gen_ai.tool.call.id", + "gen_ai.tool.type", +} + +# Invoke agent. +# Required: gen_ai.operation.name, gen_ai.provider.name. +# Most instrumentations should have agent.id; flag it as always-emit. +_invoke_agent_expected := { + "gen_ai.operation.name", + "gen_ai.provider.name", + "gen_ai.agent.id", +} + +# Create agent. After creation completes the provider returns an agent.id; +# flag it as always-emit on create_agent. +_create_agent_expected := { + "gen_ai.operation.name", + "gen_ai.provider.name", + "gen_ai.agent.id", +} + +# Retrieval. Only gen_ai.operation.name is unconditionally required. +_retrieval_expected := { + "gen_ai.operation.name", + # "server.address", sometimes not available +} + +# Per expected attribute, one violation if missing. +deny contains _span_finding( + "genai_expected_attribute_missing", + "violation", + input.sample.span, + { + "operation": op, + "missing_attribute": attr_name, + }, + sprintf( + "Span '%v' (operation '%v') is missing expected attribute '%v'", + [input.sample.span.name, op, attr_name], + ), +) if { + input.sample.span + op := _attr_value(input.sample.span, "gen_ai.operation.name") + expected := _expected_for_op[op] + some attr_name in expected + not _has_attr(input.sample.span, attr_name) +} + +# ─── Helpers ──────────────────────────────────────────────────────────────── + +# Span attributes arrive as `[{"name": ..., "value": ..., "type": ...}]`. + +# True when the span has an attribute named `name`. +_has_attr(span, name) if { + some attr in span.attributes + attr.name == name +} + +# Returns the value of the named attribute. Undefined (rule body fails) when +# the attribute isn't present — callers must guard with `_has_attr` first if +# they need to distinguish "absent" from "set to a falsy value". +_attr_value(span, name) := value if { + some attr in span.attributes + attr.name == name + value := attr.value +} + +# A valid span name is either exactly `{op}` (when the keyed attribute is +# absent) or `{op} {value}` (when present). +_valid_op_and_attr_span_name(span, op, attr_key) if { + span.name == op + not _has_attr(span, attr_key) +} + +_valid_op_and_attr_span_name(span, op, attr_key) if { + value := _attr_value(span, attr_key) + # concat (not sprintf): see the note above the deny rule. sprintf("%v %v", ...) + # silently produces "" with no space, so every span with a `{op} {value}` + # name would be reported as a violation. + span.name == concat(" ", [op, value]) +} + +# PolicyFinding format per +# https://github.com/open-telemetry/weaver/blob/main/crates/weaver_live_check/README.md#policyfinding +_span_finding(id, level, span, context, message) := { + "id": id, + "level": level, + "signal_type": "span", + "signal_name": span.name, + "context": context, + "message": message, +} diff --git a/pyproject.toml b/pyproject.toml index ba3d5d4b..122d7dd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ opentelemetry-instrumentation-openai-v2 = { workspace = true } opentelemetry-instrumentation-openai-agents-v2 = { workspace = true } opentelemetry-instrumentation-weaviate = { workspace = true } + # https://docs.astral.sh/uv/reference/settings/#workspace [tool.uv.workspace] members = [ diff --git a/pytest.ini b/pytest.ini index 013d2c55..4e8b8a74 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,3 +2,5 @@ addopts = -rs -v log_cli = true log_cli_level = warning +markers = + conformance: GenAI semconv conformance scenario (run via the *-conformance tox envs) diff --git a/tox.ini b/tox.ini index 27cdab84..e7a4081f 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ envlist = ; instrumentation-openai py3{10,11,12,13,14}-test-instrumentation-openai-v2-{oldest,latest} pypy3-test-instrumentation-openai-v2-{oldest,latest} + py3{12,13}-test-instrumentation-openai-v2-conformance lint-instrumentation-openai-v2 ; instrumentation-openai-agents @@ -25,6 +26,7 @@ envlist = ; instrumentation-anthropic py3{10,11,12,13,14}-test-instrumentation-anthropic-{oldest,latest} # Disabling pypy3 as jiter (anthropic dep) requires PyPy 3.11+ + py3{12,13}-test-instrumentation-anthropic-conformance lint-instrumentation-anthropic ; instrumentation-claude-agent-sdk @@ -36,6 +38,7 @@ envlist = ; TODO: add tests/requirements.{oldest,latest}.txt and langchain-{oldest,latest} ; factors below; declare opentelemetry-util-genai in the package pyproject.toml ; (latent bug — code imports it but it's not in runtime deps). + py3{12,13}-test-instrumentation-langchain-conformance lint-instrumentation-langchain ; instrumentation-weaviate @@ -70,6 +73,7 @@ pytest_deps = pytest-vcr -e {toxinidir}/util/opentelemetry-test-util-genai deps = + conformance: -r {toxinidir}/dev-requirements-conformance.txt lint: -r dev-requirements.txt coverage: pytest coverage: pytest-cov @@ -81,6 +85,8 @@ deps = openai-latest: {[testenv]test_deps} openai-latest: {[testenv]pytest_deps} openai-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt + openai-conformance: {[testenv]pytest_deps} + openai-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt lint-instrumentation-openai-v2: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt openai_agents-oldest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-agents-v2/tests/requirements.oldest.txt @@ -102,6 +108,8 @@ deps = anthropic-latest: {[testenv]test_deps} anthropic-latest: {[testenv]pytest_deps} anthropic-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-anthropic/tests/requirements.latest.txt + anthropic-conformance: {[testenv]pytest_deps} + anthropic-conformance: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-anthropic/tests/requirements.latest.txt lint-instrumentation-anthropic: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-anthropic/tests/requirements.oldest.txt claude-agent-sdk-oldest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-claude-agent-sdk/tests/requirements.oldest.txt @@ -111,6 +119,14 @@ deps = claude-agent-sdk-latest: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-claude-agent-sdk/tests/requirements.latest.txt lint-instrumentation-claude-agent-sdk: -r {toxinidir}/instrumentation/opentelemetry-instrumentation-claude-agent-sdk/tests/requirements.oldest.txt + ; Langchain has no oldest/latest matrix yet (TODO above); pin the client + ; libs the conftest imports inline. + langchain-conformance: {[testenv]pytest_deps} + langchain-conformance: {toxinidir}/instrumentation/opentelemetry-instrumentation-langchain[instruments] + langchain-conformance: langchain-openai + langchain-conformance: langchain-aws + langchain-conformance: langchain-google-genai + langchain-conformance: boto3 util-genai: {[testenv]test_deps} util-genai: {[testenv]pytest_deps} @@ -130,7 +146,8 @@ commands_pre = coverage: python {toxinidir}/scripts/eachdist.py install --editable commands = - test-instrumentation-openai-v2: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/tests {posargs} + test-instrumentation-openai-v2-{oldest,latest}: pytest -m "not conformance" {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/tests {posargs} + test-instrumentation-openai-v2-conformance: pytest -m conformance {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/tests --vcr-record=none {posargs} lint-instrumentation-openai-v2: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-openai-v2" test-instrumentation-openai_agents-v2: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-agents-v2/tests {posargs} lint-instrumentation-openai_agents-v2: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-openai-agents-v2" @@ -138,12 +155,14 @@ commands = test-instrumentation-google-genai: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-google-genai/tests --vcr-record=none {posargs} lint-instrumentation-google-genai: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-google-genai" - test-instrumentation-anthropic: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-anthropic/tests --vcr-record=none {posargs} + test-instrumentation-anthropic-{oldest,latest}: pytest -m "not conformance" {toxinidir}/instrumentation/opentelemetry-instrumentation-anthropic/tests --vcr-record=none {posargs} + test-instrumentation-anthropic-conformance: pytest -m conformance {toxinidir}/instrumentation/opentelemetry-instrumentation-anthropic/tests --vcr-record=none {posargs} lint-instrumentation-anthropic: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-anthropic" test-instrumentation-claude-agent-sdk: pytest {toxinidir}/instrumentation/opentelemetry-instrumentation-claude-agent-sdk/tests --vcr-record=none {posargs} lint-instrumentation-claude-agent-sdk: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-claude-agent-sdk" + test-instrumentation-langchain-conformance: pytest -m conformance {toxinidir}/instrumentation/opentelemetry-instrumentation-langchain/tests --vcr-record=none {posargs} lint-instrumentation-langchain: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-langchain" lint-instrumentation-weaviate: sh -c "cd instrumentation && ruff check opentelemetry-instrumentation-weaviate" diff --git a/util/opentelemetry-test-util-genai/pyproject.toml b/util/opentelemetry-test-util-genai/pyproject.toml index 2cbe24f4..4979dde1 100644 --- a/util/opentelemetry-test-util-genai/pyproject.toml +++ b/util/opentelemetry-test-util-genai/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "opentelemetry-instrumentation", "opentelemetry-sdk", "opentelemetry-semantic-conventions", + "opentelemetry-test-utils", "opentelemetry-util-genai", "pytest", "PyYAML", diff --git a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py new file mode 100644 index 00000000..9db6dd5d --- /dev/null +++ b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/_setup_weaver.py @@ -0,0 +1,296 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Provision advice policies and the semconv registry for weaver. + +The registry source is ``open-telemetry/semantic-conventions-genai``, +whose ``model/manifest.yaml`` depends on a filtered copy of the upstream +``open-telemetry/semantic-conventions`` registry with the migrated GenAI +subdirectories and groups stripped out (so Weaver doesn't see duplicate +group ids). This module reproduces the genai repo's ``make filter-upstream`` +target in Python. + +Once https://github.com/open-telemetry/weaver/issues/1455 is fixed and the +genai repo drops its ``.build/sc-upstream-filtered`` workaround, the +filter step and migration tables below become dead code. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import shutil +import tarfile +import tempfile +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +# Bounds the fetch of the registry tarballs so a slow/unreachable +# GitHub doesn't hang conformance runs until the OS-level socket timeout. +_FETCH_TIMEOUT_SECONDS = 60 + +# Mirrors `SC_UPSTREAM_MIGRATED_{DIRS,GROUPS}` in the genai repo's Makefile. +_MIGRATED_DIRS: tuple[str, ...] = ("gen-ai", "mcp", "openai") +_MIGRATED_GROUPS: tuple[tuple[str, str], ...] = ( + ("aws/registry.yaml", "registry.aws.bedrock"), +) + +logger = logging.getLogger(__name__) + + +def _workspace_root() -> Path: + here = Path(__file__).resolve() + for ancestor in here.parents: + if (ancestor / "versions.env").is_file() and ( + ancestor / "policies" + ).is_dir(): + return ancestor + raise RuntimeError( + f"Could not locate the genai workspace root (walked up from {here} " + "looking for versions.env + policies/)." + ) + + +def _load_version_pins(path: Path) -> dict[str, str]: + pins: dict[str, str] = {} + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + key, sep, value = line.partition("=") + if not sep: + raise RuntimeError(f"Invalid version pin in {path}: {raw_line!r}") + pins[key.strip()] = value.strip().strip('"').strip("'") + return pins + + +def _cache_dir() -> Path: + override = os.environ.get("SEMCONV_CACHE") + if override: + return Path(override) + return Path.home() / ".cache" / "otel-conformance" / "semconv" + + +def _download_and_extract(url: str, target: Path, label: str) -> None: + """Download ``url`` (a .tar.gz) and extract its single top-level dir into ``target``.""" + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + dir=str(target.parent), prefix=f"{label}-" + ) as tmp: + tmp_path = Path(tmp) + archive_path = tmp_path / "src.tar.gz" + extract_dir = tmp_path / "extract" + extract_dir.mkdir() + + logger.info("Fetching %s from %s", label, url) + try: + with ( + urllib.request.urlopen( + url, timeout=_FETCH_TIMEOUT_SECONDS + ) as response, + archive_path.open("wb") as out, + ): + shutil.copyfileobj(response, out) + except (TimeoutError, urllib.error.URLError) as exc: + raise RuntimeError( + f"Failed to fetch {label} from {url}: {exc}" + ) from exc + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(extract_dir, filter="data") + + entries = [p for p in extract_dir.iterdir() if p.is_dir()] + if len(entries) != 1: + raise RuntimeError( + f"Unexpected layout in {label} archive: " + f"{[p.name for p in entries]}" + ) + if target.exists(): + shutil.rmtree(target) + shutil.move(str(entries[0]), str(target)) + + +def _strip_group_block(text: str, group_id: str) -> str: + """Drop the YAML block for ``- id: `` from a Weaver registry file.""" + keep: list[str] = [] + skip = False + prefix = " - id: " + target_line = prefix + group_id + for line in text.splitlines(keepends=True): + if line.startswith(prefix): + skip = line.rstrip("\r\n") == target_line + if not skip: + keep.append(line) + return "".join(keep) + + +def _materialize_filtered_upstream( + genai_root: Path, upstream_root: Path +) -> Path: + """Build ``/.build/sc-upstream-filtered`` from ``upstream_root``.""" + filtered = genai_root / ".build" / "sc-upstream-filtered" + filtered.parent.mkdir(parents=True, exist_ok=True) + if filtered.exists(): + shutil.rmtree(filtered) + shutil.copytree(upstream_root / "model", filtered) + + for migrated in _MIGRATED_DIRS: + migrated_path = filtered / migrated + if migrated_path.exists(): + shutil.rmtree(migrated_path) + + for relative_file, group_id in _MIGRATED_GROUPS: + target = filtered / relative_file + if not target.is_file(): + continue + original = target.read_text(encoding="utf-8") + stripped = _strip_group_block(original, group_id) + if stripped == original: + logger.warning( + "Migrated group %r not found in %s — list may be stale", + group_id, + relative_file, + ) + target.write_text(stripped, encoding="utf-8") + return filtered + + +def _rewrite_manifest_dependency(genai_root: Path, filtered: Path) -> None: + """Bake an absolute ``registry_path`` into ``model/manifest.yaml``. + + Weaver resolves the manifest's relative ``./.build/sc-upstream-filtered`` + against the *current working directory*, not the manifest file, so a + relative path only works when weaver is invoked from the genai repo root. + """ + manifest = genai_root / "model" / "manifest.yaml" + pattern = re.compile( + r"^(\s*registry_path:\s*)\./\.build/sc-upstream-filtered\s*$", + re.MULTILINE, + ) + abs_path = filtered.resolve().as_posix() + new_text, count = pattern.subn( + lambda m: f"{m.group(1)}{abs_path}", + manifest.read_text(encoding="utf-8"), + ) + if count != 1: + raise RuntimeError( + f"Expected exactly one filtered-upstream registry_path entry in " + f"{manifest}, found {count}." + ) + manifest.write_text(new_text, encoding="utf-8") + + +def _provision_genai_root() -> Path: + """Fetch the pinned genai registry, materialize its upstream dependency, return its root.""" + pins = _load_version_pins(_workspace_root() / "versions.env") + try: + genai_ref = pins["SEMCONV_GENAI_REF"] + except KeyError as missing: + raise RuntimeError( + f"versions.env is missing required pin {missing!s}" + ) from missing + + cache_root = _cache_dir() + genai_target = cache_root / f"genai-{genai_ref}" + stamp = genai_target / ".provisioned" + if stamp.is_file(): + return genai_target + + cache_root.mkdir(parents=True, exist_ok=True) + genai_archive_url = ( + "https://github.com/open-telemetry/semantic-conventions-genai/" + f"archive/{genai_ref}.tar.gz" + ) + _download_and_extract( + genai_archive_url, genai_target, label="genai-semconv" + ) + + upstream_pins = _load_version_pins(genai_target / "versions.env") + try: + upstream_version = upstream_pins["SEMCONV_VERSION"] + except KeyError as missing: + raise RuntimeError( + f"genai repo's versions.env is missing {missing!s}" + ) from missing + + upstream_target = cache_root / f"upstream-{upstream_version}" + if not (upstream_target / "model").is_dir(): + upstream_archive_url = ( + "https://github.com/open-telemetry/semantic-conventions/" + f"archive/refs/tags/{upstream_version}.tar.gz" + ) + _download_and_extract( + upstream_archive_url, upstream_target, label="upstream-semconv" + ) + + filtered = _materialize_filtered_upstream(genai_target, upstream_target) + _rewrite_manifest_dependency(genai_target, filtered) + stamp.touch() + return genai_target + + +# `_schema_` constants referenced from +# policies/genai_content_validation.rego. +_GENAI_SCHEMA_FILES: dict[str, str] = { + "input_messages": "gen-ai-input-messages.json", + "output_messages": "gen-ai-output-messages.json", + "system_instructions": "gen-ai-system-instructions.json", + "tool_definitions": "gen-ai-tool-definitions.json", + "retrieval_documents": "gen-ai-retrieval-documents.json", +} + + +def _generate_schemas_rego(schemas: dict[str, Any]) -> str: + lines = [ + "# Auto-generated from semantic-conventions. Do not edit.", + "# Re-generated each time _setup_weaver.policies_dir() runs.", + "package live_check_advice", + "", + "import rego.v1", + "", + ] + for key, schema in schemas.items(): + if schema is None: + lines.append(f"_schema_{key} := null") + else: + # indent=2 to stay under weaver's 1024-char-per-line rego limit. + lines.append(f"_schema_{key} := {json.dumps(schema, indent=2)}") + lines.append("") + return "\n".join(lines) + + +def policies_dir() -> Path: + """Write ``policies/_schemas.rego`` and return the policies directory.""" + docs_genai = _provision_genai_root() / "docs" / "gen-ai" + + schemas: dict[str, Any] = {} + for key, filename in _GENAI_SCHEMA_FILES.items(): + schema_path = docs_genai / filename + if schema_path.exists(): + # OPA's json.match_schema can't fetch the draft-07 meta-schema at + # eval time; swap the external $ref for a local "must be an object". + schemas[key] = json.loads( + schema_path.read_text(encoding="utf-8").replace( + '"$ref": "http://json-schema.org/draft-07/schema#"', + '"type": "object"', + ) + ) + else: + logger.warning( + "GenAI schema not found: %s (emitting null stub)", schema_path + ) + schemas[key] = None + + policies = _workspace_root() / "policies" + (policies / "_schemas.rego").write_text( + _generate_schemas_rego(schemas), encoding="utf-8" + ) + return policies + + +def semconv_registry() -> Path: + """Return the path to ``/model`` for the pinned ref.""" + return _provision_genai_root() / "model" diff --git a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/conformance.py b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/conformance.py new file mode 100644 index 00000000..99037340 --- /dev/null +++ b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/conformance.py @@ -0,0 +1,186 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +"""Per-scenario conformance runner for GenAI instrumentations. + +Intended call shape from a per-package ``tests/test_conformance.py``:: + + pytestmark = pytest.mark.conformance + + @pytest.mark.parametrize( + "scenario", [InferenceScenario(), ToolCallingScenario()] + ) + def test_conformance(scenario, vcr, weaver_live_check): + report = run_conformance(scenario, vcr=vcr, weaver=weaver_live_check) + # Optionally layer lib-specific assertions on `report` here. + +The module-level ``pytestmark = pytest.mark.conformance`` is required: the +``*-conformance`` tox envs select these tests via ``-m conformance``, and the +regular ``*-{oldest,latest}`` envs deselect them via ``-m "not conformance"``. + +Each ``tests/conformance/.py`` defines a :class:`Scenario` subclass with: + +- ``expected_spans`` — ``gen_ai.operation.name`` values that must appear in + the report's span samples. +- ``expected_metrics`` — metric names that must appear in + ``statistics.seen_registry_metrics``. +- ``run(*, tracer_provider, meter_provider, logger_provider, vcr)`` — wires + the instrumentor against the providers and exercises one semconv operation + type's happy path inside ``vcr.use_cassette(...)``. +- ``validate(report)`` — asserts the emitted telemetry matches the scenario. + The base implementation enforces ``expected_spans`` / ``expected_metrics`` + presence; per-scenario overrides call ``super().validate(report)`` and + layer on additional checks against the weaver report. +""" + +from __future__ import annotations + +import json +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, ClassVar + +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import ( + OTLPMetricExporter, +) +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import SimpleLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.test.weaver_live_check import ( + LiveCheckReport, + WeaverLiveCheck, +) + + +class Scenario(ABC): + """Base class every ``tests/conformance/.py`` scenario must subclass.""" + + expected_spans: ClassVar[tuple[str, ...]] = () + expected_metrics: ClassVar[tuple[str, ...]] = () + + @abstractmethod + def run( + self, + *, + tracer_provider: TracerProvider, + meter_provider: MeterProvider, + logger_provider: LoggerProvider, + vcr: Any, + ) -> None: ... + + def validate(self, report: LiveCheckReport) -> None: + """Assert the weaver live-check report matches the scenario. + + The base implementation enforces that every ``expected_spans`` and + ``expected_metrics`` entry appears at least once. Subclasses should + override and call ``super().validate(report)`` to layer on extra + scenario-specific checks against the report. + """ + expected_spans = set(self.expected_spans) + seen_spans = _seen_span_operations(report) + missing_spans = expected_spans - seen_spans + assert not missing_spans, ( + f"Expected span operations {sorted(expected_spans)} but weaver " + f"only saw {sorted(seen_spans)} — missing {sorted(missing_spans)}" + ) + + expected_metrics = set(self.expected_metrics) + seen_metrics = _seen_metric_names(report) + missing_metrics = expected_metrics - seen_metrics + assert not missing_metrics, ( + f"Expected metrics {sorted(expected_metrics)} but weaver only " + f"saw {sorted(seen_metrics)} — missing {sorted(missing_metrics)}" + ) + + +def _build_providers( + endpoint: str, +) -> tuple[TracerProvider, MeterProvider, LoggerProvider]: + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + SimpleSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True)) + ) + + # Disable periodic export — metrics flush via the explicit force_flush() + # at the end of the scenario, so the report is deterministic. + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint, insecure=True), + export_interval_millis=2**31 - 1, + ) + meter_provider = MeterProvider(metric_readers=[metric_reader]) + + logger_provider = LoggerProvider() + logger_provider.add_log_record_processor( + SimpleLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint, insecure=True) + ) + ) + + return tracer_provider, meter_provider, logger_provider + + +def _seen_metric_names(report: LiveCheckReport) -> set[str]: + """Names of metrics weaver observed at least one data point for.""" + seen = report["statistics"]["seen_registry_metrics"] + return {name for name, count in seen.items() if count} + + +def _seen_span_operations(report: LiveCheckReport) -> set[str]: + """`gen_ai.operation.name` values observed across the report's span samples.""" + return { + attr["value"] + for entry in report["samples"] + if "span" in entry + for attr in entry["span"]["attributes"] + if attr["name"] == "gen_ai.operation.name" + } + + +def _dump_report(scenario: Scenario, report: LiveCheckReport) -> None: + out = Path("weaver_reports") / f"{type(scenario).__name__}.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report._report, indent=2, sort_keys=True)) # noqa: SLF001 + + +def run_conformance( + scenario: Scenario, + *, + vcr: Any, + weaver: WeaverLiveCheck, +) -> LiveCheckReport: + """Run one conformance scenario and return the weaver report. + + Raises :class:`LiveCheckError` on semconv violations. + """ + tracer_provider, meter_provider, logger_provider = _build_providers( + weaver.otlp_endpoint + ) + + try: + scenario.run( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + logger_provider=logger_provider, + vcr=vcr, + ) + tracer_provider.force_flush() + meter_provider.force_flush() + logger_provider.force_flush() + + report = weaver.end_and_check(timeout=120) + _dump_report(scenario, report) + scenario.validate(report) + return report + finally: + tracer_provider.shutdown() + meter_provider.shutdown() + logger_provider.shutdown() diff --git a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/fixtures.py b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/fixtures.py index d633acb5..6ecbbc37 100644 --- a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/fixtures.py +++ b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/fixtures.py @@ -18,22 +18,34 @@ friends) are deliberately **not** set so tests stay isolated and don't leak across the session. -Two-mode parametrization ------------------------- +Parametrized fixtures +--------------------- -``content_capture`` is a parametrized fixture that yields each -``ContentCapturingMode`` enum value in ``CONTENT_CAPTURE_MODES`` in turn -(``NO_CONTENT`` and ``SPAN_ONLY``). It sets +``content_capture`` yields each ``ContentCapturingMode`` enum value in +``CONTENT_CAPTURE_MODES`` in turn (``NO_CONTENT`` and ``SPAN_ONLY``). It sets ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`` for the duration of the test and restores the previous value afterwards. ``SPAN_AND_EVENT`` and ``EVENT_ONLY`` coverage lives in targeted per-package tests rather than the default matrix. + +Conformance fixture +------------------- + +``weaver_live_check`` yields a started ``WeaverLiveCheck`` for a single +conformance scenario. Consumed by ``tests/test_conformance.py`` via +``opentelemetry.test_util_genai.conformance.run_conformance``. Auto-skips +when the OTLP/gRPC exporter or the ``weaver`` binary aren't available — +local runs typically skip; CI installs ``weaver`` ahead of the +``*-conformance`` tox envs. """ from __future__ import annotations import os +import shutil +import tarfile from collections.abc import Iterator +from typing import Any import pytest @@ -51,6 +63,10 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( InMemorySpanExporter, ) +from opentelemetry.test_util_genai._setup_weaver import ( + policies_dir, + semconv_registry, +) from opentelemetry.util.genai.environment_variables import ( OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT, ) @@ -156,3 +172,46 @@ def content_capture( os.environ[OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT] = ( previous ) + + +# ─── Weaver live-check ────────────────────────────────────────────────────── + + +@pytest.fixture +def weaver_live_check() -> Iterator[Any]: + """Yield a started ``WeaverLiveCheck`` for one conformance scenario. + + Function-scoped so violations don't leak across scenarios. Auto-skips + when the OTLP/gRPC exporter, the ``weaver`` binary, or the + semantic-conventions registry can't be resolved. + """ + try: + import opentelemetry.exporter.otlp.proto.grpc.trace_exporter # noqa: F401, PLC0415 + except ImportError: + pytest.skip("opentelemetry-exporter-otlp-proto-grpc not installed") + + if shutil.which("weaver") is None: + pytest.skip( + "weaver binary not on PATH — install it from " + "https://github.com/open-telemetry/weaver/releases (CI installs " + "it via the test.yml conformance setup step)" + ) + + # WeaverLiveCheck transitively imports the OTLP/gRPC exporter, so it + # stays inside the function body — the probe above is what gates this. + from opentelemetry.test.weaver_live_check import ( # noqa: PLC0415 + WeaverLiveCheck, + ) + + try: + policies = str(policies_dir()) + registry = str(semconv_registry()) + except (OSError, RuntimeError, ValueError, tarfile.TarError) as exc: + pytest.skip(f"could not provision semantic-conventions: {exc}") + + with WeaverLiveCheck( + registry=registry, + policies_dir=policies, + extra_args=["--include-unreferenced"], + ) as weaver: + yield weaver diff --git a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/vcr.py b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/vcr.py index 654ad9e2..78f4632b 100644 --- a/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/vcr.py +++ b/util/opentelemetry-test-util-genai/src/opentelemetry/test_util_genai/vcr.py @@ -130,20 +130,6 @@ def fixture_vcr(vcr: Any) -> Any: return vcr -@pytest.fixture -def vcr_cassette_name(request: pytest.FixtureRequest) -> str: - """Cassette name = the test function name, without parametrize suffix. - - Override of pytest-vcr's default, which uses ``request.node.name`` and - so includes the ``[...]`` parametrize ID — causing tests that fan out - via the shared ``content_capture`` matrix to look for cassettes named - ``test_foo[NO_CONTENT].yaml`` instead of ``test_foo.yaml``. The actual - HTTP request is identical across capture-mode cells, so all cells - share the same cassette. - """ - return request.node.originalname or request.node.name - - def scrub_response_headers( headers_to_scrub: Iterable[str], ) -> Callable[[dict[str, Any]], dict[str, Any]]: diff --git a/uv.lock b/uv.lock index cfb8326d..0f450ec1 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,11 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version < '3.13'", +] [manifest] members = [ @@ -1397,6 +1402,7 @@ dependencies = [ { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-test-utils" }, { name = "opentelemetry-util-genai" }, { name = "pytest" }, { name = "pyyaml" }, @@ -1408,6 +1414,7 @@ requires-dist = [ { name = "opentelemetry-instrumentation" }, { name = "opentelemetry-sdk" }, { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-test-utils" }, { name = "opentelemetry-util-genai", editable = "util/opentelemetry-util-genai" }, { name = "pytest" }, { name = "pyyaml" }, diff --git a/versions.env b/versions.env new file mode 100644 index 00000000..d08850cd --- /dev/null +++ b/versions.env @@ -0,0 +1,9 @@ +# Version pins for conformance tooling. Renovate manages this file via the +# customManagers block in .github/renovate.json5. + +# renovate: datasource=github-releases depName=open-telemetry/weaver versioning=semver-coerced +WEAVER_VERSION=v0.23.0 + +# The genai semconv registry has no tagged releases yet, so we pin a SHA on `main`. +# renovate: datasource=git-refs depName=open-telemetry/semantic-conventions-genai packageName=https://github.com/open-telemetry/semantic-conventions-genai.git versioning=git +SEMCONV_GENAI_REF=8508fbfa5189ae50c7e95aa2fcd90c5c4998cbc7