Skip to content

Commit b669443

Browse files
lmolkovalmolkova
authored
Add conformance testing based on weaver live check (open-telemetry#14)
* add conformance testing based on weaver live check
1 parent 975a98d commit b669443

42 files changed

Lines changed: 4391 additions & 24 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/instructions/instrumentation.instructions.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,12 @@ prefer opt-in or additive. Breaking changes need explicit justification in the P
6969
(`from opentelemetry.test_util_genai.fixtures import *` and
7070
`from opentelemetry.test_util_genai.vcr import fixture_vcr, scrub_response_headers`). Do not
7171
re-implement in-memory provider/exporter setup or the VCR pretty-print serializer locally.
72+
- Conformance: packages ship `tests/conformance/<scenario>.py` modules (each
73+
defining a subclass of
74+
`opentelemetry.test_util_genai.conformance.Scenario` that sets
75+
`expected_spans`, `expected_metrics`, and implements `run(...)`) and a
76+
`tests/test_conformance.py` that runs them via
77+
`opentelemetry.test_util_genai.conformance.run_conformance`.
7278

7379
## 6. Examples
7480

‎.github/renovate.json5‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,18 @@
1313
// requirements.latest.txt files are skipped without this.
1414
"managerFilePatterns": ["/(^|/)requirements\\.latest\\.txt$/"]
1515
},
16+
// Manage WEAVER_VERSION and SEMCONV_GENAI_REF in versions.env via their
17+
// `# renovate:` annotations.
18+
"customManagers": [
19+
{
20+
"customType": "regex",
21+
"managerFilePatterns": ["/^versions\\.env$/"],
22+
"matchStrings": [
23+
"# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+) versioning=(?<versioning>\\S+)\\s+WEAVER_VERSION=(?<currentValue>\\S+)",
24+
"# renovate: datasource=(?<datasource>\\S+) depName=(?<depName>\\S+) packageName=(?<packageName>\\S+) versioning=(?<versioning>\\S+)\\s+SEMCONV_GENAI_REF=(?<currentValue>\\S+)"
25+
]
26+
}
27+
],
1628
"packageRules": [
1729
{
1830
"groupName": "all patch versions",

‎.github/workflows/generate_workflows_lib/src/generate_workflows_lib/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ def get_test_job_datas(tox_envs: list, operating_systems: list) -> list:
8888
"python_version": aliased_python_version,
8989
"tox_env": tox_env,
9090
"os": operating_system,
91+
"needs_weaver": tox_env.endswith("-conformance"),
9192
}
9293
)
9394

‎.github/workflows/generate_workflows_lib/src/generate_workflows_lib/test.yml.j2‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,47 @@ jobs:
4545
- name: Configure git to support long filenames
4646
run: git config --system core.longpaths true
4747
{%- endif %}
48+
{%- if job_data.needs_weaver %}
49+
50+
- name: Read pinned weaver version
51+
id: versions
52+
{%- if job_data.os == "windows-latest" %}
53+
shell: pwsh
54+
run: |
55+
Select-String -Path versions.env -Pattern '^WEAVER_VERSION=' | ForEach-Object { $_.Line } | Add-Content $env:GITHUB_OUTPUT
56+
{%- else %}
57+
run: |
58+
# shellcheck disable=SC2002
59+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
60+
{%- endif %}
61+
62+
- name: Install weaver ${% raw %}{{ steps.versions.outputs.WEAVER_VERSION }}{% endraw %}
63+
env:
64+
WEAVER_VERSION: ${% raw %}{{ steps.versions.outputs.WEAVER_VERSION }}{% endraw %}
65+
{%- if job_data.os == "windows-latest" %}
66+
shell: pwsh
67+
run: |
68+
$url = "https://github.com/open-telemetry/weaver/releases/download/$env:WEAVER_VERSION/weaver-x86_64-pc-windows-msvc.zip"
69+
Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\weaver.zip"
70+
Expand-Archive -Path "$env:RUNNER_TEMP\weaver.zip" -DestinationPath "$env:RUNNER_TEMP\weaver"
71+
"$env:RUNNER_TEMP\weaver" | Add-Content $env:GITHUB_PATH
72+
& "$env:RUNNER_TEMP\weaver\weaver.exe" --version
73+
{%- elif job_data.os == "macos-latest" %}
74+
run: |
75+
WEAVER_ASSET="weaver-aarch64-apple-darwin"
76+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
77+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
78+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
79+
weaver --version
80+
{%- else %}
81+
run: |
82+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
83+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
84+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
85+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
86+
weaver --version
87+
{%- endif %}
88+
{%- endif %}
4889

4990
- name: Run tests
5091
run: tox -e {{ job_data.tox_env }} -- -ra

‎.github/workflows/test.yml‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,76 @@ jobs:
252252
- name: Run tests
253253
run: tox -e pypy3-test-instrumentation-openai-v2-latest -- -ra
254254

255+
py312-test-instrumentation-openai-v2-conformance_ubuntu-latest:
256+
name: instrumentation-openai-v2-conformance 3.12 Ubuntu
257+
runs-on: ubuntu-latest
258+
timeout-minutes: 30
259+
steps:
260+
- name: Checkout repo @ SHA - ${{ github.sha }}
261+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
262+
263+
- name: Set up Python 3.12
264+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
265+
with:
266+
python-version: "3.12"
267+
268+
- name: Install tox
269+
run: pip install tox-uv
270+
271+
- name: Read pinned weaver version
272+
id: versions
273+
run: |
274+
# shellcheck disable=SC2002
275+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
276+
277+
- name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
278+
env:
279+
WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
280+
run: |
281+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
282+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
283+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
284+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
285+
weaver --version
286+
287+
- name: Run tests
288+
run: tox -e py312-test-instrumentation-openai-v2-conformance -- -ra
289+
290+
py313-test-instrumentation-openai-v2-conformance_ubuntu-latest:
291+
name: instrumentation-openai-v2-conformance 3.13 Ubuntu
292+
runs-on: ubuntu-latest
293+
timeout-minutes: 30
294+
steps:
295+
- name: Checkout repo @ SHA - ${{ github.sha }}
296+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
297+
298+
- name: Set up Python 3.13
299+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
300+
with:
301+
python-version: "3.13"
302+
303+
- name: Install tox
304+
run: pip install tox-uv
305+
306+
- name: Read pinned weaver version
307+
id: versions
308+
run: |
309+
# shellcheck disable=SC2002
310+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
311+
312+
- name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
313+
env:
314+
WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
315+
run: |
316+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
317+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
318+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
319+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
320+
weaver --version
321+
322+
- name: Run tests
323+
run: tox -e py313-test-instrumentation-openai-v2-conformance -- -ra
324+
255325
py310-test-instrumentation-openai_agents-v2-oldest_ubuntu-latest:
256326
name: instrumentation-openai_agents-v2-oldest 3.10 Ubuntu
257327
runs-on: ubuntu-latest
@@ -822,6 +892,76 @@ jobs:
822892
- name: Run tests
823893
run: tox -e py314-test-instrumentation-anthropic-latest -- -ra
824894

895+
py312-test-instrumentation-anthropic-conformance_ubuntu-latest:
896+
name: instrumentation-anthropic-conformance 3.12 Ubuntu
897+
runs-on: ubuntu-latest
898+
timeout-minutes: 30
899+
steps:
900+
- name: Checkout repo @ SHA - ${{ github.sha }}
901+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
902+
903+
- name: Set up Python 3.12
904+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
905+
with:
906+
python-version: "3.12"
907+
908+
- name: Install tox
909+
run: pip install tox-uv
910+
911+
- name: Read pinned weaver version
912+
id: versions
913+
run: |
914+
# shellcheck disable=SC2002
915+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
916+
917+
- name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
918+
env:
919+
WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
920+
run: |
921+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
922+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
923+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
924+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
925+
weaver --version
926+
927+
- name: Run tests
928+
run: tox -e py312-test-instrumentation-anthropic-conformance -- -ra
929+
930+
py313-test-instrumentation-anthropic-conformance_ubuntu-latest:
931+
name: instrumentation-anthropic-conformance 3.13 Ubuntu
932+
runs-on: ubuntu-latest
933+
timeout-minutes: 30
934+
steps:
935+
- name: Checkout repo @ SHA - ${{ github.sha }}
936+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
937+
938+
- name: Set up Python 3.13
939+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
940+
with:
941+
python-version: "3.13"
942+
943+
- name: Install tox
944+
run: pip install tox-uv
945+
946+
- name: Read pinned weaver version
947+
id: versions
948+
run: |
949+
# shellcheck disable=SC2002
950+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
951+
952+
- name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
953+
env:
954+
WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
955+
run: |
956+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
957+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
958+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
959+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
960+
weaver --version
961+
962+
- name: Run tests
963+
run: tox -e py313-test-instrumentation-anthropic-conformance -- -ra
964+
825965
py310-test-instrumentation-claude-agent-sdk-oldest_ubuntu-latest:
826966
name: instrumentation-claude-agent-sdk-oldest 3.10 Ubuntu
827967
runs-on: ubuntu-latest
@@ -974,6 +1114,76 @@ jobs:
9741114
- name: Run tests
9751115
run: tox -e py313-test-instrumentation-claude-agent-sdk-latest -- -ra
9761116

1117+
py312-test-instrumentation-langchain-conformance_ubuntu-latest:
1118+
name: instrumentation-langchain-conformance 3.12 Ubuntu
1119+
runs-on: ubuntu-latest
1120+
timeout-minutes: 30
1121+
steps:
1122+
- name: Checkout repo @ SHA - ${{ github.sha }}
1123+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
1124+
1125+
- name: Set up Python 3.12
1126+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
1127+
with:
1128+
python-version: "3.12"
1129+
1130+
- name: Install tox
1131+
run: pip install tox-uv
1132+
1133+
- name: Read pinned weaver version
1134+
id: versions
1135+
run: |
1136+
# shellcheck disable=SC2002
1137+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
1138+
1139+
- name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
1140+
env:
1141+
WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
1142+
run: |
1143+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
1144+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
1145+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
1146+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
1147+
weaver --version
1148+
1149+
- name: Run tests
1150+
run: tox -e py312-test-instrumentation-langchain-conformance -- -ra
1151+
1152+
py313-test-instrumentation-langchain-conformance_ubuntu-latest:
1153+
name: instrumentation-langchain-conformance 3.13 Ubuntu
1154+
runs-on: ubuntu-latest
1155+
timeout-minutes: 30
1156+
steps:
1157+
- name: Checkout repo @ SHA - ${{ github.sha }}
1158+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
1159+
1160+
- name: Set up Python 3.13
1161+
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
1162+
with:
1163+
python-version: "3.13"
1164+
1165+
- name: Install tox
1166+
run: pip install tox-uv
1167+
1168+
- name: Read pinned weaver version
1169+
id: versions
1170+
run: |
1171+
# shellcheck disable=SC2002
1172+
cat versions.env | grep -E '^WEAVER_VERSION=' >> "$GITHUB_OUTPUT"
1173+
1174+
- name: Install weaver ${{ steps.versions.outputs.WEAVER_VERSION }}
1175+
env:
1176+
WEAVER_VERSION: ${{ steps.versions.outputs.WEAVER_VERSION }}
1177+
run: |
1178+
WEAVER_ASSET="weaver-x86_64-unknown-linux-gnu"
1179+
WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${WEAVER_VERSION}/${WEAVER_ASSET}.tar.xz"
1180+
curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp "${WEAVER_ASSET}/weaver"
1181+
sudo mv "/tmp/${WEAVER_ASSET}/weaver" /usr/local/bin/weaver
1182+
weaver --version
1183+
1184+
- name: Run tests
1185+
run: tox -e py313-test-instrumentation-langchain-conformance -- -ra
1186+
9771187
py310-test-util-genai_ubuntu-latest:
9781188
name: util-genai 3.10 Ubuntu
9791189
runs-on: ubuntu-latest

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,10 @@ target
6666
opentelemetry-admin-jobs.txt
6767

6868
.claude/settings.local.json
69+
70+
# Generated by test_util_genai._setup_weaver
71+
policies/_schemas.rego
72+
.build/
73+
74+
# Conformance test weaver live-check reports
75+
weaver_reports/

‎AGENTS.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,21 @@ Apply to packages under `instrumentation/`.
109109
`from opentelemetry.test_util_genai.vcr import fixture_vcr, scrub_response_headers`) rather
110110
than re-implementing provider/exporter/VCR plumbing.
111111

112+
### Conformance tests
113+
114+
Packages with substantive instrumentation ship `tests/conformance/<scenario>.py`
115+
scenarios and a `tests/test_conformance.py` that validates emitted telemetry
116+
against the [GenAI semantic conventions](https://github.com/open-telemetry/semantic-conventions-genai)
117+
via Weaver live-check. Each scenario module defines a subclass of
118+
`opentelemetry.test_util_genai.conformance.Scenario` that sets
119+
`expected_spans`, `expected_metrics`, and implements
120+
`run(*, tracer_provider, meter_provider, logger_provider, vcr)`.
121+
122+
`tests/test_conformance.py` must set `pytestmark = pytest.mark.conformance` at
123+
module level.
124+
125+
Run via `tox -e py312-test-instrumentation-<pkg>-conformance`.
126+
112127
The parallel PR-review rules live in
113128
[`.github/instructions/instrumentation.instructions.md`](.github/instructions/instrumentation.instructions.md)
114129
and should be kept in sync with this section.

‎CONTRIBUTING.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,27 @@ Run type checking across the workspace:
6666
uv run tox -e typecheck
6767
```
6868

69+
#### Managing cassettes (test recordings)
70+
71+
GenAI tests replay recorded HTTP interactions (cassettes) stored under each
72+
package's `tests/cassettes/`.
73+
74+
- **Run**: nothing extra — cassettes replay automatically when present. Tests
75+
that need a cassette skip if it is missing and no real API key is set.
76+
- **Record**: delete the target `tests/cassettes/<test_name>.yaml`, export a
77+
real provider API key (e.g. `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`), and
78+
rerun the test. `pytest-vcr` writes the cassette on the live call.
79+
- **Sanitize**: every package's `vcr_config()` in `tests/conftest.py` must
80+
scrub auth via `filter_headers` and strip identifying response headers via
81+
`scrub_response_headers(...)` from `opentelemetry.test_util_genai.vcr`.
82+
Diff each new cassette before committing — leaked API keys, org ids, or
83+
`Set-Cookie` values block the PR.
84+
- **AI-generated cassettes**: if you lack provider access, you may
85+
synthesize a cassette from the provider's API reference via AI. Make sure
86+
to mention it in the PR and open a follow-up issue to re-record it in CI
87+
against the real provider.
88+
- **CI**: replay-only; recording in CI is a future improvement.
89+
6990
### 4. Update the changelog
7091

7192
This repo uses [towncrier](https://towncrier.readthedocs.io/) to manage
@@ -136,3 +157,4 @@ For more information about the maintainer role, see the [community repository](h
136157
- [Leighton Chen](https://github.com/lzchen), Microsoft
137158

138159
For more information about the approver role, see the [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#approver).
160+

‎dev-requirements-conformance.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Conformance tests use `opentelemetry.test.weaver_live_check`, not yet on
2+
# PyPI. Pin the whole stack from git so versions stay coherent. Mirrors
3+
# pyproject.toml's [tool.uv.sources]. Drop once the file ships to PyPI.
4+
#
5+
# TODO: switch to PyPI versions once the test utils are released
6+
7+
opentelemetry-api @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-api
8+
opentelemetry-sdk @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-sdk
9+
opentelemetry-semantic-conventions @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-semantic-conventions
10+
opentelemetry-test-utils @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=tests/opentelemetry-test-utils
11+
opentelemetry-instrumentation @ git+https://github.com/open-telemetry/opentelemetry-python-contrib@d2f396de68a969dfb74b8afc247e1d0dc6739b67#subdirectory=opentelemetry-instrumentation
12+
opentelemetry-exporter-otlp-proto-grpc @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=exporter/opentelemetry-exporter-otlp-proto-grpc
13+
opentelemetry-exporter-otlp-proto-common @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=exporter/opentelemetry-exporter-otlp-proto-common
14+
opentelemetry-proto @ git+https://github.com/open-telemetry/opentelemetry-python@1731583b4e7bc6ec6a33345aa19706fc83acc8d5#subdirectory=opentelemetry-proto

0 commit comments

Comments
 (0)