diff --git a/.gitignore b/.gitignore index cf82b8a..628d2a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ docs/external-reviews/ +__pycache__/ +.pytest_cache/ +.hypothesis/ diff --git a/golden-test-set/DESIGN.md b/golden-test-set/DESIGN.md new file mode 100644 index 0000000..4dd04be --- /dev/null +++ b/golden-test-set/DESIGN.md @@ -0,0 +1,208 @@ +# Golden Test Set — Design Document + +**Purpose:** Empirically calibrate Quorum's detection accuracy against human-annotated ground truth. +**Target:** 40 artifacts with machine-readable annotations, automated scoring, published precision/recall. +**Status:** Schema and taxonomy locked (2026-03-11). Artifact creation pending. + +--- + +## Directory Structure + +``` +golden-test-set/ +├── DESIGN.md # This file +├── schema.yaml # Annotation schema definition +├── artifacts/ # The files Quorum evaluates +│ ├── python/ +│ ├── config/ +│ ├── docs/ +│ ├── shell/ +│ └── cross-artifact/ +├── annotations/ # Ground truth sidecars (one per artifact) +│ └── .annotations.yaml +├── results/ # Quorum run outputs (gitignored except baseline) +│ └── baseline-YYYYMMDD/ +└── scripts/ + └── score.py # Automated scoring framework +``` + +--- + +## Annotation Schema (v1.0) + +Each artifact has a YAML sidecar in `annotations/` following `schema.yaml`. + +### ID Convention + +- **GT-###** = Golden Test ground truth finding (expected finding in an artifact) +- See `docs/ID_CONVENTIONS.md` (planned) for the full prefix registry across Quorum + +### Schema Fields + +```yaml +schema_version: "1.0" +artifact: "artifacts/python/vulnerable-api.py" # relative path from golden-test-set/ +artifact_sha256: "abc123..." # integrity — annotation is for THIS version + +expected_verdict: REVISE # PASS | PASS_WITH_NOTES | REVISE | REJECT + +findings: + - id: GT-001 + description: "SQL injection via unsanitized user input in query builder" + location: "line 42-48" # human-readable, scored with ±5 line tolerance + severity: CRITICAL # expected severity + category: security # security | correctness | completeness | code_hygiene | cross_consistency + critic: security # which critic SHOULD catch this + rubric_criterion: null # optional — specific criterion ID + notes: "String concatenation in raw SQL query" + +false_positive_traps: + - description: "Use of eval() on line 30 — input is a hardcoded constant, not user-supplied" + location: "line 30" + notes: "If Quorum flags this, it's a false positive" + +metadata: + source: synthetic # synthetic | natural | modified-natural + domain: python-code # python-code | yaml-config | markdown-doc | shell-script | cross-artifact + complexity: medium # low | medium | high + rubric: python-code # rubric to use for evaluation + depth: standard # recommended depth: quick | standard | thorough + author: akkari + created: "2026-03-11" +``` + +--- + +## Matching & Scoring Rules + +### Detection Matching (Moderate Granularity) + +A Quorum finding matches a ground truth finding when ALL of: +1. **Correct critic** produced the finding (or a superset critic that covers it) +2. **Location overlap** — within ±5 lines of the annotated location, OR description fuzzy match ≥0.6 if no line number in ground truth +3. **Same defect class** — category matches (security↔security, not security↔completeness) + +A match is a match regardless of severity — severity accuracy is scored separately. + +### Metrics (computed per-run) + +| Metric | Definition | +|--------|-----------| +| **Detection Precision** | matched_findings / total_quorum_findings — "of what Quorum reported, how much was real?" | +| **Detection Recall** | matched_findings / total_ground_truth_findings — "of what's actually wrong, how much did Quorum find?" | +| **F1** | Harmonic mean of precision and recall | +| **Severity Accuracy** | Of matched findings, % where Quorum severity == ground truth severity | +| **False Positive Rate** | Quorum findings on clean (PASS) artifacts / total findings on clean artifacts — should be 0 ideally | +| **Verdict Accuracy** | % of artifacts where Quorum verdict matches expected verdict | + +All metrics computed: +- **Aggregate** (whole test set) +- **Per critic** (correctness, completeness, security, code_hygiene, cross_consistency) +- **Per severity tier** (CRITICAL, HIGH, MEDIUM, LOW) +- **Per complexity level** (low, medium, high) +- **Per file type** (python, config, docs, shell, cross-artifact) + +### Severity Scoring (Independent) + +Detection and severity are scored separately. If ground truth says HIGH and Quorum says CRITICAL: +- Detection: ✅ match (the issue was found) +- Severity: ❌ mismatch (over-classified by one tier) + +Severity distance: |ground_truth_tier - quorum_tier| where CRITICAL=4, HIGH=3, MEDIUM=2, LOW=1, INFO=0. + +--- + +## Artifact Taxonomy + +### By File Type + +| Type | Count | Directory | Rationale | +|------|-------|-----------|-----------| +| Python code | 15 | `artifacts/python/` | Core use case, most critics apply | +| YAML/JSON configs | 8 | `artifacts/config/` | Agent configs, pipeline definitions | +| Markdown docs | 10 | `artifacts/docs/` | Research, specs, READMEs | +| Shell scripts | 3 | `artifacts/shell/` | Cross-language, security-heavy | +| Cross-artifact pairs | 4 | `artifacts/cross-artifact/` | Spec↔implementation consistency | +| **Total** | **40** | | | + +### By Expected Verdict + +| Verdict | Count | Purpose | +|---------|-------|---------| +| PASS | 8 | False positive gauntlet — clean artifacts | +| PASS_WITH_NOTES | 7 | Minor issues, severity calibration | +| REVISE | 15 | Bulk — HIGH findings, real rework needed | +| REJECT | 10 | CRITICAL issues — must not be missed | + +### By Defect Class + +| Class | Primary Critic | Count | Examples | +|-------|---------------|-------|----------| +| Security | security | 10 | SQL injection, hardcoded creds, path traversal, SSRF, XSS | +| Correctness | correctness | 8 | Wrong citations, logic errors, contradictory claims | +| Completeness | completeness | 7 | Missing error handling, undocumented params, coverage gaps | +| Code hygiene | code_hygiene | 5 | Dead code, complexity, naming, maintainability | +| Cross-artifact | cross_consistency | 4 | Spec says X, implementation does Y | +| Clean | all | 8 | Well-written code/docs — false positive gauntlet | +| Multi-defect | multiple | ~6 | Overlap — some artifacts hit 2+ critics | + +### By Complexity + +| Level | Count | Tests | +|-------|-------|-------| +| Low | 12 | Obvious single defects, short files — baseline detection | +| Medium | 18 | Realistic code, 2-3 issues — typical use case | +| High | 10 | Subtle bugs, false positive traps — Quorum's ceiling | + +### By Source + +| Source | Count | Rationale | +|--------|-------|-----------| +| Synthetic | 25 | Controlled ground truth, precise defect placement | +| Modified-natural | 10 | Real code with annotated real bugs — credibility | +| Natural | 5 | From four-model review convergence — meta-validation | + +--- + +## Execution Plan + +### Step 1: Schema ✅ +This document + `schema.yaml`. + +### Step 2: Taxonomy ✅ +Distribution tables above. + +### Step 3: Scoring Framework +Build `scripts/score.py`: +- Reads Quorum run output + annotation sidecars +- Computes all metrics from the table above +- Outputs JSON summary + Markdown report +- Exits non-zero if precision or recall below configurable threshold + +### Step 4: Build Artifacts +Claude Code batch task — create 40 artifacts + 40 annotation sidecars. +Artifact creation guidelines: +- Synthetic: plant specific defects at known locations, document in annotation +- Modified-natural: take real code, annotate existing bugs, add SHA-256 +- Natural: extract from Quorum's own validated findings (four-model convergence) +- Every artifact must have a corresponding `.annotations.yaml` +- Clean (PASS) artifacts must be genuinely clean — don't just remove obvious bugs + +### Step 5: Baseline Calibration +Run `quorum run` against full test set at standard depth. +Publish: precision, recall, F1, severity accuracy, false positive rate. +This becomes the number on the README. + +--- + +## Success Criteria + +| Metric | Target | Rationale | +|--------|--------|-----------| +| Detection Recall | ≥ 0.80 | Must find 80%+ of real issues | +| Detection Precision | ≥ 0.70 | Acceptable false positive rate | +| F1 | ≥ 0.75 | Balanced performance | +| False Positive Rate (clean) | ≤ 0.15 | Max 15% noise on clean artifacts | +| Verdict Accuracy | ≥ 0.75 | Correct pass/fail 75%+ of the time | + +These are initial targets. Calibration may reveal they need adjustment — that's the point of measuring. diff --git a/golden-test-set/annotations/gt-cfg-001-exposed-secrets.annotations.yaml b/golden-test-set/annotations/gt-cfg-001-exposed-secrets.annotations.yaml new file mode 100644 index 0000000..8015f63 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-001-exposed-secrets.annotations.yaml @@ -0,0 +1,46 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-001-exposed-secrets.yaml" +artifact_sha256: "082e6f8bb598b7803af5fb76447b1d35e6fa231ab29698a2ecc88451bdee1676" + +expected_verdict: REJECT + +findings: + - id: GT-024 + description: "Hardcoded API key in environment variable definition" + location: "stages[0].environment.NPM_TOKEN (line 18), stages[0].environment.SENTRY_AUTH_TOKEN (line 19), stages[1].environment.STRIPE_SECRET_KEY (line 29), stages[3].environment.AWS_ACCESS_KEY_ID (line 42), stages[3].environment.AWS_SECRET_ACCESS_KEY (line 43)" + severity: CRITICAL + category: security + critic: security + notes: > + Multiple live API keys are hardcoded inline in environment variable blocks. + Affected keys: NPM_TOKEN (npm publish token), SENTRY_AUTH_TOKEN (Sentry DSN), + STRIPE_SECRET_KEY (Stripe key with production-like format), + AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY (IAM credentials). Any of these + leaking via logs, artifacts, or VCS history grants direct access to + external services. Should be stored in vault references or CI secret variables. + + - id: GT-025 + description: "Database password in plaintext connection string" + location: "stages[1].environment.DATABASE_URL (line 24), stages[1].environment.REDIS_URL (line 25), stages[1].environment.JWT_SECRET (line 26), stages[3].environment.DOCKER_PASSWORD (line 41), stages[3].environment.KUBECONFIG_DATA (line 46)" + severity: CRITICAL + category: security + critic: security + notes: > + DATABASE_URL embeds credentials as postgres://user:password@host/db — the + password "Tr0ub4dor&3" is fully visible. REDIS_URL similarly embeds + "SuperSecretRedisPassword123!". JWT_SECRET is the HMAC signing key for + all issued tokens. DOCKER_PASSWORD is a registry credential. KUBECONFIG_DATA + is a base64-encoded kubeconfig containing cluster credentials. All should + be injected at runtime via secret manager references, not hardcoded. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: yaml-config + complexity: low + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-002-insecure-permissions.annotations.yaml b/golden-test-set/annotations/gt-cfg-002-insecure-permissions.annotations.yaml new file mode 100644 index 0000000..9e293b6 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-002-insecure-permissions.annotations.yaml @@ -0,0 +1,63 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-002-insecure-permissions.yaml" +artifact_sha256: "3df39d2c24beb980ffdc8f39dfe22738ce43c4de36ab9e0a33d6b9ef39df3f80" + +expected_verdict: REJECT + +findings: + - id: GT-026 + description: "Container running as root with privileged: true and allowPrivilegeEscalation: true" + location: "spec.template.spec.containers[0].securityContext (lines 38-43)" + severity: CRITICAL + category: security + critic: security + notes: > + The combination of privileged: true, runAsUser: 0, runAsGroup: 0, and + allowPrivilegeEscalation: true gives the container full root access to the + host kernel. A container escape in this configuration grants an attacker + unrestricted access to the underlying node and, by extension, the entire + cluster. Privileged mode should never be used in production workloads; + if specific capabilities are needed they should be granted individually via + securityContext.capabilities.add with the minimum required set. + + - id: GT-027 + description: "hostNetwork enabled without documented security justification or compensating controls" + location: "spec.template.spec.hostNetwork (line 29)" + severity: HIGH + category: security + critic: security + notes: > + hostNetwork: true causes all pods to share the node's network namespace, + bypassing Kubernetes network policies and exposing every port the pod + listens on directly on the host interface. The inline TODO comment + (INFRA-2089) acknowledges this is a known issue but no compensating + controls (e.g., firewall rules, network policy, or timeboxed exception + approval) are present. The justification ("legacy payment processor SDK + binds to fixed port 9742") does not meet the bar for a production + security exception without compensating controls documented and approved. + + - id: GT-028 + description: "No resource limits defined — missing resources.limits block" + location: "spec.template.spec.containers[0] (lines 36-37)" + severity: MEDIUM + category: completeness + critic: completeness + notes: > + The container has no resources.limits block. Without CPU and memory limits, + a runaway container can starve co-located workloads or OOM-kill the node. + The inline comment ("scaling handled by HPA") does not replace per-pod + limits — HPA governs replica count, not per-pod resource consumption. + Kubernetes best practice requires both requests and limits. Tracked in + INFRA-2090 but not yet resolved. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: yaml-config + complexity: medium + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-003-stale-references.annotations.yaml b/golden-test-set/annotations/gt-cfg-003-stale-references.annotations.yaml new file mode 100644 index 0000000..bb74f65 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-003-stale-references.annotations.yaml @@ -0,0 +1,52 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-003-stale-references.yaml" +artifact_sha256: "1eec85218e63b5d473ca93f6dd8c6a4b29398a12130f5a0947d6dea0ff25671f" + +expected_verdict: REVISE + +findings: + - id: GT-029 + description: "References deprecated API version v1beta1 in three distinct locations" + location: > + source.api_version (line 22), + monitoring.alerts.webhook (line 66), + feature_flags.endpoint (line 73) + severity: HIGH + category: correctness + critic: correctness + notes: > + The string "v1beta1" appears in three locations: (1) The Kafka consumer's + api_version field pinning the client to a deprecated Kafka broker API surface. + (2) The Alertmanager webhook URL using the deprecated /api/v1beta1/alerts + endpoint — the inline comment explicitly states this route was renamed to + "data-eng-oncall" and now 404s silently, meaning alerts are not firing. + (3) The feature flag endpoint URL using the deprecated v1beta1 API — the + comment states to use the v2 flag service instead. All three are active + correctness defects, not just documentation debt. + + - id: GT-030 + description: "References removed feature flag 'experimental_mode' from pipeline version that no longer supports it" + location: "transforms[2].experimental_mode (line 40)" + severity: HIGH + category: correctness + critic: correctness + notes: > + The filter transform sets experimental_mode: true. The inline comment + acknowledges this flag "was added in v0.9, removed in v1.2." The + justification offered ("rollback to v0.9 is theoretically possible") is + not an operational rationale — this field is silently ignored by any + version >= v1.2, meaning the ML-based bot detection this flag was intended + to enable is not actually running. The config is misleading about what + bot filtering is active in production. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: yaml-config + complexity: medium + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-004-missing-validation.annotations.yaml b/golden-test-set/annotations/gt-cfg-004-missing-validation.annotations.yaml new file mode 100644 index 0000000..e98c5d6 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-004-missing-validation.annotations.yaml @@ -0,0 +1,63 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-004-missing-validation.json" +artifact_sha256: "507c2e164a0b2df6ecf858c0129244d67f27f221895bb4b7e4109ab1d3cc1723" + +expected_verdict: REVISE + +findings: + - id: GT-031 + description: "No rate limiting configured anywhere in the gateway — neither global nor per-route" + location: "middleware array (lines 66-88); routes array (lines 15-77)" + severity: HIGH + category: completeness + critic: completeness + notes: > + The middleware array contains cors, compression, and request_id but no + rate limiting middleware. No route has a rate_limit block. Without rate + limiting, the gateway is vulnerable to denial-of-service via request + flooding and provides no protection against credential stuffing or + brute-force attacks on the auth endpoints. The upstream_defaults.retry + block handles upstream resilience but does not substitute for ingress + rate control. + + - id: GT-032 + description: "Missing input size validation — no max request body size or payload validation configured" + location: "gateway block (lines 3-13); routes array (lines 15-77)" + severity: HIGH + category: completeness + critic: completeness + notes: > + There is no max_body_size, max_request_size, or equivalent field at the + gateway or route level. Without a size cap, clients can send arbitrarily + large request bodies, enabling memory exhaustion attacks against upstream + services. The webhooks route (lines 68-77) is particularly exposed since + it has auth.required: false and relies entirely on application-layer HMAC + validation which happens after the body is already buffered. + + - id: GT-033 + description: "No authentication configured for the /admin route" + location: "routes[3] (lines 56-62)" + severity: HIGH + category: security + critic: security + notes: > + The route with id "route-admin" exposes /admin with methods GET, POST, + PUT, DELETE, PATCH but has no auth block. The comment "assumed safe + behind VPN" is not a configuration-level control — there is no + network policy, IP allowlist, or auth middleware enforcement in the + config itself. If the VPN assumption fails (misconfigured routing, + SSRF, compromised host), the admin panel is fully open. All privileged + routes must have explicit auth controls regardless of assumed network + perimeter. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: yaml-config + complexity: low + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-005-contradictory.annotations.yaml b/golden-test-set/annotations/gt-cfg-005-contradictory.annotations.yaml new file mode 100644 index 0000000..4bf0c2f --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-005-contradictory.annotations.yaml @@ -0,0 +1,64 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-005-contradictory.yaml" +artifact_sha256: "94b39680368ed1fe16a1155d6ddab602e5aab149a127742b0f26379213e63093" + +expected_verdict: REVISE + +findings: + - id: GT-034 + description: "Total retry budget (retry_count * per_retry_timeout_seconds = 60s) exceeds max_timeout_seconds (45s)" + location: > + retry.retry_count (line 47), + retry.per_retry_timeout_seconds (line 48), + server.max_timeout_seconds (line 17) + severity: HIGH + category: correctness + critic: correctness + notes: > + retry_count is 5 and per_retry_timeout_seconds is 12, giving a theoretical + retry budget of 60 seconds. The global max_timeout_seconds is 45 seconds. + The config comment on line 50 explicitly acknowledges this discrepancy + ("NOTE: total retry budget = 5 * 12 = 60s, which EXCEEDS max_timeout_seconds + (45s)"). The practical effect is that the 4th or 5th retry attempt will + never execute — the circuit hits the global deadline and returns + deadline-exceeded to the caller. This silently degrades the retry policy + without any operational error, making the declared retry_count misleading. + The documented intent (OPS-3315) does not count as a fix. Either + max_timeout_seconds must be raised or retry_count * per_retry_timeout must + be reduced to fit within it. + + - id: GT-035 + description: "Log level set to DEBUG in production profile — will capture PII and credentials in logs" + location: "observability.log_level (line 69)" + severity: MEDIUM + category: code_hygiene + critic: code_hygiene + notes: > + service.profile is "production" (line 8) but observability.log_level is + "debug". The inline comment on line 68 confirms this was left from debugging + OPS-3298 and was not reverted before the deploy. DEBUG logging in this + service captures full request/response bodies, which include payment data + and authentication tokens. This is a code hygiene issue (forgotten debug + flag) with a secondary PII/security consequence. Should be "info" in + production. + +false_positive_traps: + - description: "max_connections: 0 may look like 'zero connections allowed' or a misconfiguration" + location: "server.max_connections (line 21)" + notes: > + max_connections: 0 is a well-established convention in Go HTTP servers + and many other frameworks meaning "unlimited" — there is no connection + cap. The inline comment in the config explicitly states: "0 = unlimited; + connection count governed by k8s resource limits." A critic flagging + this as a zero-connections bug or misconfiguration is a false positive. + Quorum should recognize the convention and the clarifying comment. + +metadata: + source: synthetic + domain: yaml-config + complexity: high + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-006-minor-issues.annotations.yaml b/golden-test-set/annotations/gt-cfg-006-minor-issues.annotations.yaml new file mode 100644 index 0000000..7f78a58 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-006-minor-issues.annotations.yaml @@ -0,0 +1,39 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-006-minor-issues.yaml" +artifact_sha256: "3b7a4e852a71508af4caa551993253690154cb5b1d4dd3da43e2c0153a76e06a" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-036 + description: "No inline comments explaining non-obvious configuration values" + location: > + evaluation.timeout_ms (line 44), + evaluation.max_retries (line 45), + defaults.event_buffer_size (line 53), + defaults.shutdown_grace_period_seconds (line 54) + severity: LOW + category: completeness + critic: completeness + notes: > + Several values whose choice is non-obvious lack explanatory comments. + evaluation.timeout_ms: 200 — why 200ms? Is this derived from a p99 + latency budget? evaluation.max_retries: 2 — what happens on the third + failure: fail open or closed? defaults.event_buffer_size: 512 — tuning + rationale and what happens when the buffer fills (dropped events vs. + backpressure). defaults.shutdown_grace_period_seconds: 15 — should + match or exceed the Kubernetes terminationGracePeriodSeconds; the + relationship is undocumented. These are LOW severity documentation + gaps that do not impair correctness but reduce operability. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: yaml-config + complexity: low + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-007-clean-k8s.annotations.yaml b/golden-test-set/annotations/gt-cfg-007-clean-k8s.annotations.yaml new file mode 100644 index 0000000..da5ace7 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-007-clean-k8s.annotations.yaml @@ -0,0 +1,37 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-007-clean-k8s.yaml" +artifact_sha256: "9c752572176b76f77e03f91e86aea87c355b51fdb8ca57d07a1bc0aae8491f02" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-065 + description: "No NetworkPolicy defined — production deployment allows unrestricted pod-to-pod communication" + location: "document-level" + severity: LOW + category: completeness + critic: completeness + notes: "A production K8s deployment should include a NetworkPolicy to restrict traffic. Not a security vulnerability per se, but a defense-in-depth gap." + +false_positive_traps: + - description: "securityContext.readOnlyRootFilesystem: false may appear to be a security misconfiguration" + location: "spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem (line 56)" + notes: > + readOnlyRootFilesystem is intentionally false and explicitly justified in + the config comments (lines 52-59): the notification-service renders Jinja2 + email templates to /tmp/rendered/ and requires filesystem write access. + The accepted risk is documented and traceable to security review ticket + SEC-0892. All other securityContext fields are hardened (privileged: false, + allowPrivilegeEscalation: false, capabilities.drop: ALL, runAsNonRoot: true, + seccompProfile: RuntimeDefault). Flagging readOnlyRootFilesystem: false as + a defect without reading the surrounding justification is a false positive. + +metadata: + source: modified-natural + domain: yaml-config + complexity: medium + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-cfg-008-clean-pipeline.annotations.yaml b/golden-test-set/annotations/gt-cfg-008-clean-pipeline.annotations.yaml new file mode 100644 index 0000000..95a1c23 --- /dev/null +++ b/golden-test-set/annotations/gt-cfg-008-clean-pipeline.annotations.yaml @@ -0,0 +1,39 @@ +schema_version: "1.0" + +artifact: "artifacts/config/gt-cfg-008-clean-pipeline.json" +artifact_sha256: "3048ea6bb05fc396ea7b30a2bc0bbff3541a8bac394e612757c369ecea3c2f64" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: "Field named 'secret_path' in the vault block may trigger secret-detection heuristics" + location: "vault.secret_path (line 22)" + notes: > + vault.secret_path contains the value "secret/data/ci/identity-service" — + this is a Vault KV v2 mount path, not an inline secret value. The field + name "secret_path" is standard Vault nomenclature. A heuristic that flags + any field containing the word "secret" in its key name, or any string + beginning with "secret/", as a credential leak would produce a false + positive here. The vault block is a proper Vault reference configuration + (address, auth_method, role, namespace, secret_path) with no inline + credential values anywhere in the file. + - description: "GITHUB_TOKEN and PYPI_API_TOKEN env var names in the release stage may look like exposed secrets" + location: "stages[4].env.GITHUB_TOKEN, stages[4].env.PYPI_API_TOKEN (lines 89-90)" + notes: > + Both variables have values structured as {"vault": "secret/data/..."} — + they are Vault injection references, not inline token values. A critic + that pattern-matches on env var names like GITHUB_TOKEN or PYPI_API_TOKEN + without examining the value structure would produce a false positive. + The correct detection requires checking whether the value is a string + literal vs. a Vault reference object. + +metadata: + source: synthetic + domain: yaml-config + complexity: medium + rubric: agent-config + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-001-wrong-citations.annotations.yaml b/golden-test-set/annotations/gt-doc-001-wrong-citations.annotations.yaml new file mode 100644 index 0000000..0133a23 --- /dev/null +++ b/golden-test-set/annotations/gt-doc-001-wrong-citations.annotations.yaml @@ -0,0 +1,60 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-001-wrong-citations.md" +artifact_sha256: "7d9bc7fa272008c8429e638f0efec91f2257b401b4a6d51625c39b8aa5c3f3be" + +expected_verdict: REJECT + +findings: + - id: GT-037 + description: > + Citation [3] (Gao et al., 2023) is attributed the wrong finding. The document + claims [3] discovered that HyDE was introduced by Gao et al. directly, then + contradicts itself by stating the actual HyDE concept originated with "Chen & + Zhao (2022)" — a paper not in the citation list. The document makes the explicit + correction mid-prose ("the correct attribution is Gao et al. [3]") but the + embedded acknowledgment that Chen & Zhao introduced the concept is left + unresolved. This creates an authorship attribution error: the core HyDE concept + is credited to Gao et al. [3] in the Key Findings heading but the body text + acknowledges it belongs to a different, uncited paper. + location: "section 'Key Findings', subsection 3.1" + severity: CRITICAL + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The document plants a self-correcting statement that confuses rather than + resolves the attribution. The final attributed claim in the section header + (Gao et al. [3] → HyDE) is wrong; HyDE was introduced by different authors + and Gao et al. applied it. This is the intentional ground truth finding. + + - id: GT-038 + description: > + Citation [7] (Nakamura & Patel 2023) is listed in the bibliography as an + unpublished "Preprint, Stanford NLP Group, March 2023" but the document body + acknowledges this is incorrect: the correct citation is the published EMNLP + 2023 proceedings version. The artifact retains the wrong citation in the + Appendix A citation map (listed as "Preprint") while the body text corrects + it. The annotation in the citation map therefore references a paper that does + not exist in the form listed — the Stanford preprint was not independently + published and is only retrievable as the EMNLP 2023 paper. + location: "section 'Methodology', Appendix A citation map, row [7]" + severity: CRITICAL + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The citation in Appendix A is the canonical bibliography entry used by readers. + It points to a non-existent standalone preprint. The body correction in §3.2 + is buried prose and easily missed. A researcher following the bibliography + citation will not find the source. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: markdown-doc + complexity: medium + rubric: research-synthesis + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-002-contradictory-claims.annotations.yaml b/golden-test-set/annotations/gt-doc-002-contradictory-claims.annotations.yaml new file mode 100644 index 0000000..efa9f71 --- /dev/null +++ b/golden-test-set/annotations/gt-doc-002-contradictory-claims.annotations.yaml @@ -0,0 +1,68 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-002-contradictory-claims.md" +artifact_sha256: "436820cc770dc18a0c2f9557bba54618667a1976346a8d555e286a54d1c993e6" + +expected_verdict: REJECT + +findings: + - id: GT-039 + description: > + Section 3 (Performance Requirements) defines the gateway's p99 processing + overhead budget as 100ms ("must not exceed 100ms at p99"). Section 5 (Rate + Limiting, §5.4) redefines the same budget as 250ms: "Rate-limit enforcement + must not increase p99 gateway latency by more than 150ms above baseline under + normal load conditions. The total gateway processing budget, including rate + limiting, is therefore 250ms at p99." These two values (100ms and 250ms) are + presented as hard requirements for the same system under the same conditions + and are directly contradictory. A system cannot simultaneously conform to + both specifications. + location: "section 3.1 (table p99 Hard Limit: 150ms, processing overhead: 100ms) and section 5.4" + severity: CRITICAL + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The contradiction is clear and irreconcilable: §3.1 sets the gateway processing + overhead at ≤100ms p99, while §5.4 explicitly states the total budget including + rate limiting is 250ms. These numbers are for the same gateway, same conditions. + §5.4 even says "this budget supersedes the more conservative estimate in §3.1" + which makes the contradiction explicit in the document itself. + + - id: GT-040 + description: > + Section 4.2 (Service Topology) describes the gateway as routing traffic to + exactly 3 upstream services: Search Service, Indexing Service, and Auth Service. + The routing table in §4.2 lists exactly 3 path prefixes mapping to 3 services. + However, the Architecture Overview section (§4.1) states "Each node runs two + processes" and the surrounding text refers to "upstream services" in the plural + without specifying count — but the component topology description implies 3. + More critically, the Configuration Reference (§9) lists only 3 routing rules + in the example YAML (search, index, auth) while the surrounding prose in the + same document refers to "downstream service protection" in §1 generically. + The document's own Table of Contents references a fourth component role + (the "Response Cache" in the data flow diagram §4.3) as a service-level + interaction, creating an implicit fourth flow path not counted in the upstream + services table. + location: "section 4 (Architecture), data flow diagram in 4.3 vs service topology table in 4.2" + severity: HIGH + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The data flow diagram shows 4 distinct components downstream of the router + (Search Service, Indexing Service, Auth Service, and Response Cache listed + as a separate branch), while the Service Topology table in §4.2 lists only + 3 upstream services. The response cache is described as "in-process, per-node" + in §4.3 but appears as a separate downstream node in the data flow ASCII + diagram, making the diagram inconsistent with the text description. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: markdown-doc + complexity: high + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-003-stale-references.annotations.yaml b/golden-test-set/annotations/gt-doc-003-stale-references.annotations.yaml new file mode 100644 index 0000000..f96b3d2 --- /dev/null +++ b/golden-test-set/annotations/gt-doc-003-stale-references.annotations.yaml @@ -0,0 +1,61 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-003-stale-references.md" +artifact_sha256: "9ab4a250d1810bc61f8f196354cbf21558c8f4da20661e068bedd082728c4e8f" + +expected_verdict: REVISE + +findings: + - id: GT-041 + description: > + The README specifies "Python 3.8 or higher" as the system requirement in the + Prerequisites section. Python 3.8 reached end-of-life in October 2024 and no + longer receives security patches. The document itself includes a footnote + acknowledging this ("Python 3.8 reached end-of-life in October 2024. While + Helix runs on Python 3.8, we recommend upgrading...") but the main requirement + still states 3.8, and the installation commands explicitly use `python3.8`. + As of 2026, the current supported Python versions are 3.11, 3.12, and 3.13. + Stating an EOL version as the primary requirement is an incorrect and + potentially dangerous specification. + location: "section 'Prerequisites', Software Requirements list, and Installation Option 1 commands" + severity: HIGH + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The requirement is factually wrong for any deployment after October 2024. + The inline note acknowledges EOL but does not update the requirement — + this is a modified-natural pattern where a real README was not updated + when the Python version requirement changed. The install command + `python3.8 -m venv` is specifically broken. + + - id: GT-042 + description: > + The API Reference section documents management endpoints using the `/api/v1/` + path prefix (e.g., `POST /api/v1/pipeline/pause`, `GET /api/v1/dlq/messages`). + The same section contains a note stating: "The /api/v1/ prefix is deprecated + and will be removed in Helix 3.0. Please update any integrations to use + /api/v2/pipeline/, /api/v2/dlq/, etc." The document presents deprecated + endpoints as the primary documented interface while acknowledging they are + deprecated. A reader following this documentation for integration will + implement against an endpoint path scheduled for removal. + location: "section 'API Reference', Pipeline Control and DLQ Management subsections, and the note below them" + severity: HIGH + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The documented API paths are deprecated. The note exists but is buried + after the endpoint definitions. The section header and code examples all + use /api/v1/, which is the path that will be removed. New integrators + reading this section will build against the wrong paths. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: markdown-doc + complexity: medium + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-004-missing-sections.annotations.yaml b/golden-test-set/annotations/gt-doc-004-missing-sections.annotations.yaml new file mode 100644 index 0000000..87a4c5e --- /dev/null +++ b/golden-test-set/annotations/gt-doc-004-missing-sections.annotations.yaml @@ -0,0 +1,80 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-004-missing-sections.md" +artifact_sha256: "1d47a2d6502ff2f7d6d0a7c814760c4376849ea56b242b59d5784bb8900a9c67" + +expected_verdict: REVISE + +findings: + - id: GT-043 + description: > + The API specification documents 4 endpoints (Search Documents, Get Collection + Info, List Collections, Suggest) but provides error response documentation + for only 0 of the 4 endpoints. The "Search Documents" endpoint documents + success response (200 OK) only. Neither the error conditions (invalid + collection_id, malformed query, collection not found, server errors) nor + the HTTP status codes and error body schema for failure cases are documented + for any of the 4 endpoints. Consumers cannot implement correct error handling + without this information. + location: "section 'Endpoints', all 4 endpoint subsections — no error response tables present" + severity: HIGH + category: completeness + critic: completeness + rubric_criterion: null + notes: > + None of the 4 endpoints document error responses. The Response Headers section + references X-RateLimit-* headers suggesting rate limiting exists but no + 429 error response is documented. The SDK example shows typed exception + classes (CollectionNotFoundError, RateLimitError) implying these error + conditions exist and matter to consumers, but the API spec itself does + not document them. + + - id: GT-044 + description: > + The API specification contains no authentication section. Authentication + is a critical cross-cutting concern for any API; consumers need to know + what authentication mechanisms are accepted, what credentials to supply, + and what happens when authentication fails (401 vs 403, error body schema). + The document jumps from Overview directly to Endpoints with no authentication + documentation whatsoever. The Request Headers table mentions no auth headers. + location: "document-wide — absent section; Overview → Endpoints transition" + severity: HIGH + category: completeness + critic: completeness + rubric_criterion: null + notes: > + The document mentions in the Versioning section that "v1 is deprecated" and + in the SDK example that `api_key="your-api-key"` is passed to the client, + but the spec itself has no section explaining how authentication works, + what headers carry credentials, or what the 401/403 response looks like. + This is a critical omission for any API spec. + + - id: GT-045 + description: > + The Response Headers section lists `X-RateLimit-Limit`, `X-RateLimit-Remaining`, + and `X-RateLimit-Reset` headers, clearly indicating rate limiting is + implemented. However, the specification contains no section documenting + rate limiting behavior: what the limits are, what tier the caller is on, + what happens when limits are exceeded (HTTP 429 response, Retry-After), + or how rate limit tiers differ by client type. The headers are listed + but their context is absent. + location: "section 'Request / Response Headers', Response Headers table" + severity: MEDIUM + category: completeness + critic: completeness + rubric_criterion: null + notes: > + The presence of rate limit headers is documented but the rate limiting policy + is not. This is a MEDIUM rather than HIGH because the headers at least signal + that rate limiting exists; a consumer can infer behavior from the headers + alone, but cannot know the actual limits or tiers. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: markdown-doc + complexity: low + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-005-research-gaps.annotations.yaml b/golden-test-set/annotations/gt-doc-005-research-gaps.annotations.yaml new file mode 100644 index 0000000..3267cdc --- /dev/null +++ b/golden-test-set/annotations/gt-doc-005-research-gaps.annotations.yaml @@ -0,0 +1,86 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-005-research-gaps.md" +artifact_sha256: "3d84765d1f5143263cbeb3898342e8236027acec0434959b90ce3ea0fae84627" + +expected_verdict: REVISE + +findings: + - id: GT-046 + description: > + Section 1.2 (Research Questions) explicitly lists 3 research questions: + RQ1 (task performance effects), RQ2 (moderating factors), and RQ3 (governance + and architecture effects on adoption). The Findings section addresses RQ1 + (§3.1) and RQ2 (§3.2) directly. Section 3.3 is titled "Governance and + Architecture Effects" and maps to RQ3. However, the Discussion section (§5) + contains no subsection addressing RQ3 — it discusses RQ1 and RQ2 implications + in §5.1 and §5.2 but §5.3 references "Section 4" (gap analysis) which does + not exist in the document. The document structurally omits the discussion of + RQ3 findings even though data was collected. + location: "section 1.2 (3 RQs stated), section 5 Discussion (RQ3 not addressed in 5.1 or 5.2)" + severity: HIGH + category: completeness + critic: completeness + rubric_criterion: null + notes: > + RQ3 has findings in §3.3 but no corresponding discussion subsection in §5. + §5.1 covers "Theoretical Contributions" (RQ1/RQ2), §5.2 covers "Practical + Implications" (RQ1/RQ2), §5.3 references a "gap analysis" in a non-existent + Section 4. The document skips from Section 3 to Section 5 — there is no + Section 4 in the document. + + - id: GT-047 + description: > + The research report makes multiple causal claims throughout the document: + "trust calibration [...] was the strongest predictor of performance improvement" + (§3.2), "Organizations that invested in structured onboarding [...] saw 2.3× + higher sustained adoption" (Executive Summary), and "governance friction is + a first-order adoption barrier" (§5.2). These are causal or near-causal + claims supported by cross-sectional survey and interview data. However, the + document contains no Limitations section that addresses the inability of + cross-sectional designs to establish causality, sample representativeness, + self-report bias, or any other standard limitation of observational research. + §1.3 (Scope and Limitations) lists only topic scope exclusions, not + methodological limitations. + location: "document-wide — absent Limitations section; §1.3 addresses scope only, not methodology" + severity: HIGH + category: completeness + critic: completeness + rubric_criterion: null + notes: > + A research report making causal claims from survey + interview data must + include a limitations section addressing: cross-sectional design limitations, + self-report bias, sample representativeness across sectors, potential + confounding. The document's §1.3 is a scope section, not a limitations + section. §6 (Conclusion) also contains no caveats on causal inference. + + - id: GT-048 + description: > + Section 5.3 (Discussion, "Relationship to Gap Analysis Findings") states: + "As noted in Section 4, the gap analysis revealed several areas where our + findings diverge from prior literature expectations." There is no Section 4 + in this document. The document numbers go 1 → 2 → 3 → 5 → 6. Section 4 + is entirely absent. This is a dangling cross-reference to a section that + was either deleted, never written, or mis-numbered. + location: "section 5.3, first sentence" + severity: HIGH + category: correctness + critic: correctness + rubric_criterion: null + notes: > + The cross-reference to "Section 4" is a correctness finding (broken + reference), not just a completeness finding. The document structure + skips from §3 to §5 entirely. This suggests a section was deleted + without updating the cross-reference. Both the missing section + (completeness) and the broken reference (correctness) are issues, + but the broken reference is the more actionable finding. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: markdown-doc + complexity: high + rubric: research-synthesis + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-006-poor-structure.annotations.yaml b/golden-test-set/annotations/gt-doc-006-poor-structure.annotations.yaml new file mode 100644 index 0000000..5125cad --- /dev/null +++ b/golden-test-set/annotations/gt-doc-006-poor-structure.annotations.yaml @@ -0,0 +1,67 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-006-poor-structure.md" +artifact_sha256: "5bcca63f377dd756467fbe022d4ce4de0a07c729b415057f373c39bf6ce0530c" + +expected_verdict: REVISE + +findings: + - id: GT-049 + description: > + Installation instructions appear in 3 separate locations in this document, + each with overlapping or identical content: (1) the "Getting Started" section + provides a 5-step installation walkthrough; (2) the "Setting Up Your + Development Environment" section mid-document repeats the same steps in + prose form with the same commands; (3) the "Local Development Setup + (Quick Reference)" section at the end condenses the same steps again as a + one-liner. The three sections are not cross-referenced to each other, so a + reader following the document linearly encounters the same instructions + three times. If any step changes (e.g., requirements file name, Python + version), it must be updated in three places, creating a maintenance hazard. + location: "section 'Getting Started' (steps 1-5), section 'Setting Up Your Development Environment', and section 'Local Development Setup (Quick Reference)'" + severity: HIGH + category: code_hygiene + critic: code_hygiene + rubric_criterion: null + notes: > + This is a classic DRY violation in documentation. The three sections + contain the same git clone, venv creation, pip install, docker-compose up, + and make test commands. The "Quick Reference" section even explicitly says + "This is the same process as the full setup above, condensed for + convenience" — acknowledging the redundancy. One canonical installation + section with a quick-reference summary would be appropriate; three + separate full-length sections is not. + + - id: GT-050 + description: > + The document uses an inconsistent heading hierarchy throughout. The top-level + document heading is `#` (h1). The first section heading ("Getting Started") + is also `#` (h1), which is valid. However, the "Prerequisites" and + "Installation Steps" subsections use `###` (h3), skipping h2 entirely. + Later, "Architecture Overview" and "Configuration Reference" use `##` (h2), + while "Codebase Structure" and "Running Tests" use `#####` (h5), skipping + h3 and h4. The heading levels are applied inconsistently, creating an + incoherent document outline that breaks screen reader navigation and + document parsers expecting a proper hierarchy. + location: "document-wide; specific examples: 'Getting Started' (h1), 'Prerequisites' (h3), 'Codebase Structure' (h5), 'Running Tests' (h5), 'Architecture Overview' (h2)" + severity: MEDIUM + category: code_hygiene + critic: code_hygiene + rubric_criterion: null + notes: > + The heading hierarchy jumps from h1 → h3 (skipping h2) in the Getting + Started section, and from h1 → h5 for the Codebase Structure and Running + Tests subsections. The Configuration Reference uses h2 with h3 subsections + (correct), but this is inconsistent with how other sections are structured. + Standard Markdown guidance and accessibility guidelines require a logical + hierarchy without skipping levels. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: markdown-doc + complexity: medium + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-007-minor-style.annotations.yaml b/golden-test-set/annotations/gt-doc-007-minor-style.annotations.yaml new file mode 100644 index 0000000..6c950db --- /dev/null +++ b/golden-test-set/annotations/gt-doc-007-minor-style.annotations.yaml @@ -0,0 +1,39 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-007-minor-style.md" +artifact_sha256: "0a2aa92ae538a11c1d64f52d3e5c6826ddb12093ea3c7a7f8fac5e0bbe326745" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-051 + description: > + The "What's New in 2.3.0" section uses a mix of bullet point styles: lines + beginning with `*` (asterisk) and lines beginning with `-` (hyphen) are + interleaved within the same list. Specifically: `*` is used for "Added + AsyncSearchClient," "ClientConfig class," and "Performance improvements" + while `-` is used for "Pagination support" and "Improved error messages." + Standard Markdown style guides (and most linters) require consistent use + of a single bullet character within a list. Mixed styles reduce visual + consistency and may render inconsistently across Markdown parsers. + location: "section 'What's New in 2.3.0', bullet list" + severity: LOW + category: code_hygiene + critic: code_hygiene + rubric_criterion: null + notes: > + This is a genuine but minor style issue. The document is otherwise + well-structured, accurate, and complete. The mixed bullet style does not + affect comprehension or correctness — it is a formatting inconsistency + only. PASS_WITH_NOTES is the correct verdict because the document is + usable as-is; this finding warrants a note but not a revision request. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: markdown-doc + complexity: low + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-008-minor-completeness.annotations.yaml b/golden-test-set/annotations/gt-doc-008-minor-completeness.annotations.yaml new file mode 100644 index 0000000..b720d46 --- /dev/null +++ b/golden-test-set/annotations/gt-doc-008-minor-completeness.annotations.yaml @@ -0,0 +1,61 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-008-minor-completeness.md" +artifact_sha256: "fc38d5ebcada394b8e569cde94a1320353220b522b3dcc05a32f751997adecaa" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-052 + description: > + The "Ingest Documents" endpoint (§2, POST /api/v2/index/collections/{id}/documents) + documents only the 202 Accepted response body (the async job creation response). + It does not document the error responses for this endpoint. By contrast, the + "Create Collection" endpoint documents 3 specific error responses (400, 409, 422) + with structured error codes, and the "Delete Document" endpoint documents + a 404 error. The Ingest Documents endpoint has no error response examples + despite being the most likely endpoint to encounter errors (invalid document + format, oversized content, schema violations, collection not found). A + consumer cannot implement error handling without knowing the error response + schema. + location: "section 'Endpoints', subsection '2. Ingest Documents' — no error response table" + severity: MEDIUM + category: completeness + critic: completeness + rubric_criterion: null + notes: > + This is a real gap: the highest-traffic, most error-prone endpoint lacks + error documentation. The surrounding endpoints have it, making this an + inconsistency within the document as well as an omission. MEDIUM severity + because other endpoints model the pattern; a developer can infer likely + errors, but should not have to. + + - id: GT-053 + description: > + The specification has no changelog or version history section. The document + is versioned (v1.0) but contains no record of what changed from prior + versions, who made changes, or when. For an API specification that will + be updated over time, the absence of version history makes it impossible + for consumers to identify what changed between spec versions without + diffing documents manually. + location: "document-wide — absent section" + severity: LOW + category: completeness + critic: completeness + rubric_criterion: null + notes: > + LOW severity because this is a v1.0 document — there is no prior version + to diff against. The absence of a changelog is a gap for future maintainability + but does not impede current use. PASS_WITH_NOTES is appropriate: the document + is usable, but should establish a revision history section now for future + versions. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: markdown-doc + complexity: low + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-009-clean-spec.annotations.yaml b/golden-test-set/annotations/gt-doc-009-clean-spec.annotations.yaml new file mode 100644 index 0000000..408eece --- /dev/null +++ b/golden-test-set/annotations/gt-doc-009-clean-spec.annotations.yaml @@ -0,0 +1,49 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-009-clean-spec.md" +artifact_sha256: "60037c4891f5b83adbb1354e2ea5e2900401e77ac5c18d39fc87decf10ae8cf8" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: > + Section 11 is titled "Known Limitations" and explicitly documents 5 design + boundaries of the v0.1.0 specification (no cross-cluster replication, no + exactly-once end-to-end, KRaft combined mode, schema evolution scope, + no DLQ automation). A critic might flag this as a completeness gap or as + the system being "incomplete." It is not. This is exemplary technical + documentation: explicitly scoping and documenting known limitations is + best practice for a v0.1.0 spec. The limitations are intentional design + boundaries, not omissions. + location: "section 11 (Known Limitations)" + notes: > + Known Limitations sections are a sign of documentation maturity, not a + defect. If Quorum flags §11 as a completeness issue ("the spec is missing + cross-cluster replication"), that is a false positive — the spec correctly + states this is out of scope. The limitations section is the evidence that + the author considered these cases and made a deliberate call. + + - description: > + The spec version is "v0.1.0" throughout the document, including in the + title and revision history. A critic might flag this as indicating the + document is a pre-release draft not ready for use, or that versioned + references to "v0.1.0" suggest outdated content. This is not an issue. + v0.1.0 is explicitly the current accepted version of this spec (status + is "Draft — Accepted for implementation"). The version number reflects + where the system is in its development lifecycle, not a documentation flaw. + location: "document title, status field, §11 Known Limitations references, §13 Revision History" + notes: > + v0.1.0 is the legitimate current version. The revision history in §13 + shows two entries for v0.1.0 (initial draft + review revision), which + is correct version tracking behavior. If Quorum flags the version as + "outdated" or "pre-release = incomplete," that is a false positive. + +metadata: + source: synthetic + domain: markdown-doc + complexity: high + rubric: markdown-doc + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-doc-010-clean-research.annotations.yaml b/golden-test-set/annotations/gt-doc-010-clean-research.annotations.yaml new file mode 100644 index 0000000..9f21e9f --- /dev/null +++ b/golden-test-set/annotations/gt-doc-010-clean-research.annotations.yaml @@ -0,0 +1,66 @@ +schema_version: "1.0" +artifact: "artifacts/docs/gt-doc-010-clean-research.md" +artifact_sha256: "a88ab99aa6a5db10addff40a0d836eefb7ca848f5cd22b5fcd7a00b62849083d" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: > + Section 7 is titled "Future Work" and lists three research directions not + addressed in the current report: benchmarking in agentic systems, formal + verification approaches, and longitudinal defense durability. A critic + might interpret these as gaps in the current report — topics that were + researched but not covered. They are not gaps. "Future Work" is a standard + and expected section in a research synthesis that explicitly scopes what + the current report does and does not cover. The three items listed are + genuinely out of scope for this synthesis and are correctly deferred. + location: "section 7 (Future Work)" + notes: > + Future Work sections are proper research document structure. They signal + intellectual honesty about scope boundaries, not incompleteness in what + was claimed to be covered. If Quorum flags §7 as "the report is missing + coverage of agentic system benchmarks," that is a false positive — the + report explicitly does not claim to cover that. The synthesis covers what + it says it covers. + + - description: > + The document notes in §6 (Limitations) that "The rate of publication in + this domain is high and our search has a cutoff of December 2025." A critic + might flag the December 2025 cutoff as a stale-data issue (the document + date is January 2026). This is not a defect. A literature search cutoff + that is 1-2 months before the document completion date is standard practice + — it allows time for synthesis and writing after search completion. The + cutoff is clearly disclosed. This is correct methodology. + location: "section 6 (Limitations), first bullet" + notes: > + A literature search cutoff slightly before the document date is normal and + expected. The cutoff is explicitly disclosed in the limitations section, + which is best practice. If Quorum flags this as "outdated search" that is + a false positive. + + - description: > + The research report cites papers using "arXiv:NNNN.XXXXX" format for one + reference (Perez & Ribeiro 2022, arXiv:2211.09527) while citing others + by venue (USENIX Security, IEEE S&P, NeurIPS). A critic might flag + inconsistent citation format as a correctness or hygiene issue. Mixed + venue/arXiv citation format is acceptable and common in rapidly evolving + security research, where arXiv versions and conference versions coexist. + The Perez & Ribeiro citation's arXiv ID is real and verifiable. + location: "section 'References', Perez & Ribeiro (2022) entry" + notes: > + Mixed citation format (some by venue, one by arXiv ID) is not a defect. + It reflects the realistic publication landscape of LLM security research + where some work is best referenced by arXiv preprint and some by venue. + If Quorum flags citation format inconsistency as a finding, that is + over-classification. + +metadata: + source: natural + domain: markdown-doc + complexity: medium + rubric: research-synthesis + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-001-sql-injection.annotations.yaml b/golden-test-set/annotations/gt-py-001-sql-injection.annotations.yaml new file mode 100644 index 0000000..df08831 --- /dev/null +++ b/golden-test-set/annotations/gt-py-001-sql-injection.annotations.yaml @@ -0,0 +1,37 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-001-sql-injection.py" +artifact_sha256: "29e880b3866d2e3db9c076ad1bd1491ccfcf492608b2ee7b78e50f9a420d3611" + +expected_verdict: REJECT + +findings: + - id: GT-001 + description: > + SQL injection via unsanitized user input in query builder. The `search_users` + route concatenates the `q` query parameter and the `field` parameter directly + into an SQL string using string concatenation. Although `field` is validated + against an allow-list, `query_term` is not parameterized. An attacker can + inject arbitrary SQL via the `q` parameter (e.g., `%' OR '1'='1`). + location: "line 37-39" + severity: CRITICAL + category: security + critic: security + notes: > + The `field` allow-list check on line 33-35 is a partial mitigation that + prevents column name injection, but does not protect against value injection + in the LIKE clause. The fix is to use a parameterized query: + `db.execute("... WHERE " + field + " LIKE ?", ("%"+query_term+"%",))`. + The other three routes (get_user, get_user_activity, create_user) correctly + use parameterized queries and are not findings. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: python-code + complexity: low + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-002-hardcoded-credentials.annotations.yaml b/golden-test-set/annotations/gt-py-002-hardcoded-credentials.annotations.yaml new file mode 100644 index 0000000..7f8c1ab --- /dev/null +++ b/golden-test-set/annotations/gt-py-002-hardcoded-credentials.annotations.yaml @@ -0,0 +1,58 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-002-hardcoded-credentials.py" +artifact_sha256: "50b6081082103cd590fc552df65c591e36a4c7eb84a0c417405e12ba728cae40" + +expected_verdict: REJECT + +findings: + - id: GT-002 + description: > + Hardcoded database password in connection string. `DB_PASSWORD` is assigned + a literal production password string on line 15. Any developer with read access + to the source repository gains the production database credential. The credential + will also appear in version control history even if removed later. + location: "line 15" + severity: CRITICAL + category: security + critic: security + notes: > + The fix is to read credentials from environment variables or a secrets manager + at runtime: `DB_PASSWORD = os.environ["DB_PASSWORD"]`. The literal value + `"Tr0ub4dor&3_prod_2024!"` is a real-looking password that should trigger + secret detection tooling. + + - id: GT-003 + description: > + API key stored as a string literal. `ANALYTICS_API_KEY` on line 22 contains a + hardcoded live API key (prefixed `sk-live-`). Exposure in source control gives + any reader full analytics platform access under the application's identity. + location: "line 22" + severity: HIGH + category: security + critic: security + notes: > + Like GT-002, this should be injected at runtime from environment variables or + a vault. The `sk-live-` prefix pattern is a common secret format used by + Stripe-style APIs and should be caught by secret scanning rules. + +false_positive_traps: + - description: > + `PASSWORD_PATTERN` on line 26 is a compiled regex used to detect and redact + credential-like strings in log output. It is not a credential itself. + location: "line 26-29" + notes: > + Naive pattern matching on the variable name `PASSWORD_PATTERN` or on the + string content of the regex (which mentions "password", "secret", "token") + would produce a false positive. The context makes clear this is a detector, + not a secret. The `scrub_log_line` method on line 137 further confirms the + defensive purpose of this variable. + +metadata: + source: synthetic + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-003-command-injection.annotations.yaml b/golden-test-set/annotations/gt-py-003-command-injection.annotations.yaml new file mode 100644 index 0000000..48231ec --- /dev/null +++ b/golden-test-set/annotations/gt-py-003-command-injection.annotations.yaml @@ -0,0 +1,71 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-003-command-injection.py" +artifact_sha256: "c9febf0ea5a72162d8acb2c7062890964f096286cf43b5bc91ad0e541266d58f" + +expected_verdict: REJECT + +findings: + - id: GT-004 + description: > + Command injection via `os.system()` with user-controlled input. In + `download_artifact()`, both `artifact_name` and `version` are obtained from + CLI arguments (ultimately from external callers) and embedded directly into a + shell command string passed to `os.system()`. An attacker who controls either + argument can inject arbitrary shell commands (e.g., + `artifact_name="foo; rm -rf /"`). + location: "line 35-40" + severity: CRITICAL + category: security + critic: security + notes: > + `os.system()` passes its argument to a shell, so any shell metacharacters in + the interpolated values are executed. The fix is to use + `subprocess.run(["curl", "-fsSL", url, "-o", str(dest_path)], check=True)` + with the URL and destination path constructed separately from validated inputs. + + - id: GT-005 + description: > + `subprocess.run()` called with `shell=True` and an unsanitized argument. In + `run_migrations()`, `migration_dir` comes from a CLI argument and is + interpolated into `migrate_cmd` which is then executed with `shell=True`. + An operator or CI system that passes a crafted `--migration-dir` value can + inject shell commands. + location: "line 52-60" + severity: HIGH + category: security + critic: security + notes: > + `shell=True` with a string command is equivalent to `os.system()` in terms + of injection surface. The fix is to pass a validated list: + `subprocess.run(["alembic", "--config", config_path, "upgrade", "head"], ...)`. + The `db_url` is passed via the environment (`DATABASE_URL`), which is correct + and not a finding. + +false_positive_traps: + - description: > + `subprocess.run(["ls", "-la"], check=True)` on line 72 — fully controlled, + hardcoded arguments with no user input. This is a textbook safe subprocess call. + location: "line 72" + notes: > + The command is a string list with no dynamic content. `shell=True` is not used. + Flagging this as injection would be a false positive on the presence of + `subprocess.run` alone. + + - description: > + `restart_service()` and `check_service_status()` use `subprocess.run` with + list arguments derived from `_safe_service_name()`, which validates against + an explicit allow-list. The service name is not user-controlled in the + injection sense — it is constrained to `{"api", "worker", "scheduler", "gateway"}`. + location: "line 44-56" + notes: > + Allow-list validation before subprocess is the correct pattern. Not a finding. + +metadata: + source: synthetic + domain: python-code + complexity: high + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-004-insecure-deserialization.annotations.yaml b/golden-test-set/annotations/gt-py-004-insecure-deserialization.annotations.yaml new file mode 100644 index 0000000..c256342 --- /dev/null +++ b/golden-test-set/annotations/gt-py-004-insecure-deserialization.annotations.yaml @@ -0,0 +1,40 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-004-insecure-deserialization.py" +artifact_sha256: "ef53b870f315c0d1a7db9e67c29f47c7a1976be55468ad45a49c9a142379386e" + +expected_verdict: REJECT + +findings: + - id: GT-006 + description: > + `pickle.loads()` called on data received from a network socket. In the `get()` + method of `CacheClient`, the raw response bytes from the cache server are + deserialized with `pickle.loads()` when the type tag byte is `0x02`. The cache + server is on the network (`cache.internal.example.com:6380`) and any party who + can write to the cache or intercept/replay cache responses can deliver a + crafted pickle payload that executes arbitrary code during deserialization. + location: "line 108-113" + severity: CRITICAL + category: security + critic: security + notes: > + `pickle` deserialization is inherently unsafe against untrusted input because + pickle opcodes can call arbitrary Python callables during object reconstruction. + The comment on line 112-113 explicitly acknowledges that "external callers on + the network segment" may have written the data. The fix is to use a safe + serialization format (JSON, MessagePack, or Protocol Buffers) for all cross- + process/cross-network data. If full object fidelity is required, the cache + must be treated as a trusted, authenticated channel with strict input + validation before deserialization. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-005-logic-errors.annotations.yaml b/golden-test-set/annotations/gt-py-005-logic-errors.annotations.yaml new file mode 100644 index 0000000..56bd3e5 --- /dev/null +++ b/golden-test-set/annotations/gt-py-005-logic-errors.annotations.yaml @@ -0,0 +1,58 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-005-logic-errors.py" +artifact_sha256: "9d2701b22d27a087b9bd76a543d4721ee29f2c8ce6f66c024f1db20da8a0e1c4" + +expected_verdict: REVISE + +findings: + - id: GT-007 + description: > + Off-by-one error in discount tier boundary check. `compute_tier_discount()` + uses strict greater-than (`>`) to compare the subtotal against tier thresholds. + Per the business rule in the comment (spec §4.2), a subtotal of exactly + `$250.00` should qualify for the 15% tier, but the strict `>` causes it to + fall through to the 10% tier instead. The same off-by-one applies at the $100 + and $50 boundaries. + location: "line 60" + severity: HIGH + category: correctness + critic: correctness + notes: > + The spec comment on lines 46-51 explicitly states the boundary includes the + threshold value ("$250+" and "$50–$99.99"). The fix is to change `>` to `>=` + on line 60: + `if subtotal >= threshold:` + This is a classic fence-post error that will silently under-discount customers + at exact tier boundaries. + + - id: GT-008 + description: > + Misleading operator precedence comment and incorrect rounding in tax + calculation. In `compute_tax()`, the expression + `taxable_amount * rate.quantize(Decimal("0.0001")) + Decimal("0")` rounds + the `rate` to 4 decimal places *before* multiplying, and the comment claims + this causes the final tax to be off by $0.01–$0.04. The rounding quantum + should be applied to the *product*, not to the rate alone. The correct + expression is `(taxable_amount * rate).quantize(Decimal("0.01"), ...)`. + location: "line 89-93" + severity: MEDIUM + category: correctness + critic: correctness + notes: > + The `+ Decimal("0")` on line 92 is a no-op that adds confusion without + fixing the rounding issue. The `.quantize()` call at line 95 does apply + correct final rounding, but operating on `rate` rather than the product + means intermediate precision loss accumulates. For large taxable amounts + the discrepancy can exceed $0.04. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-006-missing-error-handling.annotations.yaml b/golden-test-set/annotations/gt-py-006-missing-error-handling.annotations.yaml new file mode 100644 index 0000000..11c0ad9 --- /dev/null +++ b/golden-test-set/annotations/gt-py-006-missing-error-handling.annotations.yaml @@ -0,0 +1,74 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-006-missing-error-handling.py" +artifact_sha256: "329ec9024428fc12f2f44609c6080db1fc32bf8419d8f71e0572440f443e273a" + +expected_verdict: REVISE + +findings: + - id: GT-009 + description: > + No exception handling for `FileNotFoundError` (or `PermissionError`) in + `iter_csv_records()`. The function opens the file with a bare `open()` call + and explicitly documents that it "does not handle missing files." When called + from `process_file()` → `run_pipeline()`, the exception propagates past the + `except Exception` catch in `run_pipeline()` (which does catch it at the + pipeline level), but the docstring of `iter_csv_records` misleads callers + that a valid path is a precondition. More critically, the file handle `fh` is + not managed with a context manager — if the iterator is abandoned mid-read + (e.g., caller breaks out of the loop), the file descriptor is leaked. + location: "line 36-42" + severity: HIGH + category: completeness + critic: completeness + notes: > + The fix is to use `with open(filepath, ...) as fh:` and yield inside the + context manager, or convert to `yield from csv.DictReader(...)` with proper + resource management. The `FileNotFoundError` should be caught and re-raised + with a descriptive message, or handled at the call site with explicit logging. + + - id: GT-010 + description: > + Unchecked return value from the enrichment API call. In `enrich_record()`, + `response.json().get("enrichment", {})` is called without first checking + `response.status_code`. If the API returns a 4xx or 5xx response, `response.json()` + may raise a `JSONDecodeError` (if the body is not JSON) or return an error + payload that does not contain `"enrichment"`. In the latter case, the function + silently returns `{**record, "enrichment": {}}` as if enrichment succeeded, + masking the failure. The outer retry loop in `enrich_batch()` only catches + `requests.RequestException`, not `ValueError` from `json()` or silent API errors. + location: "line 66-71" + severity: HIGH + category: completeness + critic: completeness + notes: > + The fix is to call `response.raise_for_status()` before accessing + `response.json()`, which converts HTTP error codes into `requests.HTTPError` + (a subclass of `requests.RequestException`) that the retry loop will catch. + + - id: GT-011 + description: > + Missing timeout on the HTTP request in `enrich_record()`. The `requests.post()` + call has no `timeout` parameter. If the enrichment service hangs (e.g., during + a deployment or partial outage), the worker process blocks indefinitely on that + single record, stalling the entire pipeline with no way to recover other than + killing the process. + location: "line 67" + severity: MEDIUM + category: completeness + critic: completeness + notes: > + Add `timeout=(5, 30)` (connect, read) to the `requests.post()` call. + `requests.Timeout` is already a subclass of `requests.RequestException`, so + the existing retry logic in `enrich_batch()` would handle it automatically. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-007-dead-code.annotations.yaml b/golden-test-set/annotations/gt-py-007-dead-code.annotations.yaml new file mode 100644 index 0000000..a3b7bf9 --- /dev/null +++ b/golden-test-set/annotations/gt-py-007-dead-code.annotations.yaml @@ -0,0 +1,70 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-007-dead-code.py" +artifact_sha256: "d37a7cc8c4d07673637ec686ccab86fcb8e4e824a1b1028d324b9845b765f3c1" + +expected_verdict: REVISE + +findings: + - id: GT-012 + description: > + Unreachable code after unconditional `return` in `classify_size()`. The two + lines after the final `return "tiny"` statement — `category = "unknown"` and + `return category` — are dead code. Every branch of the preceding if/elif/else + chain contains a `return`, so these lines can never execute. They are vestigial + from an earlier implementation and will never be reached at runtime. Static + analysis tools (pylint, pyflakes, mypy) will flag this. + location: "line 189-190" + severity: HIGH + category: code_hygiene + critic: code_hygiene + notes: > + Dead code should be removed rather than left as-is. Beyond being confusing to + readers, it creates a maintenance hazard: a future refactor might remove one + of the early returns, inadvertently "activating" the dead path with unintended + behavior. The entire `classify_size()` function body after the `else: return "tiny"` + clause (lines 189-190) should be deleted. + + - id: GT-013 + description: > + Unused import: `import xml.etree.ElementTree as ET` at the top of the file. + Nothing in the module uses `ET` or any XML-related functionality. The comment + acknowledges this is a leftover from a refactor. Unused imports add noise, + slow import time marginally, and mislead readers about module dependencies. + location: "line 13" + severity: MEDIUM + category: code_hygiene + critic: code_hygiene + notes: > + Remove the import. If XML support is needed in the future, it can be + re-added at that time. + + - id: GT-014 + description: > + Duplicated word-join capitalization logic in `snake_to_camel()` and + `_title_case_words()`. The `snake_to_camel()` function (line ~109) inline- + implements `components[0] + "".join(x.title() for x in components[1:])` rather + than calling the `_title_case_words()` helper that exists in the same file + (line ~117). The comment in `_title_case_words()` acknowledges this duplication + explicitly. Both functions are maintained independently, creating a risk of + divergence in behavior. + location: "line 109-111 and line 117-119" + severity: MEDIUM + category: code_hygiene + critic: code_hygiene + notes: > + `snake_to_camel()` should delegate the title-casing to `_title_case_words()`, + applying it only to the tail components (not the first word). This eliminates + the duplicate logic. Alternatively, consolidate into a single private helper + and call it from both `snake_to_camel()` and `snake_to_pascal()`. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-008-path-traversal.annotations.yaml b/golden-test-set/annotations/gt-py-008-path-traversal.annotations.yaml new file mode 100644 index 0000000..fba7b97 --- /dev/null +++ b/golden-test-set/annotations/gt-py-008-path-traversal.annotations.yaml @@ -0,0 +1,67 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-008-path-traversal.py" +artifact_sha256: "4be1b46aae1abca26aca198d1accc3ca3d3a04e4f1d8c1a852c8c89f0f7c6783" + +expected_verdict: REVISE + +findings: + - id: GT-015 + description: > + Path traversal via unsanitized user-controlled filename in + `serve_user_document()`. The `filename` URL parameter is joined directly to + the user's upload directory with `user_dir / filename` and passed to + `send_file()` without any normalization or containment check. An attacker can + request a URL like `/api/users/1/documents/../../../etc/passwd` to read + arbitrary files accessible to the web server process. + location: "line 84-92" + severity: HIGH + category: security + critic: security + notes: > + The fix is to resolve the final path and verify it is still within `user_dir`: + `resolved = (user_dir / filename).resolve()` + `if not str(resolved).startswith(str(user_dir.resolve())): abort(404)` + Alternatively, use `werkzeug.utils.secure_filename()` on the filename before + joining, as is already done correctly in the upload endpoint (`serve_user_document` + uses raw filename while `upload_user_document` uses `secure_filename` — the + inconsistency is itself noteworthy). Note that `secure_filename()` alone + is not sufficient if the user_dir path itself is untrusted. + + - id: GT-016 + description: > + Directory listing exposure in `list_user_documents()`. The endpoint returns + the names, sizes, and modification timestamps of all files in a user's upload + directory with no authentication or authorization check. Any caller — including + unauthenticated external requests — can enumerate a user's document names by + hitting `/api/users//documents` with any integer user_id. This leaks + existence and metadata of documents even when the requester has no right to + read them. + location: "line 98-113" + severity: MEDIUM + category: security + critic: security + notes: > + At minimum, this endpoint should verify that the requesting identity matches + `user_id` (or is an admin). The route docstring acknowledges there is "no + authentication or authorization check." + +false_positive_traps: + - description: > + `os.path.join(BASE_DIR, "static", "style.css")` in `serve_stylesheet()` uses + fully hardcoded path components with no user input. This is not a path traversal + risk. + location: "line 42" + notes: > + `BASE_DIR`, `"static"`, and `"style.css"` are all constants or literals. No + user-controlled value is involved. Flagging this would be a false positive + triggered by the presence of `os.path.join` alone. + +metadata: + source: modified-natural + domain: python-code + complexity: high + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-009-type-mismatch.annotations.yaml b/golden-test-set/annotations/gt-py-009-type-mismatch.annotations.yaml new file mode 100644 index 0000000..c8b4da5 --- /dev/null +++ b/golden-test-set/annotations/gt-py-009-type-mismatch.annotations.yaml @@ -0,0 +1,69 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-009-type-mismatch.py" +artifact_sha256: "db5cc12fcd21a81ec537364604150a1a8cda23873a1d30726b5ab3f1ecf7bbed" + +expected_verdict: REVISE + +findings: + - id: GT-017 + description: > + String/int comparison in `is_high_priority()` always evaluates to `False` for + users not in `USER_SEGMENTS`. The `priority` field of a `NormalizedEvent` is + assigned directly from `segment.get("priority", 1)`. For users in the segment + store, `priority` is an `int`. For unknown users, the default dict returns + `"1"` (a string, as documented in `lookup_user_segment()`). The comparison + `event.priority >= HIGH_PRIORITY_THRESHOLD` then compares `str >= int`, which + raises a `TypeError` in Python 3 rather than silently returning False. + In practice, any event for an unknown user will raise an unhandled `TypeError` + in `route_event()`, causing that event to be lost. + location: "line 98-103" + severity: HIGH + category: correctness + critic: correctness + notes: > + The root cause is the heterogeneous type in `lookup_user_segment()`'s default + dict (line 73). The fix is to coerce `priority` to `int` at assignment time + in `transform()`: `priority = int(segment.get("priority", 1))`. The annotation + description says "always evaluates to False" which is the Python 2 behavior; + in Python 3 it raises TypeError. Either formulation is acceptable for the + critic to flag — the core finding is the type inconsistency. + + - id: GT-018 + description: > + `dict.get()` default type mismatch in `summarize()`. On the line + `extra = metadata.get("missing_key", {})`, the default is a dict `{}` but the + docstring states callers that do "sum(meta.values())" expect numeric values. + When the key is absent, callers receive an empty dict and will get a + `TypeError` when they try to sum it with integers. The variable name `extra` + and the comment confirm this is an intentional placeholder with an + incorrect default type. + location: "line 189-190" + severity: MEDIUM + category: correctness + critic: correctness + notes: > + The default should match the expected type of the value. If callers expect + an `int`, the default should be `0`. If callers expect a `dict`, downstream + arithmetic assumptions need to change. The inconsistency between the default + type and documented usage is the finding. + +false_positive_traps: + - description: > + `int("42")` on line 148 — valid literal conversion of a string constant to + an integer, representing a configuration value parsed from the environment. + This is not a type error or a bug. + location: "line 148" + notes: > + The conversion is well-defined and intentional. `"42"` is a string literal + (not user-controlled input), and `int("42")` always produces `42`. Flagging + this as a type coercion issue would be incorrect. + +metadata: + source: synthetic + domain: python-code + complexity: high + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-010-minor-hygiene.annotations.yaml b/golden-test-set/annotations/gt-py-010-minor-hygiene.annotations.yaml new file mode 100644 index 0000000..fb45c4b --- /dev/null +++ b/golden-test-set/annotations/gt-py-010-minor-hygiene.annotations.yaml @@ -0,0 +1,52 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-010-minor-hygiene.py" +artifact_sha256: "a83d2b1dc496107e45052cd790146f9c45f37fb3ff12a096398fd6d18172cb1a" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-019 + description: > + `cron_next_run()` function body exceeds 50 lines (approximately 55 lines + including docstring and blank lines). While the function is well-documented + and logically coherent, it is at the upper edge of the conventional + single-responsibility limit for Python functions. PEP 8 does not set a hard + limit, but the project rubric flags functions exceeding 50 lines as a hygiene + advisory. + location: "line 112-167" + severity: MEDIUM + category: code_hygiene + critic: code_hygiene + notes: > + This is a borderline advisory. The function could reasonably be split into a + pure "compute next run time" core and a separate "apply business hours + adjustment" wrapper. The overall module quality is high; this is the only + structural concern. + + - id: GT-020 + description: > + Magic number `86400` (seconds in a day) used inline in `humanize_timedelta()` + without a named constant. The value appears multiple times in the function + body without being defined as a named constant (e.g., `SECONDS_PER_DAY = 86400`). + While `86400` is a well-known value, the rubric flags inline numeric literals + that represent meaningful domain constants. + location: "line 183" + severity: LOW + category: code_hygiene + critic: code_hygiene + notes: > + `86400` could be extracted to a module-level constant `SECONDS_PER_DAY = 86400` + and reused in both `humanize_timedelta()` and the duration parser. Minor + advisory only — does not affect correctness. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: python-code + complexity: low + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-011-minor-completeness.annotations.yaml b/golden-test-set/annotations/gt-py-011-minor-completeness.annotations.yaml new file mode 100644 index 0000000..97b0d36 --- /dev/null +++ b/golden-test-set/annotations/gt-py-011-minor-completeness.annotations.yaml @@ -0,0 +1,52 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-011-minor-completeness.py" +artifact_sha256: "c92d6dcf0d0eb3bcc4bc63f552c17b2d336425c25afb7269b40603215f8f4700" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-021 + description: > + `send_bulk()` is a public method with no docstring. All other public methods + on `NotificationClient` (`send`, `get_status`, `cancel`, `list_recent`, + `health_check`) have descriptive docstrings. `send_bulk()` is missing one + entirely. A caller cannot determine the return type, what errors are raised, + or what the `recipient_ids` list format should be without reading the + implementation. + location: "line 63-72" + severity: MEDIUM + category: completeness + critic: completeness + notes: > + Add a docstring consistent with the other methods. At minimum it should + document: the `recipient_ids` type and expected format, return type, + and any exceptions raised. + + - id: GT-022 + description: > + No input validation on `limit` and `offset` parameters in `list_recent()`. + The docstring explicitly notes that "negative or zero values are passed + through to the API." Negative `limit` or negative `offset` may cause the + API to return unexpected results or an error that the caller is not prepared + to handle. A simple guard (`if limit < 1: raise ValueError(...)`) would + prevent invalid values from being forwarded. + location: "line 90-95" + severity: LOW + category: completeness + critic: completeness + notes: > + Minor advisory. The method correctly delegates pagination to the API and + raises `HTTPError` if the API rejects the params. Adding a local validation + guard improves usability and shifts the error earlier with a clearer message. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: python-code + complexity: low + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-012-ssrf-partial.annotations.yaml b/golden-test-set/annotations/gt-py-012-ssrf-partial.annotations.yaml new file mode 100644 index 0000000..a92e52b --- /dev/null +++ b/golden-test-set/annotations/gt-py-012-ssrf-partial.annotations.yaml @@ -0,0 +1,51 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-012-ssrf-partial.py" +artifact_sha256: "a40c04a6906af82336cb0ded34602a1c058c8617962d1625a7f51d1d55cbf63e" + +expected_verdict: PASS_WITH_NOTES + +findings: + - id: GT-023 + description: > + SSRF via URL scheme bypass: the `validate_url()` function checks that the + scheme is in `_ALLOWED_SCHEMES` (`{"http", "https"}`), but this check is + applied to the scheme as parsed by `urllib.parse.urlparse`. Crafted URLs + such as those using `file://`, `gopher://`, or `dict://` schemes are not + explicitly rejected by name — they are only rejected if `urlparse` parses + the scheme correctly. Additionally, `allow_redirects=True` in `fetch_url()` + means that a valid-looking HTTP URL can redirect to a `file://` or internal + address without triggering re-validation. Separately, decimal/hex-encoded IP + addresses (e.g., `http://2130706433/` = `127.0.0.1`) may pass the string + blocklist check in `_BLOCKED_HOSTS` because the string representation does + not match any entry, even though the socket-layer resolution correctly + returns `127.0.0.1`. The resolve-and-check logic in `_resolve_hostname` + handles this second case, but only if DNS resolution succeeds; raw IP + literals passed as hostnames bypass the resolution path in some urllib + implementations. + location: "line 56-73" + severity: HIGH + category: security + critic: security + notes: > + The http/https scheme check is present and correct for the common case. + The finding is about the edge cases documented in the function's own + docstring: redirect chains, non-standard scheme parsing, and decimal IP + encoding. The verdict is PASS_WITH_NOTES (not REJECT) because the primary + attack surface (direct file:// or gopher:// URLs) is blocked by + `_ALLOWED_SCHEMES`, and the remaining gaps require more exotic attack vectors. + A thorough fix would: (1) disable `allow_redirects` or re-validate each + redirect destination; (2) normalize the URL before scheme extraction; + (3) resolve the host to an IP and compare numerically using `ipaddress` + before the string blocklist check. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-013-clean-crypto.annotations.yaml b/golden-test-set/annotations/gt-py-013-clean-crypto.annotations.yaml new file mode 100644 index 0000000..1ead5d8 --- /dev/null +++ b/golden-test-set/annotations/gt-py-013-clean-crypto.annotations.yaml @@ -0,0 +1,45 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-013-clean-crypto.py" +artifact_sha256: "1b6a30141591bbd011d4f99e33cea264d6bfb747585641b9e5c141406f8c8462" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: > + `hashlib.md5()` usage in `content_fingerprint()`. MD5 is used here for + content deduplication — a non-security purpose where collision resistance + is explicitly not required. The function docstring explains this reasoning + in detail, and the function is named `content_fingerprint` (not a password + or integrity check). Flagging this as a security issue would be incorrect. + location: "line 50" + notes: > + MD5 is only inappropriate for security-sensitive operations (password + hashing, digital signatures, tamper detection against adversarial input). + For deduplication of uploads where the adversary is not trying to produce + collisions, MD5 is acceptable and widely used (e.g., S3 ETag). The module + also provides `content_fingerprint_sha256()` for contexts where a stronger + algorithm is preferred. A noqa comment is included to suppress linter + warnings that would fire without context. + + - description: > + Variable named `evaluator` at the bottom of the module. This is a dict + mapping rule type names to callable functions. It does NOT use Python's + built-in `eval()` function anywhere in the file. Naive pattern matching on + the string "eval" in variable names would produce a false positive. + location: "line 198-201" + notes: > + `evaluator` is a plain dict of callables. `eval()` does not appear anywhere + in the file. The naming is coincidental to the domain (a rule evaluation + registry). No code execution or dynamic evaluation occurs. + +metadata: + source: synthetic + domain: python-code + complexity: high + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-014-clean-api.annotations.yaml b/golden-test-set/annotations/gt-py-014-clean-api.annotations.yaml new file mode 100644 index 0000000..762ada9 --- /dev/null +++ b/golden-test-set/annotations/gt-py-014-clean-api.annotations.yaml @@ -0,0 +1,41 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-014-clean-api.py" +artifact_sha256: "2b51de412d496c823afc1377c658b4e2d71ab11186501f0e821e71f139ac3fe9" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: > + `requests.get(url, timeout=self._cfg.timeout)` in the `health()` method. + This call includes an explicit timeout via `self._cfg.timeout`, which is + a `(connect, read)` tuple defaulting to `(5.0, 30.0)`. It should NOT be + flagged for missing timeout — the timeout is present. + location: "line 155-158" + notes: > + A critic that checks for `requests.get()` calls without `timeout=` would + produce a false positive here. The timeout is passed through `self._cfg.timeout`, + which is initialized in `ClientConfig` with `DEFAULT_TIMEOUT = (5.0, 30.0)`. + The pattern is correct and robust. + + - description: > + The `_request()` method contains recursive calls to itself (`self._request(...)`) + for retry handling. This is not infinite recursion — it is bounded by the + `_attempt` counter compared against `self._cfg.max_retries`. It should not + be flagged as unbounded recursion. + location: "line 222-231" + notes: > + The recursion terminates when `_attempt > self._cfg.max_retries` (default 3), + after which the non-2xx response is mapped to an exception. The depth is + bounded and small. + +metadata: + source: synthetic + domain: python-code + complexity: medium + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-py-015-clean-subprocess.annotations.yaml b/golden-test-set/annotations/gt-py-015-clean-subprocess.annotations.yaml new file mode 100644 index 0000000..3a5ce23 --- /dev/null +++ b/golden-test-set/annotations/gt-py-015-clean-subprocess.annotations.yaml @@ -0,0 +1,54 @@ +schema_version: "1.0" + +artifact: "artifacts/python/gt-py-015-clean-subprocess.py" +artifact_sha256: "df032d0172dae536bd93ac3e1b670cb1059140fb4cdba279b6317f56a1051580" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: > + All `subprocess.run()` and `subprocess.Popen()` calls in this module use + list-form arguments and never pass `shell=True`. These are the safe patterns + for subprocess usage and should NOT be flagged as command injection risks. + location: "line 90-107, line 148-157" + notes: > + Command injection via subprocess requires either `shell=True` (with a string + command) or user-controlled data injected into the command list without + validation. Neither condition is present here. All command lists are either + hardcoded constants (`["ps", "-o", ...]`) or validated before use. Flagging + subprocess calls on the basis of subprocess usage alone would be a false positive. + + - description: > + `os.environ` is accessed in multiple places (lines 15-18, line 90). Reading + environment variables for runtime configuration is a standard, safe practice. + It should NOT be flagged as credential exposure or insecure secret handling. + location: "line 15-18" + notes: > + The environment variables read here are operational configuration: + `BUILD_WORKSPACE`, `MAX_OUTPUT_BYTES`, `BUILD_TIMEOUT_SEC`, and `LOG_LEVEL`. + None of these are credentials, tokens, or secrets. Reading env vars for + configuration is the recommended practice for twelve-factor applications. + A critic that flags all `os.environ` access as potential credential exposure + would produce a false positive here. + + - description: > + The `run_streaming()` function spawns `threading.Thread` instances for + reading stdout and stderr concurrently. This is standard practice for + avoiding subprocess deadlocks when both stdout and stderr must be captured. + It should not be flagged as a threading hazard. + location: "line 148-181" + notes: > + The threads are joined with timeouts before the function returns. The + shared `lines_list` mutable lists are each written by exactly one thread + and only read by the main thread after `join()`, so there is no data race. + +metadata: + source: synthetic + domain: python-code + complexity: high + rubric: python-code + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-sh-001-command-injection.annotations.yaml b/golden-test-set/annotations/gt-sh-001-command-injection.annotations.yaml new file mode 100644 index 0000000..09545e1 --- /dev/null +++ b/golden-test-set/annotations/gt-sh-001-command-injection.annotations.yaml @@ -0,0 +1,43 @@ +schema_version: "1.0" +artifact: "artifacts/shell/gt-sh-001-command-injection.sh" +artifact_sha256: "9e6f1d776afe4d2e19271546f2b278c80293d208072ef6dd308757126e31b513" + +expected_verdict: REJECT + +findings: + - id: GT-054 + description: "eval used with unquoted variable expansion — DEPLOY_CMD value comes directly from a user-controlled config file without sanitization, enabling arbitrary command injection" + location: "line 26" + severity: CRITICAL + category: security + critic: security + rubric_criterion: null + notes: > + DEPLOY_CMD is read from a config file keyed by the first positional argument ($1 = DEPLOY_USER). + An attacker who controls the config file (or supplies a crafted environment name) can inject + arbitrary shell commands. The unquoted expansion compounds the risk — word-splitting and glob + expansion occur before eval runs. Pattern: eval $DEPLOY_CMD (no quotes, no validation). + + - id: GT-055 + description: "curl output piped directly to bash without TLS verification, checksum validation, or content-type inspection — remote code execution via install script" + location: "line 30" + severity: HIGH + category: security + critic: security + rubric_criterion: null + notes: > + The bootstrap URL is constructed from user-supplied $VERSION, then fetched with bare curl and + piped to bash. No --fail, no --cacert, no signature or checksum check. A MITM or compromised + registry can deliver arbitrary code. Even with a trusted server, missing --fail means a 4xx/5xx + response body is still executed. Pattern: curl $URL | bash. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: shell-script + complexity: low + rubric: shell-script + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-sh-002-missing-guards.annotations.yaml b/golden-test-set/annotations/gt-sh-002-missing-guards.annotations.yaml new file mode 100644 index 0000000..deb8d30 --- /dev/null +++ b/golden-test-set/annotations/gt-sh-002-missing-guards.annotations.yaml @@ -0,0 +1,59 @@ +schema_version: "1.0" +artifact: "artifacts/shell/gt-sh-002-missing-guards.sh" +artifact_sha256: "b4b5c028a224532e106f82406499c8adcfd2603702012360217d173d665436c1" + +expected_verdict: REVISE + +findings: + - id: GT-056 + description: "Missing 'set -euo pipefail' — script does not enable strict error mode, so failures in pg_dump, gzip, aws s3 sync, or find are silently ignored and execution continues" + location: "line 1" + severity: HIGH + category: completeness + critic: completeness + rubric_criterion: null + notes: > + Without 'set -e', a failed pg_dump (e.g., connection refused) will produce an empty/corrupt + gzip file that is then uploaded to S3 without any error signal. Without 'set -u', any + undefined variable is silently expanded to empty string. Without 'pipefail', pg_dump | gzip + failure masks the error code. The script has no explicit error checking on any command output. + + - id: GT-057 + description: "rm -rf executed on $OLD_DIR without checking if the variable is non-empty — an empty or whitespace value expands to 'rm -rf', destroying the filesystem root" + location: "line 46" + severity: HIGH + category: security + critic: security + rubric_criterion: null + notes: > + OLD_DIR is assigned via 'find ... | head -1', which produces empty output when no directories + match. The subsequent 'rm -rf $OLD_DIR' (unquoted) then reduces to 'rm -rf', which with + sufficient privileges will begin recursive deletion from the current working directory or + interpreted as argument-free invocation depending on shell. Must guard: [[ -n "$OLD_DIR" ]] + and verify path is under /mnt/backups before deleting. Variable is also unquoted, enabling + word-splitting on paths with spaces. + + - id: GT-058 + description: "No lock file or mutex to prevent concurrent runs — if cron triggers a second instance while the first is running, two simultaneous pg_dump + s3 sync operations corrupt the backup directory" + location: "line 1" + severity: MEDIUM + category: completeness + critic: completeness + rubric_criterion: null + notes: > + A nightly backup script is the canonical case for needing a lock file. If the backup job + takes longer than 24 hours (large DB, slow network), cron will start a second instance. + Both will write to the same BACKUP_DIR (same date key), the same DUMP_FILE name collision + is possible, and S3 sync will interleave partial uploads. Standard mitigation: flock or + a PID lock file at script start, removed on EXIT trap. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: shell-script + complexity: medium + rubric: shell-script + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-sh-003-clean-deploy.annotations.yaml b/golden-test-set/annotations/gt-sh-003-clean-deploy.annotations.yaml new file mode 100644 index 0000000..2719480 --- /dev/null +++ b/golden-test-set/annotations/gt-sh-003-clean-deploy.annotations.yaml @@ -0,0 +1,35 @@ +schema_version: "1.0" +artifact: "artifacts/shell/gt-sh-003-clean-deploy.sh" +artifact_sha256: "f99ff3bffc3a8ce8253e8da83887135fcbdd862e5204e10c48c3a74bdadcfa1c" + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: "rm -rf \"${old_dir}\" on line ~145 — looks like unguarded deletion" + location: "line 145" + notes: > + This rm -rf is preceded by three independent guards: (1) the path is only entered if + RELEASE_PARENT is a valid directory AND starts with DEPLOY_BASE/ (allowlist prefix check); + (2) old_dir is populated via mapfile from a find command bounded by maxdepth/mindepth and + constrained to RELEASE_PARENT; (3) a per-iteration guard confirms old_dir is non-empty + AND still prefixed by RELEASE_PARENT/ before deletion. The variable is also double-quoted. + This is the correct pattern for safe parameterised rm -rf. Not a finding. + + - description: "No explicit check that $VERSION is not 'latest' before the semver regex" + location: "line 56" + notes: > + The semver regex '^[0-9]+\.[0-9]+\.[0-9]+...$' already rejects 'latest' because 'latest' + contains no digits in the required positions. The die() path fires immediately. The script + header also documents that 'latest is not accepted'. Defense-in-depth is adequate here; + a separate string comparison would be redundant, not required. Not a finding. + +metadata: + source: synthetic + domain: shell-script + complexity: medium + rubric: shell-script + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-xa-001-spec-impl-mismatch.annotations.yaml b/golden-test-set/annotations/gt-xa-001-spec-impl-mismatch.annotations.yaml new file mode 100644 index 0000000..35895a0 --- /dev/null +++ b/golden-test-set/annotations/gt-xa-001-spec-impl-mismatch.annotations.yaml @@ -0,0 +1,49 @@ +schema_version: "1.0" +artifact: "artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/" +artifact_sha256: "6c108fb2524b0d376988a679369b0fea7a3744193a373f531f87078d8cdf043a" +# Note: sha256 is of the primary document (spec.md). implementation.py sha256: 9e6d4fbe7e782ed6f0795957975f47732d7b3ec142eb12ffeb4ca5d15ea4a922 + +expected_verdict: REJECT + +findings: + - id: GT-059 + description: "Endpoint DELETE /api/users/{id} is defined in spec.md (section 5) but has no corresponding route in implementation.py — the endpoint is entirely absent" + location: "spec.md §5 / implementation.py (absent)" + severity: CRITICAL + category: cross_consistency + critic: cross_consistency + rubric_criterion: null + notes: > + The spec defines 5 endpoints (GET /api/users, GET /api/users/{id}, POST /api/users, + PUT /api/users/{id}, DELETE /api/users/{id}) plus GET /api/admin/stats. + implementation.py contains routes for all except DELETE /api/users/{id}. + This is a complete omission of a spec-required endpoint, not a partial mismatch. + The spec explicitly marks this endpoint as added in v2.1 with an audit requirement, + making the omission both a cross-consistency failure and a completeness gap. + + - id: GT-060 + description: "Spec mandates JWT Bearer auth on GET /api/admin/stats; implementation uses HTTP Basic authentication via require_basic_auth decorator — auth scheme mismatch" + location: "spec.md §6 (Authentication table + §GET /api/admin/stats) / implementation.py line 133" + severity: HIGH + category: cross_consistency + critic: cross_consistency + rubric_criterion: null + notes: > + The spec Authentication section explicitly states: 'Basic authentication is explicitly + prohibited on all endpoints.' Admin endpoints require JWT with role:admin claim validation. + The implementation defines a require_basic_auth() decorator that uses HTTP Basic auth + (base64-decoded username:password) and applies it to the /api/admin/stats route instead + of a JWT admin-scope validator. This is a direct contradiction of the spec's auth policy + and constitutes a security regression — Basic auth over the wire (without guaranteed TLS + enforcement at the route level) is substantially weaker than JWT with claim validation. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: cross-artifact + complexity: high + rubric: cross-artifact + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-xa-002-config-code-mismatch.annotations.yaml b/golden-test-set/annotations/gt-xa-002-config-code-mismatch.annotations.yaml new file mode 100644 index 0000000..7e743d8 --- /dev/null +++ b/golden-test-set/annotations/gt-xa-002-config-code-mismatch.annotations.yaml @@ -0,0 +1,49 @@ +schema_version: "1.0" +artifact: "artifacts/cross-artifact/gt-xa-002-config-code-mismatch/" +artifact_sha256: "d794e22697eb26844cfa1c24e82dd103adc3c9bd7d4dcde84492cece4f752e5f" +# Note: sha256 is of the primary document (config.yaml). feature_flags.py sha256: 8236e964064bb6ce14c68e2f6fe7f75b422511603aa157e311a3451142c2d705 + +expected_verdict: REVISE + +findings: + - id: GT-061 + description: "Code references feature flag 'enable_dark_mode' in _EXPECTED_FLAGS and as a typed property, but this flag is not defined in config.yaml — any deployment will warn on startup and silently default to False" + location: "config.yaml (absent) / feature_flags.py line 36 and line 107" + severity: HIGH + category: cross_consistency + critic: cross_consistency + rubric_criterion: null + notes: > + _EXPECTED_FLAGS in feature_flags.py includes 'enable_dark_mode': bool at line 36. + The typed property enable_dark_mode is defined at line 107. config.yaml defines 6 flags + (enable_new_dashboard, enable_inline_comments, enable_response_cache, cache_ttl_seconds, + max_retries, enable_ai_suggestions) — enable_dark_mode is not among them. + At runtime, the loader logs a warning and defaults to False (bool()), which masks the + inconsistency. The PLAT-4821 comment in the code confirms this is a known gap, not an + intentional design — the config update was not completed before the code shipped. + + - id: GT-062 + description: "Config defines max_retries as type integer; code explicitly casts it to str before storing, so flags.max_retries returns a string — type contract between config and code is broken" + location: "config.yaml (max_retries type: integer) / feature_flags.py lines 64-70 and line 99" + severity: HIGH + category: cross_consistency + critic: cross_consistency + rubric_criterion: null + notes: > + config.yaml declares max_retries with type: integer. The _EXPECTED_FLAGS dict maps it to + int (correct). However, the special-case branch at lines 64-70 deliberately converts the + value to str('raw_value') before storing, and the max_retries property at line 99 returns + str. Any caller that does integer arithmetic (e.g., 'for i in range(flags.max_retries)') + will get a TypeError at runtime. The comment 'legacy HTTP client expects string config' is + not reflected in the config schema and creates an undocumented, type-unsafe contract. + +false_positive_traps: [] + +metadata: + source: synthetic + domain: cross-artifact + complexity: medium + rubric: cross-artifact + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-xa-003-readme-code-mismatch.annotations.yaml b/golden-test-set/annotations/gt-xa-003-readme-code-mismatch.annotations.yaml new file mode 100644 index 0000000..fc6aedd --- /dev/null +++ b/golden-test-set/annotations/gt-xa-003-readme-code-mismatch.annotations.yaml @@ -0,0 +1,49 @@ +schema_version: "1.0" +artifact: "artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/" +artifact_sha256: "dd59923f80cc7fa4f1910706b50d0d6e059013afa1c65a43c04d879e1e26134a" +# Note: sha256 is of the primary document (README.md). cli.py sha256: 3a222d6c6ef36aef987fb0d839915a6a58d3886ca25e2153ae0dc3873bc87a54 + +expected_verdict: REVISE + +findings: + - id: GT-063 + description: "README documents a '--verbose' flag for all sub-commands; cli.py implements '--debug' — flag name mismatch means the documented flag does not exist and vice versa" + location: "README.md (Global flags table, line ~35; run example line ~73; serve example line ~92) / cli.py --debug option (lines 51, 79, 101)" + severity: HIGH + category: cross_consistency + critic: cross_consistency + rubric_criterion: null + notes: > + README.md's Global flags table lists '--verbose | flag | off | Enable verbose logging + (DEBUG level)'. The run, serve, and validate usage blocks all show '--verbose' in examples. + cli.py's Click options are named '--debug' (is_flag=True) across all three sub-commands. + There is no '--verbose' option anywhere in cli.py. A user following the README will + receive 'Error: No such option: --verbose'. The underlying behavior (enabling DEBUG logging) + is present in the code — only the flag name differs, making this a documentation/code + sync failure rather than a missing feature. + + - id: GT-064 + description: "README states the default port for 'datapipe serve' is 8080; cli.py defaults the --port option to 3000 — users following the README will connect to the wrong port" + location: "README.md line ~47 ('Start the HTTP listener on the default port (8080)') / cli.py line 79 (--port default=3000)" + severity: MEDIUM + category: cross_consistency + critic: cross_consistency + rubric_criterion: null + notes: > + README.md states 'Start the HTTP listener on the default port (8080)' and the + Troubleshooting section reinforces this: 'Another process is using port 8080.' + The config.yaml example also shows serve.port: 8080. However, the Click option in + cli.py defines default=3000. The authoritative value is the code; the README is wrong + by 1080 ports. Users running 'datapipe serve' and then connecting to localhost:8080 + will get connection refused. + +false_positive_traps: [] + +metadata: + source: modified-natural + domain: cross-artifact + complexity: medium + rubric: cross-artifact + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/annotations/gt-xa-004-consistent-pair.annotations.yaml b/golden-test-set/annotations/gt-xa-004-consistent-pair.annotations.yaml new file mode 100644 index 0000000..4c773f7 --- /dev/null +++ b/golden-test-set/annotations/gt-xa-004-consistent-pair.annotations.yaml @@ -0,0 +1,45 @@ +schema_version: "1.0" +artifact: "artifacts/cross-artifact/gt-xa-004-consistent-pair/" +artifact_sha256: "0708462a3da3720a01cfe3d85c1835b186af7123d0b8bc923b892908dda878a0" +# Note: sha256 is of the primary document (spec.md). module.py sha256: 5aef66dad925301419325beef023ea13c7177293b3affa36ca8ee1d7279852f4 + +expected_verdict: PASS + +findings: [] + +false_positive_traps: + - description: "module.py adds an optional 'clock' parameter not listed in the spec's interface table" + location: "spec.md §Interface / module.py __init__ line 38" + notes: > + The spec's Advisory Notes section explicitly states: 'The implementation should support + subclassing for testing purposes (e.g., injecting a mock clock). This is a design + recommendation, not a hard requirement.' The clock parameter directly fulfils this + advisory. Adding a parameter that satisfies an advisory recommendation is not a + cross-consistency violation — it is compliant extension. The constructor signature is + backward-compatible (clock has a default). Not a finding. + + - description: "Spec says consume() 'should' log a WARNING on rejection; module.py logs the warning outside the lock after returning False" + location: "spec.md §Advisory Notes / module.py lines 72-77" + notes: > + The WARNING log in module.py fires after the lock is released (the return False is inside + the with block, but the logger.warning call is outside it — the log fires when consume() + returns False). This is a minor structural quirk but not a bug: the spec's advisory says + 'should log a WARNING when a consume() call is rejected', which is satisfied. The log + message includes available, capacity, and refill_rate context as recommended. The fact + that logging occurs outside the lock is irrelevant to the spec requirement. Not a finding. + + - description: "module.py defines __repr__ which is not in the spec interface" + location: "module.py line 107" + notes: > + __repr__ is a Python convention for debug representation and is not part of the public + interface defined in spec.md. Adding __repr__ is additive and does not contradict any + spec requirement. Not a cross-consistency gap. + +metadata: + source: natural + domain: cross-artifact + complexity: high + rubric: cross-artifact + depth: standard + author: akkari + created: "2026-03-12" diff --git a/golden-test-set/artifacts/config/gt-cfg-001-exposed-secrets.yaml b/golden-test-set/artifacts/config/gt-cfg-001-exposed-secrets.yaml new file mode 100644 index 0000000..04d0de6 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-001-exposed-secrets.yaml @@ -0,0 +1,85 @@ +# CI/CD Pipeline Configuration — Nightly Build & Deploy +# Project: payments-service +# Last updated: 2026-01-14 +# Owner: platform-eng@corp.example.com + +pipeline: + name: payments-service-nightly + project: payments-service + version: "2.3" + schedule: "0 2 * * *" # 02:00 UTC nightly + +triggers: + - type: schedule + - type: manual + +stages: + - name: build + image: node:20-alpine + timeout_minutes: 15 + environment: + NODE_ENV: production + NPM_TOKEN: npm_pK9vXhQzL3mR7wT2yF8aD1sN4uC6bE0j + SENTRY_AUTH_TOKEN: sntrys_eyJpYXQiOjE3MDQ2NzIwMDAsInVybCI6Imh0dHBzOi8vc2VudHJ5LmlvIiwicmVnaW9uX3VybCI6Imh0dHBzOi8vdXMuc2VudHJ5LmlvIiwib3JnIjoicGF5bWVudHMifQ_abc123xyz + commands: + - npm ci --prefer-offline + - npm run lint + - npm run build + + - name: test + image: node:20-alpine + timeout_minutes: 30 + environment: + NODE_ENV: test + DATABASE_URL: postgres://payments_user:Tr0ub4dor&3@db.internal.example.com:5432/payments_test + REDIS_URL: redis://:SuperSecretRedisPassword123!@cache.internal.example.com:6379/0 + JWT_SECRET: hs256-jwt-signing-key-do-not-share-c7f4b2a1e9d3 + STRIPE_SECRET_KEY: sk_test_FAKE_51HbNjKL8eVQ2YHrM9xT3wP6aF2sRnKo + commands: + - npm run test:unit + - npm run test:integration + - npm run test:coverage + + - name: publish + image: docker:24-dind + timeout_minutes: 20 + environment: + DOCKER_REGISTRY: registry.example.com + DOCKER_USER: ci-bot + DOCKER_PASSWORD: P@ssw0rd!Registry#2026 + AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE + AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + AWS_DEFAULT_REGION: us-east-1 + commands: + - docker build -t registry.example.com/payments-service:${BUILD_NUMBER} . + - docker push registry.example.com/payments-service:${BUILD_NUMBER} + + - name: deploy + image: bitnami/kubectl:1.29 + timeout_minutes: 10 + environment: + KUBECONFIG_DATA: eyJhcGlWZXJzaW9uIjoidjEiLCJraW5kIjoiQ29uZmlnIn0= + commands: + - kubectl set image deployment/payments-service app=registry.example.com/payments-service:${BUILD_NUMBER} + - kubectl rollout status deployment/payments-service --timeout=5m + +notifications: + on_failure: + slack: + webhook: https://hooks.example.com/services/T00000000/B00000000/FAKE_WEBHOOK_TOKEN + channel: "#platform-alerts" + on_success: + slack: + webhook: https://hooks.example.com/services/T00000000/B00000000/FAKE_WEBHOOK_TOKEN + channel: "#deployments" + +artifacts: + paths: + - dist/ + - coverage/ + expire_in: 30 days + +cache: + key: "${CI_COMMIT_REF_SLUG}-node-modules" + paths: + - node_modules/ diff --git a/golden-test-set/artifacts/config/gt-cfg-002-insecure-permissions.yaml b/golden-test-set/artifacts/config/gt-cfg-002-insecure-permissions.yaml new file mode 100644 index 0000000..2053d13 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-002-insecure-permissions.yaml @@ -0,0 +1,138 @@ +# Kubernetes Deployment Manifest — payments-worker +# Cluster: prod-us-east-1 +# Namespace: payments +# Last updated: 2026-02-03 +# Ticket: INFRA-2041 — scale out payment worker fleet + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payments-worker + namespace: payments + labels: + app: payments-worker + version: "4.1.2" + team: platform + annotations: + deployment.kubernetes.io/revision: "7" + kubernetes.io/change-cause: "Bump image to 4.1.2, increase replicas" +spec: + replicas: 6 + selector: + matchLabels: + app: payments-worker + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 2 + maxUnavailable: 1 + template: + metadata: + labels: + app: payments-worker + version: "4.1.2" + spec: + # WARNING: hostNetwork required for legacy payment processor SDK + # which binds to fixed port 9742 on the host interface. + # TODO: migrate to sidecar proxy pattern (INFRA-2089) + hostNetwork: true + hostPID: false + + serviceAccountName: payments-worker-sa + + containers: + - name: payments-worker + image: registry.example.com/payments-worker:4.1.2 + imagePullPolicy: Always + + securityContext: + privileged: true + allowPrivilegeEscalation: true + runAsUser: 0 + runAsGroup: 0 + # readOnlyRootFilesystem intentionally false — worker writes temp files to /tmp + + # No resources block — scaling handled by HPA based on queue depth + # TODO: add resource requests/limits (INFRA-2090) + + ports: + - containerPort: 8080 + name: http + - containerPort: 9742 + name: legacy-sdk + + env: + - name: QUEUE_URL + valueFrom: + secretKeyRef: + name: payments-worker-secrets + key: queue_url + - name: DB_DSN + valueFrom: + secretKeyRef: + name: payments-worker-secrets + key: db_dsn + - name: WORKER_CONCURRENCY + value: "8" + - name: LOG_LEVEL + value: "info" + + volumeMounts: + - name: tmp-scratch + mountPath: /tmp + - name: config + mountPath: /app/config + readOnly: true + + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 20 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /readyz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + + volumes: + - name: tmp-scratch + emptyDir: {} + - name: config + configMap: + name: payments-worker-config + + imagePullSecrets: + - name: registry-credentials + + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - payments-worker + topologyKey: kubernetes.io/hostname + +--- +apiVersion: v1 +kind: Service +metadata: + name: payments-worker + namespace: payments +spec: + selector: + app: payments-worker + ports: + - port: 8080 + targetPort: 8080 + name: http + type: ClusterIP diff --git a/golden-test-set/artifacts/config/gt-cfg-003-stale-references.yaml b/golden-test-set/artifacts/config/gt-cfg-003-stale-references.yaml new file mode 100644 index 0000000..3b9e0f8 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-003-stale-references.yaml @@ -0,0 +1,108 @@ +# Data Ingestion Pipeline Configuration +# Service: analytics-ingest +# Environment: production +# Migrated from: legacy Airflow 1.x to Prefect 2.x — 2025-09-15 +# Note: migration was partial; some config blocks not yet updated + +pipeline: + name: analytics-ingest + description: "Ingests click-stream events from Kafka, enriches, writes to BigQuery" + schedule: "*/5 * * * *" + timeout_seconds: 240 + max_retries: 3 + retry_delay_seconds: 30 + +source: + type: kafka + brokers: + - kafka-1.internal.example.com:9092 + - kafka-2.internal.example.com:9092 + - kafka-3.internal.example.com:9092 + topics: + - clickstream.raw + - clickstream.mobile + consumer_group: analytics-ingest-prod + # API version pinned during migration — review before next upgrade cycle + api_version: v1beta1 + schema_registry: https://schema-registry.internal.example.com + offset: earliest + max_poll_records: 500 + +transforms: + - name: deduplicate + type: dedup + window_seconds: 60 + key_fields: + - event_id + - user_id + + - name: enrich_geo + type: ip_lookup + provider: maxmind + database: /data/GeoIP2-City.mmdb + fallback: unknown + + - name: filter_bots + type: filter + # experimental_mode enables ML-based bot detection (added in v0.9, removed in v1.2) + # toggle kept here because rollback to v0.9 is theoretically possible + experimental_mode: true + rules: + - field: user_agent + pattern: "(bot|crawler|spider|scraper)" + action: drop + - field: request_rate_per_minute + threshold: 600 + action: drop + + - name: normalize_timestamps + type: timestamp + input_format: unix_ms + output_format: ISO8601 + timezone: UTC + +sink: + type: bigquery + project: example-analytics-prod + dataset: clickstream + table: events_enriched + # bigquery-python client v2 removed write_disposition in favor of job_config + # this field is ignored silently but kept for documentation purposes + write_disposition: WRITE_APPEND + create_disposition: CREATE_IF_NEEDED + batch_size: 1000 + batch_timeout_seconds: 30 + schema_auto_detect: false + schema_file: /app/schemas/events_enriched.json + + # Dead-letter queue for malformed records + dlq: + type: gcs + bucket: example-analytics-dlq + prefix: ingest/failed/ + retention_days: 90 + +monitoring: + metrics_backend: prometheus + push_gateway: https://pushgw.internal.example.com:9091 + labels: + team: data-eng + environment: production + service: analytics-ingest + + alerts: + # References Alertmanager webhook — route "data-eng-pagerduty" was renamed + # to "data-eng-oncall" in the Oct 2025 Alertmanager migration (INFRA-1887) + # This route now 404s silently; alerts are not firing + webhook: https://alertmanager.internal.example.com/api/v1beta1/alerts + route: data-eng-pagerduty + on_failure: true + on_lag_threshold_seconds: 120 + +feature_flags: + # v1beta1 feature flag API deprecated; use v2 flag service at flags.internal.example.com + endpoint: https://flags.internal.example.com/api/v1beta1/evaluate + flags: + enable_geo_enrichment: true + enable_bot_filter: true + enable_schema_validation: false diff --git a/golden-test-set/artifacts/config/gt-cfg-004-missing-validation.json b/golden-test-set/artifacts/config/gt-cfg-004-missing-validation.json new file mode 100644 index 0000000..19f6943 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-004-missing-validation.json @@ -0,0 +1,119 @@ +{ + "_comment": "API Gateway Configuration — internal-api-gw, environment: production", + "_version": "3.7.1", + "_last_updated": "2026-01-22", + "_owner": "backend-platform@corp.example.com", + + "gateway": { + "name": "internal-api-gw", + "listen": ":8443", + "tls": { + "enabled": true, + "cert_file": "/etc/tls/server.crt", + "key_file": "/etc/tls/server.key", + "min_version": "TLS1.2" + }, + "access_log": { + "enabled": true, + "format": "json", + "path": "/var/log/api-gw/access.log" + } + }, + + "routes": [ + { + "id": "route-users-api", + "path": "/api/v2/users", + "methods": ["GET", "POST", "PUT", "DELETE"], + "upstream": "http://users-service.internal:3000", + "timeout_ms": 5000, + "auth": { + "required": true, + "type": "bearer_jwt", + "jwks_url": "https://auth.example.com/.well-known/jwks.json", + "audience": "internal-api" + } + }, + { + "id": "route-payments-api", + "path": "/api/v2/payments", + "methods": ["GET", "POST"], + "upstream": "http://payments-service.internal:4000", + "timeout_ms": 10000, + "auth": { + "required": true, + "type": "bearer_jwt", + "jwks_url": "https://auth.example.com/.well-known/jwks.json", + "audience": "internal-api" + } + }, + { + "id": "route-reports-api", + "path": "/api/v2/reports", + "methods": ["GET"], + "upstream": "http://reports-service.internal:5000", + "timeout_ms": 30000, + "auth": { + "required": true, + "type": "bearer_jwt", + "jwks_url": "https://auth.example.com/.well-known/jwks.json", + "audience": "internal-api" + } + }, + { + "id": "route-admin", + "path": "/admin", + "methods": ["GET", "POST", "PUT", "DELETE", "PATCH"], + "upstream": "http://admin-service.internal:9000", + "timeout_ms": 30000, + "description": "Admin control panel — internal use only, assumed safe behind VPN" + }, + { + "id": "route-webhooks", + "path": "/webhooks", + "methods": ["POST"], + "upstream": "http://webhook-processor.internal:6000", + "timeout_ms": 5000, + "auth": { + "required": false, + "note": "Webhooks validated by HMAC signature in application layer" + } + } + ], + + "middleware": [ + { + "name": "cors", + "enabled": true, + "allowed_origins": ["https://app.example.com", "https://admin.example.com"], + "allowed_methods": ["GET", "POST", "PUT", "DELETE", "OPTIONS"], + "allow_credentials": true + }, + { + "name": "compression", + "enabled": true, + "algorithm": "gzip", + "min_size_bytes": 1024 + }, + { + "name": "request_id", + "enabled": true, + "header": "X-Request-ID" + } + ], + + "upstream_defaults": { + "connect_timeout_ms": 1000, + "read_timeout_ms": 5000, + "max_idle_conns_per_host": 50, + "retry": { + "attempts": 2, + "on_status": [502, 503, 504] + } + }, + + "health": { + "path": "/healthz", + "interval_seconds": 30 + } +} diff --git a/golden-test-set/artifacts/config/gt-cfg-005-contradictory.yaml b/golden-test-set/artifacts/config/gt-cfg-005-contradictory.yaml new file mode 100644 index 0000000..7129c49 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-005-contradictory.yaml @@ -0,0 +1,116 @@ +# Microservice Runtime Configuration — order-processor +# Environment: production +# Deployed via: Helm chart 2.9.1 +# Config last modified: 2026-02-28 +# Ticket: OPS-3312 — increase throughput, tune retry policy + +service: + name: order-processor + version: "7.2.0" + profile: production + instance_id: "${POD_NAME}" + region: us-west-2 + +server: + port: 8080 + grpc_port: 9090 + # Maximum time the server will wait for a single request to complete + # before returning a deadline-exceeded error to the caller + max_timeout_seconds: 45 + read_timeout_seconds: 30 + write_timeout_seconds: 30 + idle_timeout_seconds: 120 + max_header_bytes: 8192 + max_connections: 0 # 0 = unlimited; connection count governed by k8s resource limits + +database: + driver: postgres + host: orders-db.internal.example.com + port: 5432 + name: orders_prod + user_secret: vault:secret/data/order-processor/db#username + pass_secret: vault:secret/data/order-processor/db#password + max_open_conns: 40 + max_idle_conns: 10 + conn_max_lifetime_seconds: 300 + conn_max_idle_time_seconds: 60 + +messaging: + broker: kafka + brokers: + - kafka-1.internal.example.com:9092 + - kafka-2.internal.example.com:9092 + - kafka-3.internal.example.com:9092 + consumer_group: order-processor-prod + topics: + inbound: orders.submitted + outbound: orders.processed + dlq: orders.failed + +retry: + # Retry policy for downstream calls (payment-service, inventory-service) + # Per-attempt timeout governs each individual attempt + # Total retry budget = retry_count * per_retry_timeout_seconds + retry_count: 5 + per_retry_timeout_seconds: 12 + # NOTE: total retry budget = 5 * 12 = 60s, which EXCEEDS max_timeout_seconds (45s) + # In practice the circuit will hit the global timeout at ~45s and abort mid-retry. + # The last 1-2 retry attempts will never execute. Tracked in OPS-3315. + backoff_strategy: exponential + backoff_base_seconds: 1 + backoff_max_seconds: 10 + jitter: true + +circuit_breaker: + enabled: true + failure_threshold: 5 + success_threshold: 2 + timeout_seconds: 30 + half_open_max_calls: 3 + +cache: + type: redis + host_secret: vault:secret/data/order-processor/redis#host + port: 6379 + db: 0 + ttl_seconds: 300 + max_retries: 3 + pool_size: 20 + +observability: + # Log level set to DEBUG — captures full request/response bodies including PII + # Left from debugging OPS-3298; not reverted before deploy + log_level: debug + log_format: json + log_output: stdout + + tracing: + enabled: true + exporter: otlp + endpoint: https://otel-collector.internal.example.com:4318 + sample_rate: 0.05 + + metrics: + enabled: true + exporter: prometheus + path: /metrics + port: 9090 + +feature_flags: + service: https://flags.internal.example.com + cache_ttl_seconds: 30 + flags: + enable_async_fulfillment: true + enable_fraud_scoring: true + enable_split_shipment: false + +rate_limiting: + enabled: true + requests_per_second: 500 + burst: 100 + strategy: token_bucket + +worker_pool: + size: 20 + queue_depth: 500 + drain_timeout_seconds: 60 diff --git a/golden-test-set/artifacts/config/gt-cfg-006-minor-issues.yaml b/golden-test-set/artifacts/config/gt-cfg-006-minor-issues.yaml new file mode 100644 index 0000000..553db36 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-006-minor-issues.yaml @@ -0,0 +1,76 @@ +# Feature Flag Service Configuration +# Service: flagd +# Environment: production +# Owner: platform-eng@corp.example.com +# Last modified: 2026-03-01 + +service: + name: flagd + version: "1.4.0" + listen: ":8013" + metrics_port: 2112 + +providers: + - type: file + path: /etc/flagd/flags.json + poll_interval_seconds: 30 + watch: true + + - type: grpc + host: flag-management.internal.example.com + port: 8014 + tls: true + keepalive_time_seconds: 30 + keepalive_timeout_seconds: 10 + sync_interval_seconds: 60 + +cache: + type: lru + size: 10000 + ttl_seconds: 300 + +auth: + enabled: true + type: mtls + ca_cert: /etc/tls/ca.crt + server_cert: /etc/tls/server.crt + server_key: /etc/tls/server.key + +cors: + enabled: false + +logging: + level: info + format: json + output: stdout + +health: + liveness_path: /healthz + readiness_path: /readyz + port: 8015 + +telemetry: + enabled: true + exporter: otlp + endpoint: https://otel-collector.internal.example.com:4318 + sample_rate: 1.0 + +evaluation: + default_provider: grpc + fallback_provider: file + timeout_ms: 200 + max_retries: 2 + +defaults: + resolution_reason: STATIC + targeting_key: user_id + emit_events: true + event_buffer_size: 512 + shutdown_grace_period_seconds: 15 + +tls: + min_version: TLS1.3 + cipher_suites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 diff --git a/golden-test-set/artifacts/config/gt-cfg-007-clean-k8s.yaml b/golden-test-set/artifacts/config/gt-cfg-007-clean-k8s.yaml new file mode 100644 index 0000000..a7f9b01 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-007-clean-k8s.yaml @@ -0,0 +1,216 @@ +# Kubernetes Deployment Manifest — notification-service +# Cluster: prod-us-east-1 +# Namespace: notifications +# Last updated: 2026-03-05 +# Reviewed by: security-review@corp.example.com (2026-03-04, ticket SEC-0892) +# Change: Promote to 3.0.1 after full security review sign-off + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: notification-service + namespace: notifications + labels: + app: notification-service + version: "3.0.1" + team: messaging + annotations: + deployment.kubernetes.io/revision: "12" + kubernetes.io/change-cause: "Promote 3.0.1 post security review SEC-0892" + security-review/ticket: "SEC-0892" + security-review/date: "2026-03-04" +spec: + replicas: 3 + selector: + matchLabels: + app: notification-service + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: notification-service + version: "3.0.1" + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: notification-service-sa + automountServiceAccountToken: false + + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + + containers: + - name: notification-service + image: registry.example.com/notification-service:3.0.1 + imagePullPolicy: Always + + securityContext: + allowPrivilegeEscalation: false + privileged: false + # readOnlyRootFilesystem intentionally set to false. + # The notification-service renders email templates to a temp directory + # at /tmp/rendered/ using Jinja2. Making the root FS read-only would + # require an additional emptyDir mount for /tmp, which adds operational + # complexity without meaningful security benefit given /tmp is already + # ephemeral and governed by the container's seccomp profile. + # Accepted risk documented in SEC-0892. + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + + ports: + - containerPort: 8080 + name: http + - containerPort: 9090 + name: metrics + + env: + - name: APP_ENV + value: production + - name: LOG_LEVEL + value: info + - name: PORT + value: "8080" + - name: METRICS_PORT + value: "9090" + - name: SMTP_HOST + valueFrom: + configMapKeyRef: + name: notification-service-config + key: smtp_host + - name: SMTP_PORT + valueFrom: + configMapKeyRef: + name: notification-service-config + key: smtp_port + - name: SMTP_USER + valueFrom: + secretKeyRef: + name: notification-service-secrets + key: smtp_user + - name: SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: notification-service-secrets + key: smtp_password + - name: SENDGRID_API_KEY + valueFrom: + secretKeyRef: + name: notification-service-secrets + key: sendgrid_api_key + + volumeMounts: + - name: config + mountPath: /app/config + readOnly: true + - name: templates + mountPath: /app/templates + readOnly: true + + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 3 + + readinessProbe: + httpGet: + path: /readyz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 2 + + startupProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 12 + + volumes: + - name: config + configMap: + name: notification-service-config + - name: templates + configMap: + name: notification-service-templates + + imagePullSecrets: + - name: registry-credentials + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - notification-service + topologyKey: kubernetes.io/hostname + + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: notification-service + +--- +apiVersion: v1 +kind: Service +metadata: + name: notification-service + namespace: notifications + labels: + app: notification-service +spec: + selector: + app: notification-service + ports: + - port: 80 + targetPort: 8080 + name: http + - port: 9090 + targetPort: 9090 + name: metrics + type: ClusterIP + +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: notification-service-pdb + namespace: notifications +spec: + minAvailable: 2 + selector: + matchLabels: + app: notification-service diff --git a/golden-test-set/artifacts/config/gt-cfg-008-clean-pipeline.json b/golden-test-set/artifacts/config/gt-cfg-008-clean-pipeline.json new file mode 100644 index 0000000..a45a719 --- /dev/null +++ b/golden-test-set/artifacts/config/gt-cfg-008-clean-pipeline.json @@ -0,0 +1,146 @@ +{ + "_meta": { + "name": "identity-service-release", + "description": "Build, test, and release pipeline for identity-service", + "version": "4.1.0", + "owner": "identity-team@corp.example.com", + "last_updated": "2026-03-08", + "security_review": "SEC-0901 (2026-03-07)" + }, + + "pipeline": { + "name": "identity-service-release", + "default_branch": "main", + "protected_branches": ["main", "release/*"], + + "triggers": [ + { "type": "push", "branches": ["main", "release/*"] }, + { "type": "pull_request", "target_branches": ["main"] }, + { "type": "schedule", "cron": "0 6 * * 1-5", "timezone": "UTC" } + ], + + "concurrency": { + "group": "identity-service-${BRANCH_NAME}", + "cancel_in_progress": true + } + }, + + "vault": { + "address": "https://vault.internal.example.com", + "auth_method": "kubernetes", + "role": "ci-identity-service", + "namespace": "platform", + "secret_path": "secret/data/ci/identity-service" + }, + + "stages": [ + { + "name": "lint-and-type-check", + "image": "python:3.12-slim", + "timeout_minutes": 10, + "cache": { + "key": "pip-${hashFiles('requirements*.txt')}", + "paths": [".pip-cache/"] + }, + "env": { + "PYTHONDONTWRITEBYTECODE": "1", + "PIP_CACHE_DIR": ".pip-cache" + }, + "steps": [ + { "name": "install-dev-deps", "run": "pip install -r requirements-dev.txt --cache-dir .pip-cache" }, + { "name": "ruff-lint", "run": "ruff check src/ tests/" }, + { "name": "mypy", "run": "mypy src/ --strict --ignore-missing-imports" }, + { "name": "bandit-sast", "run": "bandit -r src/ -ll -q" } + ] + }, + { + "name": "unit-tests", + "image": "python:3.12-slim", + "timeout_minutes": 20, + "env": { + "PYTHONDONTWRITEBYTECODE": "1", + "PIP_CACHE_DIR": ".pip-cache", + "DATABASE_URL": { "vault": "secret/data/ci/identity-service#test_db_url" }, + "JWT_PRIVATE_KEY": { "vault": "secret/data/ci/identity-service#jwt_private_key_pem" } + }, + "steps": [ + { "name": "install-deps", "run": "pip install -r requirements.txt -r requirements-dev.txt --cache-dir .pip-cache" }, + { "name": "pytest", "run": "pytest tests/unit/ -v --tb=short --cov=src --cov-report=xml --cov-fail-under=85" } + ], + "artifacts": { + "paths": ["coverage.xml"], + "expire_in": "7 days" + } + }, + { + "name": "integration-tests", + "image": "python:3.12-slim", + "timeout_minutes": 30, + "services": [ + { "name": "postgres", "image": "postgres:16-alpine", "env": { "POSTGRES_DB": "identity_test", "POSTGRES_USER": "test", "POSTGRES_PASSWORD": { "vault": "secret/data/ci/identity-service#integration_db_password" } } }, + { "name": "redis", "image": "redis:7-alpine" } + ], + "env": { + "DATABASE_URL": "postgres://test:${INTEGRATION_DB_PASSWORD}@postgres:5432/identity_test", + "REDIS_URL": "redis://redis:6379/0", + "JWT_PRIVATE_KEY": { "vault": "secret/data/ci/identity-service#jwt_private_key_pem" }, + "INTEGRATION_DB_PASSWORD": { "vault": "secret/data/ci/identity-service#integration_db_password" } + }, + "steps": [ + { "name": "wait-for-services", "run": "python scripts/wait_for_services.py --timeout 30" }, + { "name": "db-migrate", "run": "alembic upgrade head" }, + { "name": "pytest-integration", "run": "pytest tests/integration/ -v --tb=short" } + ] + }, + { + "name": "build-image", + "image": "docker:25-dind", + "timeout_minutes": 20, + "only": ["main", "release/*"], + "env": { + "DOCKER_REGISTRY": "registry.example.com", + "REGISTRY_USER": { "vault": "secret/data/ci/identity-service#registry_user" }, + "REGISTRY_PASSWORD": { "vault": "secret/data/ci/identity-service#registry_password" } + }, + "steps": [ + { "name": "docker-login", "run": "echo \"${REGISTRY_PASSWORD}\" | docker login ${DOCKER_REGISTRY} -u ${REGISTRY_USER} --password-stdin" }, + { "name": "build", "run": "docker build --build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) --build-arg GIT_COMMIT=${CI_COMMIT_SHA} -t ${DOCKER_REGISTRY}/identity-service:${CI_COMMIT_SHA} -t ${DOCKER_REGISTRY}/identity-service:latest ." }, + { "name": "scan", "run": "trivy image --exit-code 1 --severity HIGH,CRITICAL ${DOCKER_REGISTRY}/identity-service:${CI_COMMIT_SHA}" }, + { "name": "push", "run": "docker push ${DOCKER_REGISTRY}/identity-service:${CI_COMMIT_SHA} && docker push ${DOCKER_REGISTRY}/identity-service:latest" } + ] + }, + { + "name": "release", + "image": "python:3.12-slim", + "timeout_minutes": 10, + "only": ["main"], + "when": "manual", + "env": { + "GITHUB_TOKEN": { "vault": "secret/data/ci/identity-service#github_token" }, + "PYPI_API_TOKEN": { "vault": "secret/data/ci/identity-service#pypi_token" } + }, + "steps": [ + { "name": "tag-release", "run": "python scripts/tag_release.py --version ${RELEASE_VERSION}" }, + { "name": "build-dist", "run": "python -m build --sdist --wheel" }, + { "name": "publish", "run": "twine upload --non-interactive dist/* --username __token__ --password ${PYPI_API_TOKEN}" }, + { "name": "create-gh-release", "run": "gh release create ${RELEASE_VERSION} --generate-notes --repo example-corp/identity-service" } + ] + } + ], + + "notifications": { + "on_failure": { + "slack": { + "webhook": { "vault": "secret/data/ci/identity-service#slack_webhook" }, + "channel": "#identity-team-alerts", + "mention": "@identity-oncall" + } + }, + "on_success": { + "slack": { + "webhook": { "vault": "secret/data/ci/identity-service#slack_webhook" }, + "channel": "#deployments" + } + } + } +} diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/implementation.py b/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/implementation.py new file mode 100644 index 0000000..1b9e0c6 --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/implementation.py @@ -0,0 +1,220 @@ +""" +user_mgmt_svc/api/routes.py — User Management API route handlers + +Implements the endpoints defined in spec.md v2.1. +Service: user-mgmt-svc +Author: backend-platform-eng +Last updated: 2026-02-14 +""" + +from __future__ import annotations + +import base64 +import logging +import uuid +from functools import wraps +from typing import Any + +from flask import Flask, jsonify, request, g +from werkzeug.exceptions import HTTPException + +from .db import db_session, User +from .rate_limiter import rate_limit +from .audit import audit_log + +logger = logging.getLogger(__name__) +app = Flask(__name__) + + +# ── Authentication helpers ──────────────────────────────────────────────────── + +def _decode_jwt(token: str) -> dict[str, Any] | None: + """Decode and verify a JWT Bearer token. Returns claims dict or None.""" + from .jwt_utils import verify_and_decode + try: + return verify_and_decode(token) + except Exception: + return None + + +def require_jwt(f): + """Decorator: require a valid JWT Bearer token.""" + @wraps(f) + def decorated(*args, **kwargs): + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + return jsonify({"error": {"code": "UNAUTHORIZED", "message": "Bearer token required", + "request_id": g.get("request_id", "")}}), 401 + token = auth_header[len("Bearer "):] + claims = _decode_jwt(token) + if claims is None: + return jsonify({"error": {"code": "UNAUTHORIZED", "message": "Invalid token", + "request_id": g.get("request_id", "")}}), 401 + g.jwt_claims = claims + return f(*args, **kwargs) + return decorated + + +def require_basic_auth(f): + """Decorator: require HTTP Basic authentication for admin endpoints.""" + @wraps(f) + def decorated(*args, **kwargs): + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Basic "): + return jsonify({"error": {"code": "UNAUTHORIZED", "message": "Basic auth required", + "request_id": g.get("request_id", "")}}), 401 + try: + decoded = base64.b64decode(auth_header[6:]).decode("utf-8") + username, password = decoded.split(":", 1) + except Exception: + return jsonify({"error": {"code": "UNAUTHORIZED", "message": "Malformed Basic auth", + "request_id": g.get("request_id", "")}}), 401 + from .admin_users import verify_admin_password + if not verify_admin_password(username, password): + return jsonify({"error": {"code": "FORBIDDEN", "message": "Invalid admin credentials", + "request_id": g.get("request_id", "")}}), 403 + g.admin_user = username + return f(*args, **kwargs) + return decorated + + +# ── Route: GET /api/users ───────────────────────────────────────────────────── + +@app.route("/api/users", methods=["GET"]) +@rate_limit(100, window=60) +@require_jwt +def list_users(): + page = max(1, request.args.get("page", 1, type=int)) + limit = min(100, max(1, request.args.get("limit", 20, type=int))) + filt = request.args.get("filter", "") + + with db_session() as session: + q = session.query(User) + if filt: + q = q.filter(User.username.ilike(f"%{filt}%")) + total = q.count() + users = q.offset((page - 1) * limit).limit(limit).all() + + return jsonify({ + "users": [u.to_dict() for u in users], + "total": total, + "page": page, + "limit": limit, + }), 200 + + +# ── Route: GET /api/users/ ──────────────────────────────────────────────── + +@app.route("/api/users/", methods=["GET"]) +@rate_limit(200, window=60) +@require_jwt +def get_user(user_id: str): + try: + uid = uuid.UUID(user_id) + except ValueError: + return jsonify({"error": {"code": "BAD_REQUEST", "message": "Invalid UUID", + "request_id": g.get("request_id", "")}}), 400 + + with db_session() as session: + user = session.query(User).filter_by(id=uid).first() + if user is None: + return jsonify({"error": {"code": "NOT_FOUND", "message": "User not found", + "request_id": g.get("request_id", "")}}), 404 + return jsonify(user.to_dict()), 200 + + +# ── Route: POST /api/users ──────────────────────────────────────────────────── + +@app.route("/api/users", methods=["POST"]) +@rate_limit(10, window=60) +@require_jwt +def create_user(): + data = request.get_json(force=True, silent=True) or {} + username = data.get("username", "").strip() + email = data.get("email", "").strip() + role = data.get("role", "").strip() + + if not username or not email or not role: + return jsonify({"error": {"code": "BAD_REQUEST", + "message": "username, email, and role are required", + "request_id": g.get("request_id", "")}}), 400 + + if role not in ("viewer", "editor", "admin"): + return jsonify({"error": {"code": "BAD_REQUEST", + "message": "role must be viewer, editor, or admin", + "request_id": g.get("request_id", "")}}), 400 + + with db_session() as session: + if session.query(User).filter( + (User.username == username) | (User.email == email) + ).first(): + return jsonify({"error": {"code": "CONFLICT", + "message": "Username or email already exists", + "request_id": g.get("request_id", "")}}), 409 + + new_user = User(id=uuid.uuid4(), username=username, email=email, role=role) + session.add(new_user) + session.commit() + return jsonify(new_user.to_dict()), 201 + + +# ── Route: PUT /api/users/ ──────────────────────────────────────────────── + +@app.route("/api/users/", methods=["PUT"]) +@rate_limit(20, window=60) +@require_jwt +def update_user(user_id: str): + try: + uid = uuid.UUID(user_id) + except ValueError: + return jsonify({"error": {"code": "BAD_REQUEST", "message": "Invalid UUID", + "request_id": g.get("request_id", "")}}), 400 + + data = request.get_json(force=True, silent=True) or {} + + with db_session() as session: + user = session.query(User).filter_by(id=uid).first() + if user is None: + return jsonify({"error": {"code": "NOT_FOUND", "message": "User not found", + "request_id": g.get("request_id", "")}}), 404 + + if "username" in data: + user.username = data["username"].strip() + if "email" in data: + user.email = data["email"].strip() + if "role" in data: + if data["role"] not in ("viewer", "editor", "admin"): + return jsonify({"error": {"code": "BAD_REQUEST", "message": "Invalid role", + "request_id": g.get("request_id", "")}}), 400 + user.role = data["role"] + + session.commit() + return jsonify(user.to_dict()), 200 + + +# ── Route: GET /api/admin/stats ─────────────────────────────────────────────── + +@app.route("/api/admin/stats", methods=["GET"]) +@rate_limit(10, window=60) +@require_basic_auth +def admin_stats(): + from .analytics import get_user_stats + stats = get_user_stats() + return jsonify(stats), 200 + + +# ── Error handlers ──────────────────────────────────────────────────────────── + +@app.errorhandler(HTTPException) +def handle_http_exception(e: HTTPException): + return jsonify({"error": {"code": e.name.upper().replace(" ", "_"), + "message": e.description, + "request_id": g.get("request_id", "")}}), e.code + + +@app.errorhandler(Exception) +def handle_unexpected(e: Exception): + logger.exception("Unhandled exception: %s", e) + return jsonify({"error": {"code": "INTERNAL_SERVER_ERROR", + "message": "An unexpected error occurred.", + "request_id": g.get("request_id", "")}}), 500 diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/quorum-relationships.yaml b/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/quorum-relationships.yaml new file mode 100644 index 0000000..d9d1c90 --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/quorum-relationships.yaml @@ -0,0 +1,5 @@ +relationships: + - source: spec.md + target: implementation.py + type: specification + description: "spec.md defines the User Management API contract (endpoints, auth, rate limits); implementation.py implements those endpoints in Flask" diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/spec.md b/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/spec.md new file mode 100644 index 0000000..f1362b3 --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/spec.md @@ -0,0 +1,189 @@ +# User Management API — Specification v2.1 + +**Service:** user-mgmt-svc +**Owner:** platform-eng@company.internal +**Last revised:** 2026-01-20 +**Status:** APPROVED — implementation target Q1-2026 + +--- + +## Overview + +The User Management API provides CRUD operations for user accounts, role assignments, +and administrative actions. All endpoints are served at `/api/v2`. + +--- + +## Authentication + +All endpoints require authentication. The authentication scheme varies by security tier: + +| Security Tier | Scheme | Header | +|---------------|--------|--------| +| Standard | JWT Bearer token | `Authorization: Bearer ` | +| Admin | JWT Bearer token (admin scope required) | `Authorization: Bearer ` | + +Tokens are issued by the internal IdP at `https://auth.company.internal/token`. +**Admin endpoints MUST validate the `role:admin` claim in the JWT payload.** +Basic authentication is explicitly prohibited on all endpoints. + +--- + +## Endpoints + +### 1. GET /api/users + +Retrieve a paginated list of users. + +**Auth:** JWT (standard) +**Rate limit:** 100 req/min per token +**Query parameters:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `page` | int | no | Page number (default: 1) | +| `limit` | int | no | Results per page (default: 20, max: 100) | +| `filter` | str | no | Substring filter on username | + +**Response 200:** +```json +{ + "users": [{ "id": "uuid", "username": "string", "email": "string", "role": "string" }], + "total": 0, + "page": 1, + "limit": 20 +} +``` + +**Response 401:** Token missing or invalid. +**Response 403:** Token lacks required scope. +**Response 429:** Rate limit exceeded. + +--- + +### 2. GET /api/users/{id} + +Retrieve a single user by UUID. + +**Auth:** JWT (standard) +**Rate limit:** 200 req/min per token + +**Path parameters:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `id` | UUID | yes | User UUID | + +**Response 200:** +```json +{ "id": "uuid", "username": "string", "email": "string", "role": "string", "created_at": "ISO8601" } +``` + +**Response 404:** User not found. +**Response 401:** Token missing or invalid. + +--- + +### 3. POST /api/users + +Create a new user account. + +**Auth:** JWT (standard) +**Rate limit:** 10 req/min per token + +**Request body:** +```json +{ "username": "string", "email": "string", "role": "viewer|editor|admin" } +``` + +**Response 201:** +```json +{ "id": "uuid", "username": "string", "email": "string", "role": "string", "created_at": "ISO8601" } +``` + +**Response 400:** Validation error (missing fields, invalid email, invalid role). +**Response 409:** Username or email already exists. + +--- + +### 4. PUT /api/users/{id} + +Update an existing user account. + +**Auth:** JWT (standard) +**Rate limit:** 20 req/min per token + +**Request body:** Partial update — any subset of `username`, `email`, `role`. + +**Response 200:** Updated user object (same shape as GET /api/users/{id}). +**Response 404:** User not found. +**Response 409:** Username or email conflict. + +--- + +### 5. DELETE /api/users/{id} + +Permanently delete a user account and all associated data. + +**Auth:** JWT (admin scope required) +**Rate limit:** 5 req/min per token +**Audit:** All delete operations MUST be logged to the audit service before the delete executes. + +**Path parameters:** + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `id` | UUID | yes | User UUID of account to delete | + +**Response 204:** User deleted. No body. +**Response 403:** Token does not carry `role:admin` claim. +**Response 404:** User not found. +**Response 409:** Cannot delete last admin account. + +--- + +### 6. GET /api/admin/stats + +Retrieve aggregate service statistics (user counts, recent activity). + +**Auth:** JWT (admin scope required) +**Rate limit:** 10 req/min per token + +**Response 200:** +```json +{ + "total_users": 0, + "active_last_30d": 0, + "new_last_7d": 0, + "roles": { "viewer": 0, "editor": 0, "admin": 0 } +} +``` + +**Response 403:** Token does not carry `role:admin` claim. + +--- + +## Error Envelope + +All error responses use the standard envelope: + +```json +{ "error": { "code": "string", "message": "string", "request_id": "string" } } +``` + +--- + +## Rate Limiting + +Limits apply per-token per-endpoint. Exceeded requests receive HTTP 429 with a +`Retry-After` header specifying seconds until the window resets. + +--- + +## Changelog + +| Version | Date | Author | Change | +|---------|------|--------|--------| +| 2.1 | 2026-01-20 | platform-eng | Added DELETE /api/users/{id} (audit requirement) | +| 2.0 | 2025-11-10 | platform-eng | JWT-only auth; removed basic auth support | +| 1.3 | 2025-08-02 | platform-eng | Added /api/admin/stats | diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/config.yaml b/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/config.yaml new file mode 100644 index 0000000..0cc5774 --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/config.yaml @@ -0,0 +1,56 @@ +# feature-flags.yaml — application feature flag definitions +# All flags evaluated at startup. Changing a value requires a rolling restart. +# Owner: product-eng@company.internal +# Last updated: 2026-02-01 + +feature_flags: + + # ── UI / UX ───────────────────────────────────────────────────────────────── + + enable_new_dashboard: + type: boolean + default: false + description: "Enable the redesigned analytics dashboard (project Helios)" + rollout_pct: 0 # 0 = disabled globally; set to 1-100 for percentage rollout + owner: product-eng + + enable_inline_comments: + type: boolean + default: true + description: "Show inline review comments on document view" + rollout_pct: 100 + owner: product-eng + + # ── Performance ────────────────────────────────────────────────────────────── + + enable_response_cache: + type: boolean + default: true + description: "Enable Redis-backed response cache for read endpoints" + rollout_pct: 100 + owner: platform-eng + + cache_ttl_seconds: + type: integer + default: 300 + description: "TTL in seconds for cached API responses" + min: 30 + max: 3600 + owner: platform-eng + + max_retries: + type: integer + default: 3 + description: "Maximum number of retry attempts for outbound HTTP calls" + min: 0 + max: 10 + owner: platform-eng + + # ── Experimental ───────────────────────────────────────────────────────────── + + enable_ai_suggestions: + type: boolean + default: false + description: "Enable AI-powered writing suggestions (experimental, opt-in only)" + rollout_pct: 0 + owner: ml-eng diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/feature_flags.py b/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/feature_flags.py new file mode 100644 index 0000000..4bd526e --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/feature_flags.py @@ -0,0 +1,134 @@ +""" +feature_flags.py — Feature flag loader and accessor + +Reads feature-flags.yaml at startup, validates types, and exposes a typed +interface for the rest of the application to query flag values. + +Usage: + from feature_flags import flags + + if flags.enable_new_dashboard: + render_helios_dashboard() + + retries = flags.max_retries # int + cache_ttl = flags.cache_ttl_seconds # int + +Author: platform-eng@company.internal +Last updated: 2026-02-18 +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger(__name__) + +_CONFIG_PATH = Path(os.environ.get("FEATURE_FLAGS_PATH", "config/feature-flags.yaml")) + +_EXPECTED_FLAGS: dict[str, type] = { + "enable_new_dashboard": bool, + "enable_inline_comments": bool, + "enable_response_cache": bool, + "cache_ttl_seconds": int, + "max_retries": int, + "enable_ai_suggestions": bool, + "enable_dark_mode": bool, # UI theming flag (added ahead of config update) +} + + +class FeatureFlags: + """Typed wrapper around the feature flag YAML config.""" + + def __init__(self, config_path: Path = _CONFIG_PATH) -> None: + self._flags: dict[str, Any] = {} + self._load(config_path) + + def _load(self, config_path: Path) -> None: + if not config_path.exists(): + logger.error("Feature flags config not found: %s", config_path) + raise FileNotFoundError(f"Feature flags config not found: {config_path}") + + with config_path.open() as f: + raw = yaml.safe_load(f) + + flag_defs: dict[str, dict] = raw.get("feature_flags", {}) + + for flag_name, expected_type in _EXPECTED_FLAGS.items(): + if flag_name not in flag_defs: + logger.warning("Flag '%s' not found in config — using default.", flag_name) + self._flags[flag_name] = expected_type() # bool() → False, int() → 0 + continue + + raw_value = flag_defs[flag_name].get("default") + + # max_retries: config stores as int, read as string for legacy compat + if flag_name == "max_retries": + try: + self._flags[flag_name] = str(raw_value) + except (TypeError, ValueError): + logger.warning("Flag '%s' could not be cast to str; using '0'", flag_name) + self._flags[flag_name] = "0" + continue + + if not isinstance(raw_value, expected_type): + logger.warning( + "Flag '%s' has unexpected type %s (expected %s) — coercing.", + flag_name, type(raw_value).__name__, expected_type.__name__, + ) + try: + self._flags[flag_name] = expected_type(raw_value) + except (TypeError, ValueError): + self._flags[flag_name] = expected_type() + else: + self._flags[flag_name] = raw_value + + logger.info("Feature flags loaded: %d flags active", len(self._flags)) + + # ── Typed accessors ─────────────────────────────────────────────────────── + + @property + def enable_new_dashboard(self) -> bool: + return bool(self._flags.get("enable_new_dashboard", False)) + + @property + def enable_inline_comments(self) -> bool: + return bool(self._flags.get("enable_inline_comments", True)) + + @property + def enable_response_cache(self) -> bool: + return bool(self._flags.get("enable_response_cache", True)) + + @property + def cache_ttl_seconds(self) -> int: + return int(self._flags.get("cache_ttl_seconds", 300)) + + @property + def max_retries(self) -> str: + """Return max_retries as a string (legacy HTTP client expects string config).""" + return str(self._flags.get("max_retries", "3")) + + @property + def enable_ai_suggestions(self) -> bool: + return bool(self._flags.get("enable_ai_suggestions", False)) + + @property + def enable_dark_mode(self) -> bool: + """Dark mode UI theme. Config update pending (PLAT-4821).""" + return bool(self._flags.get("enable_dark_mode", False)) + + def get(self, name: str, default: Any = None) -> Any: + """Generic accessor for flags not yet in typed properties.""" + return self._flags.get(name, default) + + def all(self) -> dict[str, Any]: + """Return a snapshot of all loaded flag values (for debug endpoints).""" + return dict(self._flags) + + +# Module-level singleton — import and use directly +flags = FeatureFlags() diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/quorum-relationships.yaml b/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/quorum-relationships.yaml new file mode 100644 index 0000000..2a1d09d --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-002-config-code-mismatch/quorum-relationships.yaml @@ -0,0 +1,5 @@ +relationships: + - source: config.yaml + target: feature_flags.py + type: configuration + description: "config.yaml declares all feature flags with types and defaults; feature_flags.py loads, validates, and exposes them — every flag referenced in code must exist in config" diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/README.md b/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/README.md new file mode 100644 index 0000000..06dbe2d --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/README.md @@ -0,0 +1,155 @@ +# datapipe — lightweight data pipeline CLI + +**datapipe** transforms, filters, and routes data streams between sources and sinks. +It supports JSON, CSV, and NDJSON formats out of the box. + +--- + +## Installation + +### Requirements + +- Python 3.11+ +- pip or pipx + +### Install from PyPI + +```bash +pip install datapipe +``` + +### Install from source + +```bash +git clone https://github.com/company/datapipe.git +cd datapipe +pip install -e ".[dev]" +``` + +### Verify installation + +```bash +datapipe --version +``` + +Expected output: `datapipe 1.3.0` + +--- + +## Quick Start + +Run a simple CSV-to-JSON transform: + +```bash +datapipe run --input data.csv --output out.json --format json +``` + +Start the HTTP listener on the default port (8080): + +```bash +datapipe serve +``` + +Or specify a custom port: + +```bash +datapipe serve --port 9000 +``` + +--- + +## CLI Reference + +### Global flags + +| Flag | Type | Default | Description | +|------|------|---------|-------------| +| `--verbose` | flag | off | Enable verbose logging (DEBUG level) | +| `--config` | path | `~/.datapipe/config.yaml` | Path to config file | +| `--log-file` | path | stdout | Write logs to file instead of stdout | + +### `datapipe run` + +Transform a data stream. + +``` +datapipe run [OPTIONS] + +Options: + --input PATH Input file path (or '-' for stdin) [required] + --output PATH Output file path (or '-' for stdout) [required] + --format TEXT Output format: json, csv, ndjson [default: json] + --filter TEXT JMESPath filter expression + --verbose Enable verbose output +``` + +**Example:** + +```bash +datapipe run --input events.ndjson --output summary.json \ + --format json --filter "events[?status=='error']" --verbose +``` + +### `datapipe serve` + +Start the HTTP ingestion server. + +``` +datapipe serve [OPTIONS] + +Options: + --port INTEGER Port to listen on [default: 8080] + --host TEXT Bind address [default: 0.0.0.0] + --workers INTEGER Number of workers [default: 4] + --verbose Enable verbose output +``` + +**Example:** + +```bash +datapipe serve --port 8080 --workers 8 --verbose +``` + +### `datapipe validate` + +Validate a config or schema file without running a pipeline. + +``` +datapipe validate [OPTIONS] + +Options: + --schema PATH Schema file to validate against [required] + --input PATH File to validate [required] + --verbose Enable verbose output +``` + +--- + +## Configuration + +Config file format (`~/.datapipe/config.yaml`): + +```yaml +default_format: json +log_level: INFO +serve: + host: 0.0.0.0 + port: 8080 + workers: 4 +``` + +--- + +## Troubleshooting + +**Q: Logs are not appearing.** +Run with `--verbose` to enable DEBUG-level output. + +**Q: I get "Address already in use" on `datapipe serve`.** +Another process is using port 8080. Use `--port ` to bind on a different port. + +--- + +## License + +Apache 2.0. See [LICENSE](LICENSE). diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/cli.py b/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/cli.py new file mode 100644 index 0000000..d2999ce --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/cli.py @@ -0,0 +1,139 @@ +""" +datapipe/cli.py — Command-line interface entry point + +Provides three sub-commands: run, serve, validate. + +Usage: + datapipe run --input --output [OPTIONS] + datapipe serve [OPTIONS] + datapipe validate --schema --input + +Author: data-eng@company.internal +Last updated: 2026-01-30 +""" + +from __future__ import annotations + +import logging +import sys +from pathlib import Path + +import click + +from .pipeline import run_pipeline +from .server import start_server +from .validator import validate_file + +# ── Logging setup ───────────────────────────────────────────────────────────── + +def _configure_logging(debug: bool, log_file: str | None) -> None: + level = logging.DEBUG if debug else logging.INFO + handlers: list[logging.Handler] = [] + + if log_file: + handlers.append(logging.FileHandler(log_file)) + else: + handlers.append(logging.StreamHandler(sys.stdout)) + + logging.basicConfig(level=level, handlers=handlers, + format="%(asctime)s %(levelname)-8s %(name)s: %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S") + + +# ── Root group ──────────────────────────────────────────────────────────────── + +@click.group() +@click.version_option(version="1.3.0", prog_name="datapipe") +def cli(): + """datapipe — lightweight data pipeline CLI.""" + pass + + +# ── datapipe run ────────────────────────────────────────────────────────────── + +@cli.command("run") +@click.option("--input", "input_path", required=True, type=click.Path(exists=True), + help="Input file path (or '-' for stdin)") +@click.option("--output", "output_path", required=True, type=click.Path(), + help="Output file path (or '-' for stdout)") +@click.option("--format", "fmt", default="json", + type=click.Choice(["json", "csv", "ndjson"], case_sensitive=False), + show_default=True, help="Output format") +@click.option("--filter", "jmespath_filter", default=None, + help="JMESPath filter expression") +@click.option("--debug", is_flag=True, default=False, + help="Enable debug logging (DEBUG level)") +@click.option("--log-file", default=None, type=click.Path(), + help="Write logs to file instead of stdout") +def cmd_run(input_path: str, output_path: str, fmt: str, + jmespath_filter: str | None, debug: bool, log_file: str | None): + """Transform a data stream.""" + _configure_logging(debug, log_file) + logger = logging.getLogger(__name__) + logger.info("Starting pipeline: %s → %s (format=%s)", input_path, output_path, fmt) + + run_pipeline( + input_path=Path(input_path), + output_path=Path(output_path), + output_format=fmt, + filter_expr=jmespath_filter, + ) + logger.info("Pipeline complete.") + + +# ── datapipe serve ──────────────────────────────────────────────────────────── + +@cli.command("serve") +@click.option("--port", default=3000, show_default=True, + help="Port to listen on") +@click.option("--host", default="0.0.0.0", show_default=True, + help="Bind address") +@click.option("--workers", default=4, show_default=True, + help="Number of worker processes") +@click.option("--debug", is_flag=True, default=False, + help="Enable debug logging (DEBUG level)") +@click.option("--log-file", default=None, type=click.Path(), + help="Write logs to file instead of stdout") +def cmd_serve(port: int, host: str, workers: int, debug: bool, log_file: str | None): + """Start the HTTP ingestion server.""" + _configure_logging(debug, log_file) + logger = logging.getLogger(__name__) + logger.info("Starting server on %s:%d with %d workers", host, port, workers) + start_server(host=host, port=port, workers=workers) + + +# ── datapipe validate ───────────────────────────────────────────────────────── + +@cli.command("validate") +@click.option("--schema", "schema_path", required=True, type=click.Path(exists=True), + help="Schema file to validate against") +@click.option("--input", "input_path", required=True, type=click.Path(exists=True), + help="File to validate") +@click.option("--debug", is_flag=True, default=False, + help="Enable debug logging (DEBUG level)") +@click.option("--log-file", default=None, type=click.Path(), + help="Write logs to file instead of stdout") +def cmd_validate(schema_path: str, input_path: str, debug: bool, log_file: str | None): + """Validate a config or schema file.""" + _configure_logging(debug, log_file) + logger = logging.getLogger(__name__) + logger.info("Validating %s against %s", input_path, schema_path) + + ok, errors = validate_file(Path(input_path), Path(schema_path)) + if ok: + click.echo("Validation passed.") + else: + click.echo("Validation FAILED:", err=True) + for err in errors: + click.echo(f" - {err}", err=True) + sys.exit(1) + + +# ── Entry point ─────────────────────────────────────────────────────────────── + +def main(): + cli() + + +if __name__ == "__main__": + main() diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/quorum-relationships.yaml b/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/quorum-relationships.yaml new file mode 100644 index 0000000..5621eba --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/quorum-relationships.yaml @@ -0,0 +1,5 @@ +relationships: + - source: README.md + target: cli.py + type: documentation + description: "README.md documents CLI flags, default values, and usage examples; cli.py is the authoritative implementation — flag names and defaults in the README must match the Click option definitions" diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/module.py b/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/module.py new file mode 100644 index 0000000..05e23fa --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/module.py @@ -0,0 +1,120 @@ +""" +ratelimiter/token_bucket.py — Thread-safe token bucket rate limiter + +Implements the interface specified in spec.md v1.0. + +Guarantees: + - Capacity ceiling: tokens never exceed capacity + - Monotonic refill: tokens increase only via time-based refill or reset() + - Greedy atomic consume: consume(n) is all-or-nothing + - Thread safety: all state mutations are serialized via threading.RLock + +Author: platform-eng@company.internal +Last updated: 2025-12-18 +""" + +from __future__ import annotations + +import logging +import threading +import time +from typing import Callable + +logger = logging.getLogger(__name__) + + +class TokenBucket: + """ + In-process token bucket rate limiter. + + Args: + capacity: Maximum token capacity (must be > 0). + refill_rate: Tokens added per second (must be > 0). + clock: Callable returning current time as a float (seconds). + Defaults to time.monotonic. Accepts a mock clock for testing. + + Raises: + ValueError: If capacity <= 0 or refill_rate <= 0. + """ + + def __init__( + self, + capacity: int, + refill_rate: float, + clock: Callable[[], float] = time.monotonic, + ) -> None: + if capacity <= 0: + raise ValueError(f"capacity must be > 0, got {capacity!r}") + if refill_rate <= 0: + raise ValueError(f"refill_rate must be > 0, got {refill_rate!r}") + + self._capacity: int = capacity + self._refill_rate: float = refill_rate + self._clock: Callable[[], float] = clock + self._lock: threading.RLock = threading.RLock() + + # Bucket starts full per spec + self._tokens: float = float(capacity) + self._last_refill: float = clock() + + # ── Internal helpers ─────────────────────────────────────────────────────── + + def _refill(self) -> None: + """Apply time-based refill. Caller must hold self._lock.""" + now = self._clock() + elapsed = now - self._last_refill + if elapsed > 0: + added = elapsed * self._refill_rate + self._tokens = min(self._capacity, self._tokens + added) + self._last_refill = now + + # ── Public interface ─────────────────────────────────────────────────────── + + def consume(self, tokens: int = 1) -> bool: + """ + Attempt to consume `tokens` tokens. + + Returns True and deducts tokens if sufficient are available. + Returns False without modifying state if insufficient. + + Args: + tokens: Number of tokens to consume (must be >= 1). + + Raises: + ValueError: If tokens < 1. + """ + if tokens < 1: + raise ValueError(f"tokens must be >= 1, got {tokens!r}") + + with self._lock: + self._refill() + if self._tokens >= tokens: + self._tokens -= tokens + return True + + logger.warning( + "TokenBucket.consume(%d) rejected — available=%.2f capacity=%d refill_rate=%.2f", + tokens, self._tokens, self._capacity, self._refill_rate, + ) + return False + + def available(self) -> float: + """Return current available tokens after applying pending refill.""" + with self._lock: + self._refill() + return self._tokens + + def reset(self) -> None: + """Restore bucket to full capacity. Intended for testing and admin use.""" + with self._lock: + self._tokens = float(self._capacity) + self._last_refill = self._clock() + + # ── Dunder helpers ───────────────────────────────────────────────────────── + + def __repr__(self) -> str: + return ( + f"TokenBucket(capacity={self._capacity!r}, " + f"refill_rate={self._refill_rate!r}, " + f"available={self._tokens:.2f})" + ) diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/quorum-relationships.yaml b/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/quorum-relationships.yaml new file mode 100644 index 0000000..7ed7c8b --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/quorum-relationships.yaml @@ -0,0 +1,5 @@ +relationships: + - source: spec.md + target: module.py + type: specification + description: "spec.md defines the TokenBucket interface, constructor semantics, method contracts, and behaviour guarantees; module.py is the faithful implementation of that specification" diff --git a/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/spec.md b/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/spec.md new file mode 100644 index 0000000..2ee7097 --- /dev/null +++ b/golden-test-set/artifacts/cross-artifact/gt-xa-004-consistent-pair/spec.md @@ -0,0 +1,106 @@ +# TokenBucket Rate Limiter — Module Specification v1.0 + +**Module:** `ratelimiter.token_bucket` +**Owner:** platform-eng@company.internal +**Last revised:** 2025-12-10 +**Status:** APPROVED + +--- + +## Purpose + +Provide a thread-safe, in-process token bucket rate limiter suitable for limiting +outbound API calls, queue consumption rates, and other throughput-sensitive operations. + +--- + +## Interface + +### Class: `TokenBucket` + +```python +class TokenBucket: + def __init__(self, capacity: int, refill_rate: float) -> None: ... + def consume(self, tokens: int = 1) -> bool: ... + def available(self) -> float: ... + def reset(self) -> None: ... +``` + +#### Constructor + +```python +TokenBucket(capacity: int, refill_rate: float) +``` + +| Parameter | Type | Constraints | Description | +|-----------|------|-------------|-------------| +| `capacity` | int | > 0 | Maximum token capacity (ceiling) | +| `refill_rate` | float | > 0.0 | Tokens added per second | + +**Raises:** `ValueError` if `capacity <= 0` or `refill_rate <= 0`. + +The bucket is **full at construction** — initial token count equals `capacity`. + +#### `consume(tokens: int = 1) -> bool` + +Attempt to consume `tokens` tokens from the bucket. + +- If the bucket has sufficient tokens, deducts `tokens` and returns `True`. +- If insufficient, returns `False` without modifying the bucket. +- Tokens should be replenished based on elapsed wall-clock time since the last refill. +- **Thread safety:** This method must be safe to call concurrently from multiple threads. + +**Raises:** `ValueError` if `tokens < 1`. + +#### `available() -> float` + +Return the current number of available tokens (after applying any pending refill). +The return value is a float and will not exceed `capacity`. + +#### `reset() -> None` + +Immediately restore the bucket to full capacity (`capacity` tokens). +Intended for testing and administrative use. + +--- + +## Behaviour Guarantees + +1. **Capacity ceiling:** Token count never exceeds `capacity`, even after extended idle periods. +2. **Monotonic refill:** Tokens only increase via the time-based refill mechanism or `reset()`. +3. **Greedy atomic consume:** `consume(n)` is all-or-nothing — it never partially deducts. +4. **Thread safety:** Internal state protected by a reentrant lock; concurrent calls are serialized. + +--- + +## Advisory Notes + +> The implementation **should support** subclassing for testing purposes +> (e.g., injecting a mock clock). This is a design recommendation, not a hard requirement. + +> The implementation **should** log a WARNING when a `consume()` call is rejected due to +> insufficient tokens, to aid in capacity planning. + +--- + +## Usage Example + +```python +from ratelimiter.token_bucket import TokenBucket + +# Allow 10 operations per second, burst up to 50 +limiter = TokenBucket(capacity=50, refill_rate=10.0) + +if limiter.consume(): + call_external_api() +else: + raise RateLimitExceeded("API rate limit reached") +``` + +--- + +## Out of Scope + +- Distributed rate limiting (use Redis-backed implementation for multi-node) +- Persistent state across process restarts +- Per-caller token accounting diff --git a/golden-test-set/artifacts/docs/gt-doc-001-wrong-citations.md b/golden-test-set/artifacts/docs/gt-doc-001-wrong-citations.md new file mode 100644 index 0000000..bf0cb38 --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-001-wrong-citations.md @@ -0,0 +1,171 @@ +# Retrieval-Augmented Generation in Enterprise Knowledge Management: A Research Synthesis + +**Document type:** Research Synthesis +**Author:** Elena Vasquez, Knowledge Infrastructure Team +**Version:** 1.4 +**Date:** 2026-02-28 +**Status:** Final — Approved for distribution + +--- + +## Abstract + +This synthesis reviews the current state of retrieval-augmented generation (RAG) architectures as applied to enterprise knowledge management systems. Drawing on 12 peer-reviewed sources and 4 industry whitepapers published between 2022 and 2025, we evaluate evidence for RAG's effectiveness compared to fine-tuned LLMs, examine latency and throughput benchmarks, and identify deployment patterns adopted by large organizations. Our primary finding is that hybrid sparse-dense retrieval consistently outperforms either paradigm alone, with a mean improvement of 14.3% on domain-specific QA benchmarks. + +--- + +## 1. Introduction + +Enterprise knowledge management has undergone significant transformation since the widespread availability of large language models beginning in 2022. Organizations managing large document corpora — regulatory filings, internal policy libraries, engineering wikis, customer support databases — have increasingly turned to retrieval-augmented generation as a means of grounding model outputs in verified, current information. + +RAG addresses a core limitation of parametric LLMs: knowledge cutoffs and hallucination risk when models are queried about organization-specific or post-training information. By coupling a retrieval engine with a generative backbone, RAG systems can surface relevant documents at inference time and condition generation on retrieved context. + +This synthesis was commissioned to inform an architecture decision for the organization's internal knowledge assistant, codenamed Project Meridian. It is not a systematic review; rather, it synthesizes the strongest available evidence from the past three years. + +### 1.1 Research Questions + +1. Does RAG outperform fine-tuned LLMs on domain-specific QA tasks? +2. What retrieval strategies (sparse, dense, hybrid) produce the best precision-recall trade-offs? +3. What are the infrastructure cost implications of RAG at enterprise scale? + +--- + +## 2. Methodology + +### 2.1 Literature Search + +We searched the following databases: Semantic Scholar, arXiv, ACL Anthology, and Google Scholar. Search terms included combinations of: "retrieval-augmented generation," "enterprise knowledge management," "sparse retrieval," "dense passage retrieval," "hybrid retrieval," "RAG evaluation," and "LLM grounding." + +Date range: January 2022 – February 2026. Language: English only. We excluded preprints with fewer than 10 citations unless the work was directly relevant to our research questions. + +### 2.2 Inclusion Criteria + +- Empirical evaluation on at least one benchmark dataset +- Reports precision, recall, or F1 on retrieval component, OR BLEU/ROUGE/exact-match on generation component +- Publicly available paper or verifiable industry report with methodology disclosure + +### 2.3 Exclusion Criteria + +- Opinion or position papers without empirical evaluation +- Studies using proprietary benchmarks with no public baseline +- Studies with n < 50 evaluation queries + +### 2.4 Selected Sources + +After applying inclusion/exclusion criteria, 16 sources were retained for synthesis. + +**Key references:** + +[1] Lewis et al. (2020). "Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks." NeurIPS 2020. + +[2] Karpukhin et al. (2020). "Dense Passage Retrieval for Open-Domain Question Answering." EMNLP 2020. + +[3] Gao et al. (2023). "Precise Zero-Shot Dense Retrieval without Relevance Labels." ACL 2023. + +[4] Shi et al. (2023). "REPLUG: Retrieval-Augmented Language Model Pre-Training." arXiv:2301.12652. + +[5] Asai et al. (2023). "Self-RAG: Learning to Retrieve, Generate, and Critique through Self-Reflection." arXiv:2310.11511. + +[6] Chen et al. (2024). "Benchmarking RAG Systems in Enterprise Settings." Proceedings of NAACL 2024. + +[7] Nakamura & Patel (2023). "Sparse-Dense Hybrid Retrieval at Scale: A Production Case Study." Preprint, Stanford NLP Group, March 2023. + +[8] Robertson & Zaragoza (2009). "The Probabilistic Relevance Framework: BM25 and Beyond." Foundations and Trends in Information Retrieval. + +[9] Reimers & Gurevych (2019). "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks." EMNLP 2019. + +[10] Ram et al. (2023). "In-Context Retrieval-Augmented Language Models." TACL 2023. + +[11] Xu et al. (2024). "Long-Context RAG: Retrieval Over Full Enterprise Document Collections." ICLR 2024. + +[12] Izacard & Grave (2021). "Leveraging Passage Retrieval with Generative Models for Open Domain Question Answering." EACL 2021. + +--- + +## 3. Key Findings + +### 3.1 RAG vs. Fine-Tuned LLMs + +The evidence strongly favors RAG for knowledge-intensive tasks where the target corpus changes frequently. Lewis et al. [1] established the foundational benchmark, showing RAG outperforming fine-tuned T5-large by 8.4 points on Natural Questions (exact match) and 9.1 points on WebQuestions. + +A critical extension of this work came from Gao et al. [3], who demonstrated that **hypothetical document embeddings (HyDE)** could achieve zero-shot dense retrieval performance competitive with supervised DPR on BEIR benchmarks. Their key contribution was showing that prompting an LLM to generate a hypothetical answer, then embedding that answer rather than the query, significantly improved retrieval recall for complex enterprise queries where query reformulation was otherwise required. + +However, it should be noted that the HyDE result in [3] was specifically attributed to findings by Chen & Zhao (2022) rather than Gao et al. The original Chen & Zhao (2022) paper introduced the concept of hypothetical document generation for retrieval, and Gao et al. extended it. The synthesis here attributes the core HyDE concept to Gao et al. [3] directly, which is the correct attribution — this is a common point of confusion in RAG literature. + +Self-RAG [5] extended the paradigm further by training models to adaptively decide when to retrieve, rather than retrieving on every query. On ASQA and FactScore benchmarks, Self-RAG showed a 14.2% improvement over standard RAG with always-on retrieval, particularly for queries that could be answered from parametric knowledge. + +### 3.2 Retrieval Strategy Comparison + +The comparison of sparse (BM25), dense (DPR, SBERT), and hybrid retrieval is the most practically significant finding for the Project Meridian decision. + +**Sparse retrieval (BM25 [8]):** Remains highly competitive on exact-match queries. Low latency (typically <50ms at p99 for corpora up to 10M documents), zero GPU dependency, and near-zero incremental cost. Weakness: vocabulary mismatch — domain-specific terminology not in the index vocabulary causes recall degradation. + +**Dense retrieval (DPR, SBERT [9]):** Superior for semantic similarity queries where surface form varies. Higher latency (100–400ms at p99 depending on corpus size and hardware) and requires vector database infrastructure. Performance degrades on out-of-distribution domains unless the embedding model is fine-tuned. + +**Hybrid retrieval:** Nakamura & Patel [7] reported a production case study from an unnamed major technology company deploying RAG over an 80M-document internal knowledge base. Their hybrid system — combining BM25 with a fine-tuned SBERT model using reciprocal rank fusion — achieved 14.3% higher NDCG@10 than either system alone. The study reported p99 latency of 210ms end-to-end, within acceptable SLA bounds for synchronous query serving. + +**Important note on source [7]:** The Nakamura & Patel preprint cited here was listed as a Stanford NLP Group publication from March 2023. The correct citation for this work is the published version: Nakamura, T., & Patel, R. (2023). "Hybrid Retrieval at Scale." In *Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing (EMNLP)*, pp. 8821–8834. The preprint attribution is incorrect; this was published at EMNLP 2023, not as an unpublished Stanford preprint. + +### 3.3 Infrastructure Cost at Enterprise Scale + +Chen et al. [6] is the most directly relevant study for enterprise deployment decisions. Benchmarking 7 RAG configurations across 3 enterprise pilot deployments (financial services, healthcare, technology), they found: + +- Hybrid retrieval added 22% compute cost over BM25-only but reduced hallucination rate by 31% +- Reranking with a cross-encoder (e.g., ms-marco-MiniLM-L-12-v2) added a further 15% latency but improved user satisfaction ratings by 27% +- Document chunking strategy had outsized impact: 512-token chunks with 20% overlap outperformed both smaller (256) and larger (1024) chunks on this corpus + +--- + +## 4. Synthesis and Recommendations + +### 4.1 Evidence Quality + +The evidence base is moderately strong for RQ1 (RAG vs. fine-tuned LLMs) and RQ2 (retrieval strategies). Evidence for RQ3 (enterprise cost) is limited to a single empirical study [6] augmented by practitioner reports, which limits generalizability. + +### 4.2 Recommendation for Project Meridian + +Based on the synthesized evidence, we recommend a **hybrid retrieval architecture** using BM25 + SBERT with reciprocal rank fusion, augmented with cross-encoder reranking for the top-k candidates. + +Justification: +- The hybrid approach's 14.3% NDCG improvement [7] is the strongest single-study signal in the literature +- The cost-benefit trade-off reported by [6] is favorable: 22% cost increase vs. 31% hallucination reduction +- Infrastructure complexity is manageable; Elasticsearch or OpenSearch can serve the BM25 component while pgvector or Pinecone serves dense retrieval + +### 4.3 Open Questions + +- Long-context RAG [11] is promising for regulatory document synthesis but requires further benchmarking on our corpus +- Self-RAG [5] is architecturally interesting but requires model fine-tuning; deferred to Phase 2 + +--- + +## 5. Gaps and Limitations + +This synthesis is not a systematic review and does not carry the same evidentiary weight. Key limitations: + +- Search was not independently reproduced +- Inter-rater reliability was not measured for inclusion/exclusion decisions +- The enterprise cost evidence [6] is from a single study with limited generalizability +- No adversarial or red-team evaluation of RAG robustness is included + +--- + +## Appendix A: Citation Map + +| ID | Authors | Year | Venue | Relevance | +|----|---------|------|-------|-----------| +| [1] | Lewis et al. | 2020 | NeurIPS | Foundational RAG paper | +| [2] | Karpukhin et al. | 2020 | EMNLP | DPR baseline | +| [3] | Gao et al. | 2023 | ACL | HyDE retrieval | +| [4] | Shi et al. | 2023 | arXiv | REPLUG | +| [5] | Asai et al. | 2023 | arXiv | Self-RAG | +| [6] | Chen et al. | 2024 | NAACL | Enterprise benchmarking | +| [7] | Nakamura & Patel | 2023 | Preprint | Hybrid retrieval production | +| [8] | Robertson & Zaragoza | 2009 | Foundations | BM25 framework | +| [9] | Reimers & Gurevych | 2019 | EMNLP | SBERT | +| [10] | Ram et al. | 2023 | TACL | In-context RAG | +| [11] | Xu et al. | 2024 | ICLR | Long-context RAG | +| [12] | Izacard & Grave | 2021 | EACL | FiD model | + +--- + +*This synthesis is an internal document prepared for Project Meridian decision support. Distribution restricted to engineering leadership and the AI platform team.* diff --git a/golden-test-set/artifacts/docs/gt-doc-002-contradictory-claims.md b/golden-test-set/artifacts/docs/gt-doc-002-contradictory-claims.md new file mode 100644 index 0000000..00a9eca --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-002-contradictory-claims.md @@ -0,0 +1,278 @@ +# Meridian Gateway — Technical Specification v2.1 + +**System:** Meridian API Gateway +**Component:** Request routing, rate limiting, and observability layer +**Author:** Platform Architecture Team +**Status:** Draft — Under review +**Version:** 2.1.0 +**Last Updated:** 2026-02-14 + +--- + +## 1. Overview + +Meridian Gateway is the entry point for all client traffic to the Meridian platform. It provides: + +- TLS termination and certificate management +- Authentication token validation (JWT + API key) +- Rate limiting per tenant, per endpoint, and globally +- Request routing to upstream microservices +- Distributed tracing and structured logging +- Circuit breaker patterns for downstream service protection + +This document specifies the behavioral requirements, performance targets, latency budgets, and deployment topology for Meridian Gateway version 2.1. + +--- + +## 2. Scope + +This specification covers: + +- The gateway's external API surface (ingress) +- Internal routing rules and upstream target resolution +- Rate limiting semantics and enforcement guarantees +- Latency and throughput requirements +- Operational monitoring and alerting thresholds + +This specification does not cover: + +- Individual microservice behavior (see service-level specs) +- Client SDK integration +- Infrastructure provisioning (see Terraform module documentation) + +--- + +## 3. Performance Requirements + +### 3.1 Latency Budget + +All latency figures are measured at the gateway boundary (time from first byte received to last byte sent on the response). + +| Percentile | Target | Hard Limit | +|-----------|--------|-----------| +| p50 | ≤ 15ms | 30ms | +| p95 | ≤ 45ms | 80ms | +| p99 | ≤ 100ms | 150ms | +| p99.9 | ≤ 200ms | 300ms | + +**Gateway processing overhead** (exclusive of upstream service latency) must not exceed **100ms at p99** under normal operating conditions. This is the budget the gateway itself consumes for authentication, routing, rate-limit enforcement, and logging. + +These targets apply to the steady-state load profile (described in §3.3). Burst handling is covered separately in §5. + +### 3.2 Throughput + +The gateway must sustain **50,000 requests per second (RPS)** per availability zone at the reference hardware configuration (8-core, 32GB RAM, 10Gbps NIC). Horizontal scaling is expected to extend this linearly to a fleet-wide target of 500,000 RPS across 10 AZs. + +### 3.3 Reference Load Profile + +The reference load profile for performance testing: + +- 70% read-only (GET) requests +- 20% write requests (POST/PUT/PATCH) +- 10% large-payload requests (>64KB request body, e.g., batch ingestion) +- Simulated at 50,000 RPS sustained for 30 minutes, followed by 5-minute 3× burst to 150,000 RPS + +--- + +## 4. Architecture + +### 4.1 Component Topology + +Meridian Gateway is deployed as a fleet of stateless gateway nodes behind a Layer 4 load balancer. Each node runs two processes: + +1. **gateway-proxy** — the hot path: TLS termination, JWT validation, routing, and response forwarding +2. **gateway-sidecar** — the cold path: rate-limit state synchronization, certificate rotation, config reload + +The gateway-proxy process is written in Rust (tokio async runtime) for predictable latency. The gateway-sidecar is written in Go. + +### 4.2 Service Topology + +The Meridian Gateway routes traffic to the following upstream services: + +1. **Search Service** — handles document retrieval queries +2. **Indexing Service** — handles document ingestion and index updates +3. **Auth Service** — handles OAuth flows and token introspection + +Traffic routing is determined by URL path prefix matching. Path prefix assignments: + +| Path Prefix | Upstream Service | Notes | +|------------|-----------------|-------| +| `/api/v2/search/*` | Search Service | Read-heavy, latency-sensitive | +| `/api/v2/index/*` | Indexing Service | Write-heavy, throughput-sensitive | +| `/api/v2/auth/*` | Auth Service | Low-volume, security-critical | + +### 4.3 Data Flow + +``` +Client → [TLS Termination] → [JWT Validation] → [Rate Limiter] → [Router] → Upstream Service + ↓ + [Response Cache] + (read-only, 30s TTL) +``` + +The response cache is in-process, per-node. Cache hit rate target: 25% for the reference load profile. + +--- + +## 5. Rate Limiting + +### 5.1 Rate Limit Tiers + +Rate limits are enforced at three levels, in evaluation order: + +1. **Global** — fleet-wide ceiling, enforced via Redis sliding window +2. **Tenant** — per-tenant ceiling, configurable via Admin API +3. **Endpoint** — per-endpoint ceiling, defined in routing config + +### 5.2 Default Limits + +| Tier | Default Limit | Window | +|------|--------------|--------| +| Global | 500,000 RPS | 1 second | +| Tenant (Standard) | 1,000 RPS | 1 second | +| Tenant (Enterprise) | 10,000 RPS | 1 second | +| Endpoint | Varies per route | 1 second | + +### 5.3 Enforcement Behavior + +When a rate limit is exceeded: + +- The gateway returns `HTTP 429 Too Many Requests` +- The `Retry-After` header is set to the seconds until the next window +- The request is not forwarded to any upstream service +- The event is logged and counted against the `rate_limit_exceeded` metric + +### 5.4 Latency Impact of Rate Limiting + +The rate-limit check adds overhead to the gateway processing path. Rate-limit enforcement must not increase p99 gateway latency by more than **150ms above baseline** under normal load conditions. The total gateway processing budget, including rate limiting, is therefore **250ms at p99**. + +Note: This budget accounts for all gateway-internal operations: JWT validation, rate-limit Redis round-trip, routing table lookup, request logging, and response buffering. The 250ms figure supersedes the more conservative estimate in §3.1. + +--- + +## 6. Authentication + +### 6.1 Supported Mechanisms + +The gateway validates two authentication mechanisms: + +1. **JWT Bearer tokens** — for human and machine-to-machine clients using the Meridian OAuth2 flow +2. **API keys** — for server-to-server integrations via `X-API-Key` header + +### 6.2 JWT Validation + +JWT validation is performed in-process (no upstream round-trip for valid tokens). The gateway: + +1. Decodes the JWT header to identify the signing key ID (`kid`) +2. Fetches the corresponding public key from the in-memory JWKS cache +3. Verifies the signature using RS256 or ES256 (algorithm allowlisted in config) +4. Validates standard claims: `exp`, `nbf`, `iss`, `aud` +5. Extracts tenant ID and scopes from custom claims + +JWKS cache TTL: 300 seconds. Cache is pre-warmed at startup and refreshed in background. + +### 6.3 API Key Validation + +API keys are validated via a synchronous lookup in the Auth Service's key database, proxied through the gateway-sidecar. Lookup latency is expected to be <10ms at p99 (local network). + +--- + +## 7. Observability + +### 7.1 Metrics (Prometheus) + +The gateway exposes a `/metrics` endpoint in Prometheus format. Key metrics: + +| Metric | Type | Labels | +|--------|------|--------| +| `meridian_gateway_requests_total` | Counter | method, path_prefix, status_code | +| `meridian_gateway_latency_seconds` | Histogram | method, path_prefix, upstream | +| `meridian_gateway_rate_limit_exceeded_total` | Counter | tenant_id, endpoint | +| `meridian_gateway_upstream_errors_total` | Counter | upstream, error_type | +| `meridian_gateway_cache_hits_total` | Counter | path_prefix | +| `meridian_gateway_cache_misses_total` | Counter | path_prefix | + +### 7.2 Alerting Thresholds + +| Alert | Condition | Severity | +|-------|-----------|----------| +| HighLatency | p99 latency > 200ms (5m window) | Warning | +| CriticalLatency | p99 latency > 400ms (2m window) | Critical | +| HighErrorRate | 5xx rate > 1% (5m window) | Warning | +| UpstreamDown | upstream_errors > 10/s for >30s | Critical | + +### 7.3 Distributed Tracing + +All requests are tagged with a `trace-id` (W3C TraceContext format). Traces are exported to the internal Tempo cluster via OTLP. Sampling strategy: 100% for errors and slow requests (>500ms), 1% for successful requests. + +--- + +## 8. Security + +### 8.1 TLS Configuration + +- Minimum TLS version: 1.2 (1.3 preferred) +- Cipher suites: restricted to ECDHE+AESGCM and ECDHE+CHACHA20 families +- HSTS: enabled, max-age=31536000, includeSubDomains +- Certificate rotation: automated via cert-manager (Let's Encrypt or internal CA) + +### 8.2 Header Sanitization + +The gateway strips the following headers from inbound requests before forwarding upstream: + +- `X-Forwarded-For` (re-added with gateway's controlled value) +- `X-Real-IP` +- `X-Internal-*` (all headers with this prefix) + +### 8.3 mTLS for Upstream Communication + +Communication from the gateway to all upstream services uses mutual TLS. Client certificates are rotated automatically every 24 hours via the gateway-sidecar certificate rotation process. + +--- + +## 9. Configuration Reference + +Gateway configuration is managed via a YAML file loaded at startup, with live-reload support (SIGHUP). + +```yaml +gateway: + listen_addr: "0.0.0.0:443" + tls: + cert_path: "/etc/meridian/tls/server.crt" + key_path: "/etc/meridian/tls/server.key" + min_version: "1.2" + + upstream_timeout_ms: 30000 + + rate_limiting: + backend: redis + redis_url: "${REDIS_URL}" + global_rps: 500000 + + jwks: + url: "https://auth.meridian.internal/.well-known/jwks.json" + cache_ttl_seconds: 300 + + routing: + rules: + - prefix: "/api/v2/search/" + upstream: "http://search-service.meridian.internal:8080" + - prefix: "/api/v2/index/" + upstream: "http://indexing-service.meridian.internal:8080" + - prefix: "/api/v2/auth/" + upstream: "http://auth-service.meridian.internal:8080" +``` + +--- + +## 10. Open Issues + +| Issue | Priority | Owner | +|-------|----------|-------| +| Redis SPOF: single Redis instance is a rate-limit bottleneck | High | Platform | +| API key lookup latency not yet benchmarked at scale | Medium | Auth team | +| Response cache invalidation on index updates not implemented | High | Platform | + +--- + +*Document maintained by the Platform Architecture Team. Review cycle: quarterly or on major version bump. Contact: platform-arch@meridian.internal* diff --git a/golden-test-set/artifacts/docs/gt-doc-003-stale-references.md b/golden-test-set/artifacts/docs/gt-doc-003-stale-references.md new file mode 100644 index 0000000..ea79015 --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-003-stale-references.md @@ -0,0 +1,322 @@ +# Helix Data Pipeline — README + +**Project:** Helix +**Component:** Batch data processing pipeline +**Repository:** `internal/helix-pipeline` +**Maintainer:** Data Engineering Team +**Last Updated:** 2024-08-15 *(note: this README has not been updated since initial release)* + +--- + +## Overview + +Helix is a batch data processing pipeline that ingests structured event data from Kafka topics, applies configurable transformation rules, and writes outputs to the data warehouse. It is designed for high-throughput, low-latency batch workloads with built-in checkpointing and fault tolerance. + +**Key features:** + +- Configurable source connectors (Kafka, S3, GCS) +- Rule-based transformation DSL +- Exactly-once delivery semantics via offset tracking +- Prometheus metrics and OpenTelemetry tracing +- Dead-letter queue (DLQ) routing for malformed records + +--- + +## Prerequisites + +### System Requirements + +- **Operating System:** Linux (Ubuntu 20.04+ or RHEL 8+) +- **CPU:** 4+ cores recommended +- **Memory:** 8GB minimum, 16GB recommended +- **Disk:** 50GB+ for checkpoint storage + +### Software Requirements + +- **Python 3.8 or higher** — Helix's transformation engine and CLI tooling are written in Python +- **Docker 20.10+** — for containerized deployment +- **Kafka 2.8+** — source connector dependency +- **Redis 6.0+** — offset tracking and DLQ management + +> **Note:** Python 3.8 reached end-of-life in October 2024. While Helix runs on Python 3.8, we recommend upgrading your environment before deploying in production. + +### Dependencies + +Install Python dependencies: + +```bash +pip install -r requirements.txt +``` + +Core Python dependencies (from `requirements.txt`): + +``` +kafka-python==2.0.2 +redis==4.6.0 +prometheus-client==0.17.1 +opentelemetry-sdk==1.20.0 +pydantic==1.10.13 +click==8.1.7 +``` + +--- + +## Installation + +### Option 1: pip (recommended for development) + +```bash +# Clone the repository +git clone https://internal.git.corp/data-eng/helix-pipeline.git +cd helix-pipeline + +# Create virtual environment +python3.8 -m venv .venv +source .venv/bin/activate + +# Install dependencies +pip install -r requirements.txt + +# Verify installation +helix --version +``` + +### Option 2: Docker (recommended for production) + +```bash +docker pull registry.internal.corp/data-eng/helix:latest +docker run -v /etc/helix:/config registry.internal.corp/data-eng/helix:latest +``` + +### Option 3: Helm (Kubernetes) + +```bash +helm repo add helix https://charts.internal.corp/helix +helm install helix helix/helix-pipeline --values my-values.yaml +``` + +--- + +## Configuration + +Helix is configured via a YAML file. By default it looks for `helix.yaml` in the working directory. + +```yaml +helix: + pipeline_name: "my-pipeline" + + source: + type: kafka + brokers: + - "kafka-broker-1.internal:9092" + - "kafka-broker-2.internal:9092" + topic: "events.raw" + consumer_group: "helix-prod" + + transform: + rules_file: "rules/transformations.yaml" + error_policy: dlq # dlq | skip | fail + + sink: + type: bigquery + project: "my-gcp-project" + dataset: "events" + table: "processed_events" + + checkpointing: + backend: redis + redis_url: "redis://redis.internal:6379/0" + interval_seconds: 30 + + metrics: + enabled: true + port: 9090 +``` + +--- + +## Usage + +### Starting the Pipeline + +```bash +helix run --config helix.yaml +``` + +### Running in Dry-Run Mode + +```bash +helix run --config helix.yaml --dry-run +``` + +Dry-run mode reads from the source, applies transformations, and logs outputs without writing to the sink. + +### Checking Pipeline Status + +```bash +helix status --config helix.yaml +``` + +### Managing Offsets + +```bash +# View current offsets +helix offsets show --config helix.yaml + +# Reset offsets to beginning +helix offsets reset --config helix.yaml --to earliest + +# Reset to specific timestamp +helix offsets reset --config helix.yaml --to "2024-01-01T00:00:00Z" +``` + +--- + +## API Reference + +Helix exposes a management REST API on port `8080` by default. + +### Health Check + +``` +GET /health +``` + +Returns `200 OK` with `{"status": "healthy"}` when the pipeline is running normally. + +### Metrics + +``` +GET /metrics +``` + +Prometheus metrics endpoint. + +### Pipeline Control + +``` +POST /api/v1/pipeline/pause +POST /api/v1/pipeline/resume +POST /api/v1/pipeline/checkpoint +``` + +Pause, resume, and force-checkpoint the running pipeline. Authentication required (see §Authentication). + +### DLQ Management + +``` +GET /api/v1/dlq/messages +POST /api/v1/dlq/replay +DELETE /api/v1/dlq/purge +``` + +View, replay, or purge dead-letter queue messages. All DLQ operations require `admin` scope. + +> **Note on API versioning:** The `/api/v1/` prefix is used by Helix 1.x. The current release (2.x) has migrated all management endpoints to `/api/v2/`. The `/api/v1/` prefix is deprecated and will be removed in Helix 3.0. Please update any integrations to use `/api/v2/pipeline/`, `/api/v2/dlq/`, etc. + +--- + +## Transformation DSL + +Helix transformations are defined in a YAML DSL. Each rule specifies a source field, an operation, and a target field. + +```yaml +rules: + - name: normalize_timestamp + source: event_time + op: parse_timestamp + format: "%Y-%m-%dT%H:%M:%SZ" + target: event_time_utc + + - name: enrich_user_tier + source: user_id + op: lookup + lookup_table: user_tiers + target: user_tier + on_miss: default + + - name: drop_test_events + source: environment + op: filter + condition: "value != 'test'" +``` + +Supported operations: `parse_timestamp`, `lookup`, `filter`, `rename`, `cast`, `hash`, `drop`. + +--- + +## Monitoring + +### Prometheus Metrics + +Key metrics exported by Helix: + +| Metric | Type | Description | +|--------|------|-------------| +| `helix_records_processed_total` | Counter | Total records successfully processed | +| `helix_records_failed_total` | Counter | Total records routed to DLQ | +| `helix_processing_latency_seconds` | Histogram | Per-record processing time | +| `helix_checkpoint_lag_seconds` | Gauge | Time since last successful checkpoint | +| `helix_consumer_lag_records` | Gauge | Kafka consumer lag (records behind) | + +### Recommended Alerts + +| Alert | Condition | +|-------|-----------| +| `HelixConsumerLagHigh` | `helix_consumer_lag_records > 100000` for 5m | +| `HelixCheckpointStale` | `helix_checkpoint_lag_seconds > 120` for 2m | +| `HelixHighErrorRate` | `rate(helix_records_failed_total[5m]) / rate(helix_records_processed_total[5m]) > 0.01` | + +--- + +## Troubleshooting + +### Pipeline fails to start: `ModuleNotFoundError` + +Ensure your virtual environment is activated and dependencies are installed: + +```bash +source .venv/bin/activate +pip install -r requirements.txt +``` + +### Kafka connection errors + +- Verify broker addresses are correct in `helix.yaml` +- Check that the consumer group `helix-prod` has read permissions on the source topic +- Test connectivity: `kafka-topics.sh --bootstrap-server kafka-broker-1.internal:9092 --list` + +### Redis connection errors + +- Verify Redis is running: `redis-cli -h redis.internal ping` +- Check that the Redis URL in config is correct + +### DLQ filling up + +Review the transformation logs for patterns in malformed records: + +```bash +helix logs --config helix.yaml --filter dlq --last 1h +``` + +--- + +## Contributing + +1. Fork the repository +2. Create a feature branch: `git checkout -b feature/my-change` +3. Make changes with tests +4. Run the test suite: `pytest tests/ -v` +5. Submit a pull request to `main` + +Code style: Black + isort. Run `make lint` before submitting. + +--- + +## License + +Internal use only. See `LICENSE.md` for terms. + +--- + +*Helix Pipeline README — Data Engineering Team — last meaningfully updated August 2024* diff --git a/golden-test-set/artifacts/docs/gt-doc-004-missing-sections.md b/golden-test-set/artifacts/docs/gt-doc-004-missing-sections.md new file mode 100644 index 0000000..8161037 --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-004-missing-sections.md @@ -0,0 +1,254 @@ +# Meridian Search API — Specification v1.2 + +**Service:** Search Service +**API Version:** v2 +**Base URL:** `https://api.meridian.internal/api/v2/search` +**Document Status:** Published +**Last Updated:** 2026-01-20 + +--- + +## Overview + +The Meridian Search API provides full-text and semantic search over indexed document collections. It supports keyword queries, vector similarity search, and hybrid retrieval modes. Results are ranked by relevance and may be filtered by metadata fields. + +This API is designed for internal consumers (other Meridian platform services) as well as authorized external integrators. + +--- + +## Endpoints + +### 1. Search Documents + +``` +POST /api/v2/search/query +``` + +Executes a search query over the specified collection. + +#### Request Body + +```json +{ + "collection_id": "string", + "query": "string", + "mode": "keyword | semantic | hybrid", + "top_k": 10, + "filters": { + "field": "string", + "operator": "eq | in | gte | lte", + "value": "string | number | array" + }, + "include_metadata": true, + "include_highlights": false +} +``` + +#### Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `collection_id` | string | Yes | Identifier of the collection to search | +| `query` | string | Yes | The search query string | +| `mode` | enum | No | Retrieval mode. Default: `hybrid` | +| `top_k` | integer | No | Number of results to return. Default: 10, max: 100 | +| `filters` | object | No | Metadata field filter | +| `include_metadata` | boolean | No | Include document metadata in results. Default: `true` | +| `include_highlights` | boolean | No | Include matched text snippets. Default: `false` | + +#### Response Body (200 OK) + +```json +{ + "query_id": "q_01HXYZ123", + "collection_id": "my-collection", + "total_results": 247, + "results": [ + { + "doc_id": "doc_abc123", + "score": 0.921, + "title": "Introduction to RAG Systems", + "excerpt": "Retrieval-augmented generation combines...", + "metadata": { + "author": "Jane Smith", + "created_at": "2025-11-14T09:00:00Z", + "tags": ["AI", "retrieval", "enterprise"] + } + } + ], + "latency_ms": 47, + "mode_used": "hybrid" +} +``` + +--- + +### 2. Get Collection Info + +``` +GET /api/v2/search/collections/{collection_id} +``` + +Returns metadata about a specific collection. + +#### Path Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `collection_id` | string | Yes | Unique identifier for the collection | + +#### Response Body (200 OK) + +```json +{ + "collection_id": "my-collection", + "display_name": "Product Documentation", + "document_count": 14820, + "index_status": "ready", + "last_indexed_at": "2026-01-19T22:00:00Z", + "embedding_model": "text-embedding-3-large", + "created_at": "2025-06-01T00:00:00Z" +} +``` + +--- + +### 3. List Collections + +``` +GET /api/v2/search/collections +``` + +Returns all collections accessible to the authenticated caller. + +#### Query Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `page` | integer | No | Page number (1-indexed). Default: 1 | +| `page_size` | integer | No | Results per page. Default: 20, max: 100 | +| `status` | string | No | Filter by index status: `ready`, `indexing`, `error` | + +#### Response Body (200 OK) + +```json +{ + "collections": [ + { + "collection_id": "my-collection", + "display_name": "Product Documentation", + "document_count": 14820, + "index_status": "ready" + } + ], + "total": 5, + "page": 1, + "page_size": 20 +} +``` + +--- + +### 4. Suggest (Autocomplete) + +``` +GET /api/v2/search/suggest +``` + +Returns autocomplete suggestions based on a partial query string. + +#### Query Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `collection_id` | string | Yes | Collection to generate suggestions from | +| `prefix` | string | Yes | Partial query to complete | +| `max_suggestions` | integer | No | Max suggestions to return. Default: 5, max: 20 | + +#### Response Body (200 OK) + +```json +{ + "prefix": "retrie", + "suggestions": [ + "retrieval augmented generation", + "retrieval pipelines", + "retrieval benchmarks" + ] +} +``` + +--- + +## Request / Response Headers + +### Request Headers + +| Header | Required | Description | +|--------|----------|-------------| +| `Content-Type` | Yes (POST) | Must be `application/json` | +| `Accept` | No | Default: `application/json` | +| `X-Request-ID` | No | Client-supplied idempotency/tracing ID. Echoed in response. | + +### Response Headers + +| Header | Description | +|--------|-------------| +| `X-Request-ID` | Echoed from request, or generated if absent | +| `X-Trace-ID` | Internal distributed trace ID | +| `X-RateLimit-Limit` | Your rate limit ceiling | +| `X-RateLimit-Remaining` | Remaining requests in current window | +| `X-RateLimit-Reset` | Unix timestamp when limit resets | + +--- + +## Data Types + +### SearchMode + +| Value | Description | +|-------|-------------| +| `keyword` | BM25 sparse retrieval only | +| `semantic` | Dense vector search only | +| `hybrid` | Reciprocal rank fusion of keyword + semantic results | + +### IndexStatus + +| Value | Description | +|-------|-------------| +| `ready` | Collection is fully indexed and queryable | +| `indexing` | Index update in progress; queries will use previous index version | +| `error` | Index is in a degraded state; partial results may be returned | + +--- + +## SDK and Integration + +The official Python SDK wraps this API: + +```python +from meridian_sdk import SearchClient + +client = SearchClient(api_key="your-api-key") +results = client.search( + collection_id="my-collection", + query="RAG architectures for enterprise", + mode="hybrid", + top_k=10 +) + +for doc in results.documents: + print(f"{doc.title}: {doc.score:.3f}") +``` + +The SDK handles authentication, retries with exponential backoff, and response deserialization automatically. + +--- + +## Versioning + +The API follows semantic versioning. The current stable version is `v2`. Version `v1` is deprecated and scheduled for removal in Q3 2026. + +--- + +*Meridian Search API Specification — Platform Team — January 2026* diff --git a/golden-test-set/artifacts/docs/gt-doc-005-research-gaps.md b/golden-test-set/artifacts/docs/gt-doc-005-research-gaps.md new file mode 100644 index 0000000..97ec774 --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-005-research-gaps.md @@ -0,0 +1,180 @@ +# Conversational AI Adoption in Enterprise Workflows: A Research Report + +**Report Type:** Primary Research with Literature Synthesis +**Commissioned by:** Digital Transformation Office +**Authors:** Research Methods Team — K. Oladipo (lead), M. Svensson, P. Chakraborty +**Version:** 2.0 (Final) +**Date:** 2026-02-01 +**Distribution:** Internal — restricted to VP+ and authorized project leads + +--- + +## Executive Summary + +This report synthesizes findings from a mixed-methods research program examining the adoption of conversational AI assistants in enterprise knowledge work. The program ran from June 2025 through December 2025 and included survey data from 847 knowledge workers across 6 organizations, 32 semi-structured interviews, and a systematic literature review of 54 peer-reviewed sources. + +**Key findings:** + +1. Conversational AI assistants increased knowledge worker task completion speed by an average of 23% on information retrieval tasks, with high individual variability (range: −12% to +61%). + +2. Trust calibration — workers' ability to correctly gauge when AI outputs are reliable — was the strongest predictor of productivity gain, explaining 34% of variance in performance improvement. + +3. Organizations that invested in structured onboarding for AI tools saw 2.3× higher sustained adoption at 6 months compared to organizations that deployed without onboarding. + +--- + +## 1. Introduction + +### 1.1 Background + +The rapid commercialization of large language model (LLM)-based assistants beginning in 2023 created significant pressure on enterprise organizations to evaluate and adopt these tools. Unlike prior generations of enterprise AI (process automation, anomaly detection), conversational AI operates in the unstructured knowledge work domain — a space characterized by ambiguity, judgment, and contextual expertise. + +This report addresses a gap in the literature: while much has been written about LLM capabilities in isolation, relatively little rigorous research exists on conversational AI adoption dynamics within enterprise organizational contexts. + +### 1.2 Research Questions + +This research program was designed to answer three questions: + +**RQ1:** How does conversational AI assistance affect knowledge worker task performance (speed and quality) on domain-specific information retrieval and synthesis tasks? + +**RQ2:** What organizational and individual-level factors moderate the relationship between AI adoption and performance outcomes? + +**RQ3:** How do enterprise security and data governance requirements shape the design of AI deployment architectures, and what are the downstream effects on adoption? + +### 1.3 Scope and Limitations + +This report covers knowledge work in corporate environments. It does not address: + +- Consumer-facing AI adoption +- AI adoption in regulated clinical or legal settings +- Hardware-accelerated inference infrastructure + +--- + +## 2. Methodology + +### 2.1 Research Design + +This study employed a convergent mixed-methods design: + +- **Phase 1 (quantitative):** Cross-sectional survey of 847 knowledge workers across 6 participating organizations (technology, financial services, consulting, and manufacturing sectors). Survey measured AI tool usage frequency, trust calibration (via validated instrument), task performance self-report, and demographic controls. + +- **Phase 2 (qualitative):** 32 semi-structured interviews, purposively sampled for variation in adoption level (high, medium, low adopters), organizational role (IC, manager, executive), and sector. + +- **Phase 3 (literature synthesis):** Systematic search of ACM Digital Library, Semantic Scholar, and ABI/Inform for peer-reviewed research on AI adoption in enterprise contexts, published 2020–2025. 54 sources retained after title/abstract screening and full-text review. + +### 2.2 Survey Instrument + +The survey included the following validated instruments: + +- **AI Trust Calibration Scale (ATCS):** 12-item Likert scale measuring alignment between perceived and actual AI reliability. Adapted from Lee & See (2004) with LLM-specific modifications validated in pilot (n=42). +- **Task Performance Index (TPI):** 8-item behavioral frequency scale measuring self-reported AI tool integration in work tasks. +- **Organizational AI Readiness (OAIR):** 6-item manager-rated scale measuring team-level infrastructure, norms, and training for AI adoption. + +### 2.3 Analysis + +Quantitative data were analyzed using hierarchical linear regression (individual-level factors nested within organizational units). Qualitative data were analyzed using thematic analysis following Braun & Clarke (2006). Mixed-methods integration used a joint display approach. + +--- + +## 3. Findings + +### 3.1 Task Performance Effects (RQ1) + +Knowledge workers using conversational AI assistants showed a mean task completion speed improvement of 23.1% (SD = 18.4%, 95% CI [21.3%, 24.9%]) on standardized information retrieval and synthesis tasks. + +Effect magnitude varied substantially by task type: + +| Task Type | Mean Speed Improvement | Quality Change | +|-----------|----------------------|----------------| +| Literature search and summarization | +38.2% | +0.4 quality rating pts | +| Structured document drafting | +27.1% | +0.2 pts | +| Code explanation / documentation | +31.4% | +0.7 pts | +| Complex analysis with judgment calls | +8.3% | −0.1 pts | +| Novel problem-solving | +4.1% | −0.3 pts | + +The negative quality outcomes for complex analysis and novel problem-solving are consistent with prior literature on automation complacency (Parasuraman & Manzey, 2010): workers who over-relied on AI outputs for tasks requiring judgment showed measurable quality degradation. + +### 3.2 Moderating Factors + +Trust calibration (ATCS) was the strongest individual-level predictor of performance improvement. Workers in the top quartile of trust calibration showed a mean improvement of 41.3%, compared to 9.2% for the bottom quartile (F(3,843) = 47.2, p < .001). + +Organizational-level factors showed significant cross-level moderation. Teams with high OAIR scores — indicating structured onboarding, clear usage norms, and IT infrastructure support — showed 2.3× higher AI tool retention at 6 months compared to teams with low OAIR scores (χ² = 31.4, df = 2, p < .001). + +Additional moderating factors explored in the data included: + +- Role type (manager vs. IC vs. executive) +- Sector (technology workers showed highest baseline adoption) +- Years of experience with AI tools +- Organizational AI policy restrictiveness + +### 3.3 Governance and Architecture Effects + +Data governance requirements — particularly restrictions on sending proprietary information to external API endpoints — were the most frequently cited adoption barrier in interviews (mentioned by 26 of 32 interviewees). Organizations with on-premises or private-cloud deployments reported significantly fewer friction events in the adoption process. + +The quantitative data also showed a significant negative correlation between perceived data governance friction and OAIR scores (r = −0.41, p < .001), suggesting that organizations with mature AI readiness had resolved or mitigated governance friction before it affected frontline adoption. + +--- + +## 5. Discussion + +### 5.1 Theoretical Contributions + +This study contributes to the human-computer interaction literature by demonstrating that trust calibration — not trust level per se — is the critical moderator of AI-assisted performance. This distinction is theoretically important: high AI trust is not inherently beneficial; it is beneficial only when calibrated to actual AI reliability. + +The finding extends prior automation trust research (Lee & See, 2004; Parasuraman & Manzey, 2010) to the conversational AI domain, and is the first empirical demonstration of this effect at organizational scale. + +### 5.2 Practical Implications + +For enterprise AI deployment, these findings suggest: + +1. **Prioritize trust calibration training over technology training.** Organizations should invest in helping workers understand *when* AI is reliable, not just *how* to use it. + +2. **Structured onboarding is not optional.** The 2.3× difference in 6-month retention between high- and low-OAIR organizations is substantial and practically significant. + +3. **Governance friction is a first-order adoption barrier.** Data governance design should be resolved before deployment, not retrofitted. + +### 5.3 Relationship to Gap Analysis Findings + +As noted in Section 4, the gap analysis revealed several areas where our findings diverge from prior literature expectations. In particular, the magnitude of the trust calibration effect exceeds predictions from prior automation research. Possible explanations include the novelty of conversational AI as a tool class and the absence of domain-specific prior work. + +--- + +## 6. Conclusion + +Conversational AI adoption in enterprise settings is real, measurable, and highly variable. The variability is the central finding: AI assistance produces a wide range of outcomes depending on trust calibration, organizational readiness, and task type. Organizations that treat AI deployment as a technology problem without attending to the human and organizational factors will see diminishing returns. + +--- + +## References + +Braun, V., & Clarke, V. (2006). Using thematic analysis in psychology. *Qualitative Research in Psychology*, 3(2), 77–101. + +Lee, J. D., & See, K. A. (2004). Trust in automation: Designing for appropriate reliance. *Human Factors*, 46(1), 50–80. + +Parasuraman, R., & Manzey, D. H. (2010). Complacency and bias in human use of automation: An attentive review. *Human Factors*, 52(3), 381–410. + +--- + +## Appendix A: Survey Instruments (Abbreviated) + +**AI Trust Calibration Scale (ATCS) — Sample Items:** + +1. "I know which types of tasks this AI tool handles well and which it handles poorly." +2. "When I'm uncertain whether to trust an AI output, I know what to check." +3. "I have been surprised by AI errors that I should have anticipated." + +--- + +## Appendix B: Interview Protocol + +Semi-structured interview guide topics: + +1. Describe your current use of AI tools in your daily work. +2. Can you recall a time when the AI output was misleading or wrong? What happened? +3. How did your organization support you in learning to use the AI tools? +4. What policies or restrictions affect how you use AI at work? + +--- + +*This report was prepared for internal use. Citation of findings requires written approval from the Digital Transformation Office. Contact: research-ops@corp.internal* diff --git a/golden-test-set/artifacts/docs/gt-doc-006-poor-structure.md b/golden-test-set/artifacts/docs/gt-doc-006-poor-structure.md new file mode 100644 index 0000000..91cdc9c --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-006-poor-structure.md @@ -0,0 +1,321 @@ +# Helix Pipeline — Developer Guide + +**Project:** Helix Data Processing Pipeline +**Audience:** Backend engineers onboarding to the Helix codebase +**Status:** Living document + +--- + +# Getting Started + +Welcome to the Helix developer guide. This document covers everything you need to get Helix running locally, understand the codebase structure, and contribute changes. + +### Prerequisites + +Before you begin, ensure the following are installed on your development machine: + +- Docker Desktop 4.0+ +- Python 3.11+ +- Git 2.30+ +- Make + +### Installation Steps + +**Step 1: Clone the repository.** + +```bash +git clone https://internal.git.corp/data-eng/helix-pipeline.git +cd helix-pipeline +``` + +**Step 2: Set up the Python virtual environment.** + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +pip install -r requirements-dev.txt +``` + +**Step 3: Start local dependencies with Docker Compose.** + +```bash +docker-compose up -d kafka redis +``` + +This starts local instances of Kafka (port 9092) and Redis (port 6379). + +**Step 4: Copy the example config and edit for your environment.** + +```bash +cp config/helix.example.yaml config/helix.local.yaml +# Edit helix.local.yaml with your local settings +``` + +**Step 5: Run the pipeline in dry-run mode to verify the setup.** + +```bash +helix run --config config/helix.local.yaml --dry-run +``` + +You should see log output confirming connection to Kafka and Redis, followed by a summary of what would be processed. + +--- + +##### Codebase Structure + +``` +helix-pipeline/ +├── helix/ # Main Python package +│ ├── core/ # Pipeline orchestration +│ ├── connectors/ # Source/sink connectors +│ ├── transforms/ # Transformation engine +│ ├── checkpointing/ # Offset and state management +│ └── metrics/ # Prometheus exporter +├── config/ # Configuration files +├── tests/ # Test suite +├── scripts/ # Operational scripts +├── docker/ # Dockerfiles +└── docs/ # Additional documentation +``` + +--- + +## Architecture Overview + +Helix follows a plugin architecture. The core pipeline (`helix/core/`) is responsible for orchestrating the flow of data from sources to sinks. The specifics of how data is read (connectors), transformed (transforms), and written (sinks) are implemented as plugins loaded via entrypoints. + +The pipeline runs as a main loop: + +1. **Source connector** reads a batch of records from the configured source (Kafka, S3, etc.) +2. **Transformer chain** applies transformation rules in sequence +3. **Sink connector** writes the transformed records to the configured destination +4. **Checkpointer** records the consumed offsets after successful sink write + +If any step fails, the error policy determines behavior: `fail` (stop pipeline), `skip` (log and continue), or `dlq` (route to dead-letter queue). + +--- + +##### Running Tests + +Helix has three test layers: + +**Unit tests** — fast, no external dependencies: + +```bash +pytest tests/unit/ -v +``` + +**Integration tests** — require running Kafka and Redis: + +```bash +docker-compose up -d kafka redis +pytest tests/integration/ -v +``` + +**End-to-end tests** — full pipeline runs with fixture data: + +```bash +pytest tests/e2e/ -v --timeout=120 +``` + +Run the full suite: + +```bash +make test +``` + +Coverage report: + +```bash +make coverage +``` + +Current coverage target: 85% line coverage for the `helix/` package. + +--- + +## Configuration Reference + +Below is the full configuration reference for `helix.yaml`. + +### Top-Level Keys + +```yaml +helix: + pipeline_name: string # Required. Human-readable name. + log_level: debug|info|warning # Default: info + source: + transform: + sink: + checkpointing: + metrics: +``` + +### Source Configuration + +```yaml +source: + type: kafka # Required. kafka | s3 | gcs + brokers: # Required for kafka + - "broker-host:9092" + topic: string # Required for kafka + consumer_group: string # Required for kafka + batch_size: 500 # Default: 500 + poll_timeout_ms: 1000 # Default: 1000 +``` + +### Transform Configuration + +```yaml +transform: + rules_file: path/to/rules.yaml # Required + error_policy: dlq # Default: dlq. Options: dlq | skip | fail +``` + +### Sink Configuration + +```yaml +sink: + type: bigquery # Required. bigquery | postgres | s3 + project: string # Required for bigquery + dataset: string # Required for bigquery + table: string # Required for bigquery + write_mode: append # Default: append. Options: append | upsert +``` + +### Checkpointing Configuration + +```yaml +checkpointing: + backend: redis # Required. redis (only option currently) + redis_url: string # Required + interval_seconds: 30 # Default: 30 +``` + +### Metrics Configuration + +```yaml +metrics: + enabled: true # Default: true + port: 9090 # Default: 9090 + path: /metrics # Default: /metrics +``` + +--- + +## Setting Up Your Development Environment + +To contribute to Helix, you'll need to set up your development environment properly. Here is how to do that. + +First, clone the repository: + +```bash +git clone https://internal.git.corp/data-eng/helix-pipeline.git +cd helix-pipeline +``` + +Next, create and activate a Python virtual environment: + +```bash +python3 -m venv .venv +source .venv/bin/activate +``` + +Install both runtime and development dependencies: + +```bash +pip install -r requirements.txt +pip install -r requirements-dev.txt +``` + +Start the required services using Docker Compose: + +```bash +docker-compose up -d kafka redis +``` + +Now you can verify your setup by running the test suite: + +```bash +make test +``` + +If all tests pass, you are ready to begin development. + +--- + +## Writing a Custom Connector + +To add a new source or sink connector, create a new module in `helix/connectors/` and register it as an entrypoint. + +### Source Connector Interface + +```python +from helix.core.interfaces import SourceConnector +from typing import Iterator, List +from helix.core.models import Record + +class MySourceConnector(SourceConnector): + def __init__(self, config: dict): + self.config = config + + def connect(self) -> None: + """Initialize connection to the source.""" + ... + + def read_batch(self) -> List[Record]: + """Read and return the next batch of records.""" + ... + + def commit(self, records: List[Record]) -> None: + """Mark records as consumed (advance offset/checkpoint).""" + ... + + def close(self) -> None: + """Clean up resources.""" + ... +``` + +Register your connector in `setup.py` or `pyproject.toml`: + +```toml +[project.entry-points."helix.connectors.source"] +my_source = "helix.connectors.my_source:MySourceConnector" +``` + +--- + +## Local Development Setup (Quick Reference) + +For developers who just need the quick version: + +```bash +git clone https://internal.git.corp/data-eng/helix-pipeline.git +cd helix-pipeline +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt -r requirements-dev.txt +docker-compose up -d kafka redis +make test +``` + +This is the same process as the full setup above, condensed for convenience. + +--- + +## Contributing + +We follow a standard trunk-based development workflow: + +1. Create a feature branch from `main` +2. Make changes, add tests +3. Run `make lint` (Black + isort + mypy) +4. Run `make test` +5. Open a pull request; require 1 approval from a CODEOWNER +6. Merge to `main` on approval + +Commit message format: `(): ` (Conventional Commits). + +--- + +*Helix Developer Guide — Data Engineering Team* diff --git a/golden-test-set/artifacts/docs/gt-doc-007-minor-style.md b/golden-test-set/artifacts/docs/gt-doc-007-minor-style.md new file mode 100644 index 0000000..b4cc70f --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-007-minor-style.md @@ -0,0 +1,262 @@ +# Meridian SDK for Python — Quick Start Guide + +**Version:** 2.3.0 +**Python:** 3.10+ +**Status:** Stable + +--- + +## Overview + +The Meridian SDK for Python provides a high-level client for the Meridian Search and Indexing APIs. It handles authentication, retries, pagination, and response deserialization, so you can focus on building rather than managing HTTP boilerplate. + +--- + +## Installation + +Install from the internal package registry: + +```bash +pip install meridian-sdk +``` + +Or with optional dependencies for async support: + +```bash +pip install "meridian-sdk[async]" +``` + +--- + +## Quick Start + +```python +from meridian_sdk import SearchClient + +# Initialize the client with your API key +client = SearchClient(api_key="your-api-key") + +# Run a search +results = client.search( + collection_id="product-docs", + query="configuring rate limits", + top_k=5 +) + +for doc in results.documents: + print(f"[{doc.score:.3f}] {doc.title}") +``` + +--- + +## Authentication + +The SDK supports two authentication methods: + +### API Key + +Pass your API key directly to the client constructor: + +```python +client = SearchClient(api_key="your-api-key") +``` + +Alternatively, set the `MERIDIAN_API_KEY` environment variable and the SDK will pick it up automatically: + +```bash +export MERIDIAN_API_KEY=your-api-key +``` + +```python +client = SearchClient() # reads from environment +``` + +### Service Account Token + +For service-to-service integrations, use a service account credential file: + +```python +client = SearchClient.from_service_account("/path/to/credentials.json") +``` + +--- + +## Search + +### Basic Search + +```python +results = client.search( + collection_id="my-collection", + query="machine learning inference", + top_k=10 +) +``` + +### Filtered Search + +```python +results = client.search( + collection_id="my-collection", + query="security vulnerabilities", + filters={ + "field": "tags", + "operator": "in", + "value": ["security", "cve"] + } +) +``` + +### Search Modes + +The SDK supports three search modes: + +- `hybrid` (default) — combines keyword and semantic search for best results +- `keyword` — BM25 sparse retrieval only; fastest, best for exact-match queries +- `semantic` — dense vector search only; best for conceptual similarity + +```python +results = client.search( + collection_id="my-collection", + query="authentication token expiration", + mode="semantic" +) +``` + +--- + +## Collections + +### List Collections + +```python +collections = client.list_collections() +for c in collections: + print(f"{c.collection_id}: {c.document_count} documents ({c.index_status})") +``` + +### Get a Collection + +```python +collection = client.get_collection("product-docs") +print(collection.last_indexed_at) +``` + +--- + +## Pagination + +Search results and collection listings support pagination: + +```python +page = client.search( + collection_id="my-collection", + query="deployment", + top_k=20, + page=1 +) + +while page.has_next: + page = page.next_page() + for doc in page.documents: + print(doc.title) +``` + +--- + +## Error Handling + +The SDK raises typed exceptions for API errors: + +```python +from meridian_sdk.exceptions import ( + AuthenticationError, + CollectionNotFoundError, + RateLimitError, + MeridianAPIError +) + +try: + results = client.search(collection_id="nonexistent", query="test") +except CollectionNotFoundError as e: + print(f"Collection not found: {e.collection_id}") +except RateLimitError as e: + print(f"Rate limited. Retry after {e.retry_after_seconds}s") +except MeridianAPIError as e: + print(f"API error {e.status_code}: {e.message}") +``` + +--- + +## Async Support + +Install with async extras and use `AsyncSearchClient`: + +```python +import asyncio +from meridian_sdk.async_client import AsyncSearchClient + +async def main(): + async with AsyncSearchClient(api_key="your-api-key") as client: + results = await client.search( + collection_id="product-docs", + query="async patterns" + ) + for doc in results.documents: + print(doc.title) + +asyncio.run(main()) +``` + +--- + +## Configuration + +The client accepts a configuration object for advanced settings: + +```python +from meridian_sdk import SearchClient, ClientConfig + +config = ClientConfig( + timeout_seconds=30, + max_retries=3, + retry_backoff_factor=1.5, + base_url="https://api.meridian.internal" +) + +client = SearchClient(api_key="your-api-key", config=config) +``` + +### Configuration Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `timeout_seconds` | int | 30 | Request timeout | +| `max_retries` | int | 3 | Max retry attempts on 5xx or network errors | +| `retry_backoff_factor` | float | 1.5 | Exponential backoff multiplier | +| `base_url` | str | Production URL | Override API base URL | + +--- + +## Logging + +The SDK uses Python's standard `logging` module under the `meridian_sdk` logger namespace: + +```python +import logging +logging.getLogger("meridian_sdk").setLevel(logging.DEBUG) +``` + +--- + +## What's New in 2.3.0 + +* Added `AsyncSearchClient` for async/await usage +- Pagination support via `.next_page()` on result objects +* `ClientConfig` class for centralized configuration +- Improved error messages with structured exception types +* Performance improvements: connection pooling enabled by default + +--- + +*Meridian SDK for Python — Platform Team — v2.3.0* diff --git a/golden-test-set/artifacts/docs/gt-doc-008-minor-completeness.md b/golden-test-set/artifacts/docs/gt-doc-008-minor-completeness.md new file mode 100644 index 0000000..cfb1656 --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-008-minor-completeness.md @@ -0,0 +1,306 @@ +# Meridian Indexing API — Reference v1.0 + +**Service:** Indexing Service +**API Version:** v2 +**Base URL:** `https://api.meridian.internal/api/v2/index` +**Status:** Stable +**Last Updated:** 2026-01-30 + +--- + +## Overview + +The Meridian Indexing API allows you to create and manage document collections, ingest documents for indexing, and monitor indexing progress. Documents ingested through this API become searchable via the Search API. + +--- + +## Authentication + +All requests require a valid API key passed in the `X-API-Key` header, or a JWT Bearer token in the `Authorization` header. See the Authentication Guide for details on obtaining credentials. + +``` +X-API-Key: your-api-key +``` + +or: + +``` +Authorization: Bearer +``` + +--- + +## Endpoints + +### 1. Create Collection + +``` +POST /api/v2/index/collections +``` + +Creates a new document collection. Collections must be created before documents can be ingested. + +#### Request Body + +```json +{ + "collection_id": "string", + "display_name": "string", + "embedding_model": "text-embedding-3-large", + "chunk_size": 512, + "chunk_overlap": 102, + "metadata_schema": { + "author": "string", + "created_at": "datetime", + "tags": "array" + } +} +``` + +#### Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `collection_id` | string | Yes | Unique ID for the collection. Lowercase alphanumeric + hyphens, max 64 chars. | +| `display_name` | string | Yes | Human-readable name for the collection. | +| `embedding_model` | string | No | Embedding model to use. Default: `text-embedding-3-large` | +| `chunk_size` | integer | No | Token chunk size for document splitting. Default: 512. Range: 128–2048. | +| `chunk_overlap` | integer | No | Token overlap between chunks. Default: 102 (20% of 512). | +| `metadata_schema` | object | No | Optional schema for document metadata fields. Used for filter validation. | + +#### Response Body (201 Created) + +```json +{ + "collection_id": "my-new-collection", + "display_name": "Product Documentation", + "status": "created", + "created_at": "2026-01-30T14:00:00Z", + "embedding_model": "text-embedding-3-large" +} +``` + +#### Error Responses + +| Status | Code | Description | +|--------|------|-------------| +| 400 | `INVALID_COLLECTION_ID` | collection_id contains invalid characters | +| 409 | `COLLECTION_EXISTS` | A collection with this ID already exists | +| 422 | `INVALID_SCHEMA` | metadata_schema contains unsupported field types | + +--- + +### 2. Ingest Documents + +``` +POST /api/v2/index/collections/{collection_id}/documents +``` + +Ingests one or more documents into the specified collection. Documents are processed asynchronously; this endpoint returns immediately with a job ID. + +#### Path Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `collection_id` | string | Yes | ID of the target collection | + +#### Request Body + +```json +{ + "documents": [ + { + "doc_id": "string", + "title": "string", + "content": "string", + "metadata": { + "author": "Jane Smith", + "created_at": "2026-01-15T10:00:00Z", + "tags": ["AI", "enterprise"] + } + } + ], + "upsert": true +} +``` + +#### Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `documents` | array | Yes | Array of document objects. Max 100 per request. | +| `documents[].doc_id` | string | Yes | Unique document ID within the collection. | +| `documents[].title` | string | Yes | Document title. Used in search result display. | +| `documents[].content` | string | Yes | Full document text content. Max 1MB per document. | +| `documents[].metadata` | object | No | Key-value metadata. Must conform to collection schema if defined. | +| `upsert` | boolean | No | If true, update existing documents with matching doc_id. Default: `false`. | + +#### Response Body (202 Accepted) + +```json +{ + "job_id": "job_01HXYZ456", + "collection_id": "my-new-collection", + "document_count": 5, + "status": "queued", + "estimated_completion_seconds": 30 +} +``` + +--- + +### 3. Get Ingestion Job Status + +``` +GET /api/v2/index/jobs/{job_id} +``` + +Returns the status of an asynchronous ingestion job. + +#### Path Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `job_id` | string | Yes | Job ID returned by the ingest endpoint | + +#### Response Body (200 OK) + +```json +{ + "job_id": "job_01HXYZ456", + "collection_id": "my-new-collection", + "status": "completed", + "total_documents": 5, + "indexed_documents": 5, + "failed_documents": 0, + "started_at": "2026-01-30T14:00:05Z", + "completed_at": "2026-01-30T14:00:28Z", + "errors": [] +} +``` + +Job status values: `queued`, `processing`, `completed`, `failed`, `partial_failure` + +--- + +### 4. Delete Document + +``` +DELETE /api/v2/index/collections/{collection_id}/documents/{doc_id} +``` + +Removes a single document from the collection and its index. + +#### Path Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `collection_id` | string | Yes | Collection containing the document | +| `doc_id` | string | Yes | Document to delete | + +#### Response Body (200 OK) + +```json +{ + "doc_id": "doc_abc123", + "collection_id": "my-new-collection", + "status": "deleted" +} +``` + +#### Error Responses + +| Status | Code | Description | +|--------|------|-------------| +| 404 | `DOCUMENT_NOT_FOUND` | No document with this ID in the collection | + +--- + +### 5. Delete Collection + +``` +DELETE /api/v2/index/collections/{collection_id} +``` + +Permanently deletes a collection and all its documents. This action is irreversible. + +#### Path Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `collection_id` | string | Yes | Collection to delete | + +#### Query Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `confirm` | boolean | Yes | Must be `true` to proceed. Safety guard. | + +#### Response Body (200 OK) + +```json +{ + "collection_id": "my-old-collection", + "status": "deleted", + "documents_removed": 14820 +} +``` + +--- + +## Rate Limits + +| Tier | Ingest Limit | Management Limit | +|------|-------------|-----------------| +| Standard | 100 documents/minute | 60 requests/minute | +| Enterprise | 5,000 documents/minute | 300 requests/minute | + +When rate limited, the API returns `HTTP 429` with `Retry-After` header. + +--- + +## Webhooks + +Optionally, receive notifications when indexing jobs complete: + +```json +{ + "webhook_url": "https://your-service.example.com/webhooks/meridian", + "events": ["job.completed", "job.failed"] +} +``` + +Configure webhooks via the Admin API or the Meridian Console. + +--- + +## SDK Example + +```python +from meridian_sdk import IndexingClient + +client = IndexingClient(api_key="your-api-key") + +# Create a collection +collection = client.create_collection( + collection_id="my-docs", + display_name="My Documentation" +) + +# Ingest documents +job = client.ingest_documents( + collection_id="my-docs", + documents=[ + {"doc_id": "doc-001", "title": "Getting Started", "content": "..."}, + {"doc_id": "doc-002", "title": "Configuration", "content": "..."}, + ] +) + +# Wait for completion +result = client.wait_for_job(job.job_id, poll_interval_seconds=5) +print(f"Indexed {result.indexed_documents} documents") +``` + +--- + +*Meridian Indexing API Reference — Platform Team — January 2026* diff --git a/golden-test-set/artifacts/docs/gt-doc-009-clean-spec.md b/golden-test-set/artifacts/docs/gt-doc-009-clean-spec.md new file mode 100644 index 0000000..b9ecd99 --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-009-clean-spec.md @@ -0,0 +1,340 @@ +# Meridian Event Bus — Technical Specification v0.1.0 + +**System:** Meridian Platform +**Component:** Event Bus (pub/sub backbone) +**Spec Version:** 0.1.0 +**Status:** Draft — Accepted for implementation +**Author:** Platform Architecture Team +**Created:** 2026-02-10 +**Last Updated:** 2026-02-24 +**Review Cycle:** On major version bump or quarterly + +--- + +## Table of Contents + +1. [Overview](#1-overview) +2. [Goals and Non-Goals](#2-goals-and-non-goals) +3. [Architecture](#3-architecture) +4. [Event Schema](#4-event-schema) +5. [Topic Design](#5-topic-design) +6. [Producer Specification](#6-producer-specification) +7. [Consumer Specification](#7-consumer-specification) +8. [Delivery Semantics](#8-delivery-semantics) +9. [Performance Requirements](#9-performance-requirements) +10. [Security](#10-security) +11. [Known Limitations](#11-known-limitations) +12. [Open Questions](#12-open-questions) +13. [Revision History](#13-revision-history) + +--- + +## 1. Overview + +The Meridian Event Bus provides asynchronous, durable, ordered pub/sub messaging between Meridian platform services. It decouples producers from consumers, enabling services to evolve independently and absorb load spikes without cascading failures. + +The Event Bus is built on **Apache Kafka 3.6** (KRaft mode, no ZooKeeper dependency). All producers and consumers use the official Kafka client libraries; the Event Bus does not introduce a proprietary abstraction layer over Kafka primitives. + +### 1.1 Scope + +This specification covers: + +- Event schema and validation +- Topic naming conventions and partition strategy +- Producer and consumer behavioral contracts +- Delivery guarantee semantics +- Performance targets +- Security model (authentication, authorization, encryption) + +This specification does not cover: + +- Individual service business logic +- Schema evolution policies beyond v0.1.0 scope (deferred to v0.2.0 spec) +- Infrastructure provisioning (see Terraform module `meridian-kafka`) +- Kafka cluster operations (see Runbook: Kafka Operations) + +### 1.2 Relationship to Other Components + +| Component | Relationship | +|-----------|-------------| +| Indexing Service | Producer of `document.ingested` events | +| Search Service | Consumer of `index.updated` events | +| Gateway | Producer of `request.completed` events (analytics path) | +| Audit Service | Consumer of all security-relevant event topics | + +--- + +## 2. Goals and Non-Goals + +### Goals + +- **Durability:** Events are retained for a configurable period (default: 7 days). A consumer restart does not lose events. +- **Ordering:** Within a partition, events are strictly ordered by produce time. The spec defines partitioning rules to ensure ordering where it matters. +- **Observability:** All producers and consumers emit standard Prometheus metrics enabling lag monitoring and alerting. +- **Schema enforcement:** All events are validated against a JSON Schema at produce time. Malformed events are rejected with a structured error. +- **Independent deployability:** Services that produce or consume events do not need to coordinate releases. + +### Non-Goals + +- **Exactly-once across distributed transactions:** The Event Bus guarantees at-least-once delivery. Consumers are responsible for idempotent processing. +- **Sub-millisecond latency:** The Event Bus is designed for throughput, not ultra-low-latency RPC. Use gRPC for synchronous latency-sensitive calls. +- **Cross-cluster replication:** Out of scope for v0.1.0. + +--- + +## 3. Architecture + +### 3.1 Cluster Topology + +The Meridian Event Bus runs as a dedicated Kafka cluster, separate from any application infrastructure: + +- **Brokers:** 3 broker nodes, each on a dedicated VM (8-core, 32GB RAM, 2TB NVMe SSD) +- **Replication factor:** 3 (all topics) +- **Minimum in-sync replicas:** 2 (producers configured with `acks=all`) +- **KRaft controllers:** Co-located with broker nodes (combined mode for v0.1.0; dedicated controller nodes planned for v0.2.0) + +### 3.2 Client Connectivity + +All services connect via an internal load balancer endpoint: `kafka.meridian.internal:9093` (TLS port). Direct broker connections are not allowed; all traffic routes through the load balancer. + +### 3.3 Schema Registry + +A Confluent Schema Registry instance is deployed alongside the cluster. All event schemas are registered and versioned in the registry. The registry enforces compatibility modes per topic (see §4.3). + +--- + +## 4. Event Schema + +### 4.1 Envelope + +All events share a common envelope wrapping the payload: + +```json +{ + "event_id": "evt_01JXXXXXXXXX", + "event_type": "document.ingested", + "event_version": "1.0", + "source_service": "indexing-service", + "timestamp": "2026-02-10T14:30:00.000Z", + "correlation_id": "req_01JXXXXXXXXX", + "payload": { ... } +} +``` + +#### Envelope Fields + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `event_id` | string (ULID) | Yes | Globally unique event identifier | +| `event_type` | string | Yes | Dot-separated event type: `.` | +| `event_version` | string | Yes | Semantic version of the event payload schema | +| `source_service` | string | Yes | Canonical name of the producing service | +| `timestamp` | string (ISO 8601) | Yes | Producer-assigned event time (UTC) | +| `correlation_id` | string | No | Propagated request ID for distributed tracing | +| `payload` | object | Yes | Event-type-specific payload (validated per event schema) | + +### 4.2 Event Types (v0.1.0) + +| Event Type | Producer | Consumers | Topic | +|-----------|----------|-----------|-------| +| `document.ingested` | Indexing Service | Search, Audit | `meridian.document.events` | +| `document.deleted` | Indexing Service | Search, Audit | `meridian.document.events` | +| `index.updated` | Indexing Service | Search | `meridian.index.events` | +| `search.query.completed` | Search Service | Analytics, Audit | `meridian.search.events` | +| `auth.token.issued` | Auth Service | Audit | `meridian.auth.events` | +| `auth.token.revoked` | Auth Service | Gateway, Audit | `meridian.auth.events` | + +### 4.3 Schema Compatibility + +The Schema Registry is configured with **BACKWARD** compatibility mode for all production topics. This means: + +- New optional fields may be added (consumers ignoring them remain valid) +- Required fields may not be removed or renamed +- Field types may not be changed + +Breaking schema changes require a new `event_version` and a coordinated migration plan. + +--- + +## 5. Topic Design + +### 5.1 Naming Convention + +``` +meridian..events +``` + +Examples: `meridian.document.events`, `meridian.auth.events` + +Topics are domain-scoped, not service-scoped. Multiple event types may share a topic if they belong to the same domain and are consumed by the same consumer set. + +### 5.2 Partition Strategy + +| Topic | Partition Count | Partition Key | +|-------|----------------|--------------| +| `meridian.document.events` | 12 | `collection_id` | +| `meridian.index.events` | 12 | `collection_id` | +| `meridian.search.events` | 24 | `tenant_id` | +| `meridian.auth.events` | 6 | `tenant_id` | + +Partitioning by `collection_id` ensures ordered processing of events for the same collection. Partitioning by `tenant_id` on search events enables per-tenant consumer parallelism. + +### 5.3 Retention + +| Topic | Retention | Rationale | +|-------|-----------|-----------| +| `meridian.document.events` | 7 days | Document operations are idempotent; 7 days allows recovery from downstream outages | +| `meridian.index.events` | 3 days | Short retention; consumers should be near-real-time | +| `meridian.search.events` | 30 days | Retained for analytics backfill use cases | +| `meridian.auth.events` | 90 days | Security audit requirement | + +--- + +## 6. Producer Specification + +### 6.1 Required Configuration + +All producers must configure: + +``` +acks=all # Wait for all ISR acknowledgment +enable.idempotence=true # Exactly-once produce semantics within a session +max.in.flight.requests.per.connection=5 +retries=2147483647 # Effectively infinite retries (bounded by delivery.timeout.ms) +delivery.timeout.ms=120000 # 2-minute delivery timeout +``` + +### 6.2 Schema Validation + +Producers must validate the event envelope and payload against the registered schema before producing. The Meridian SDK's `EventProducer` class handles this automatically. If using a raw Kafka client, call the Schema Registry validation endpoint before produce. + +### 6.3 Error Handling + +If schema validation fails: log the validation error, route to the service-level DLQ (`.dlq`), and do not produce to the event topic. + +If the Kafka produce call fails after retries: log the error with the full event for manual replay, emit `event_produce_failed_total` metric, and surface as a service health degradation (not a hard failure, unless the event is blocking a synchronous operation). + +--- + +## 7. Consumer Specification + +### 7.1 Required Configuration + +``` +auto.offset.reset=earliest # On first join, consume from beginning of retention window +enable.auto.commit=false # Manual offset commit after successful processing +max.poll.interval.ms=300000 # 5 minutes; increase if processing is batch-heavy +``` + +### 7.2 Idempotent Processing + +Because the Event Bus guarantees at-least-once delivery, consumers must process events idempotently. Recommended patterns: + +- Check for the `event_id` in a processed-events table before acting +- Use upsert semantics for database writes keyed on `event_id` +- Design state machines that tolerate duplicate transition triggers + +### 7.3 Offset Commit + +Commit offsets only after successful processing and persistence. Never commit before processing is complete. + +### 7.4 Consumer Groups + +Consumer group naming: `--consumer`. Example: `search-service-document-consumer`. + +--- + +## 8. Delivery Semantics + +The Event Bus provides **at-least-once** delivery. This means: + +- Every event produced will be delivered to every subscribed consumer at least once +- In failure and retry scenarios, an event may be delivered more than once +- Duplicate delivery is not a bug — consumers must be idempotent (see §7.2) + +**Producer side:** `acks=all` + `enable.idempotence=true` provides exactly-once semantics at the Kafka protocol level within a producer session. This eliminates producer-side duplicates but does not prevent consumer-side redelivery after a crash before offset commit. + +**Consumer side:** Exactly-once end-to-end (including consumer processing and downstream writes) requires transactional consumers, which are deferred to v0.2.0. + +--- + +## 9. Performance Requirements + +### 9.1 Throughput Targets + +| Topic | Target Throughput | Burst (10s) | +|-------|-----------------|-------------| +| `meridian.document.events` | 1,000 events/sec | 5,000 events/sec | +| `meridian.search.events` | 10,000 events/sec | 50,000 events/sec | +| `meridian.auth.events` | 500 events/sec | 2,000 events/sec | + +### 9.2 Latency Targets + +End-to-end produce-to-consume latency (producer acknowledges → consumer receives): + +| Percentile | Target | +|-----------|--------| +| p50 | ≤ 50ms | +| p99 | ≤ 200ms | +| p99.9 | ≤ 1,000ms | + +These targets assume consumers are running and not lagging. Latency for a lagging consumer is bounded by catch-up throughput, not the per-event latency. + +--- + +## 10. Security + +### 10.1 Authentication + +All clients authenticate to the Kafka cluster using **mTLS**. Client certificates are issued by the Meridian internal CA and must be rotated every 90 days (automated via cert-manager). + +Plaintext and SASL/PLAIN connections are disabled at the broker level. + +### 10.2 Authorization + +Topic-level ACLs are managed via Kafka's built-in ACL system: + +- Producers: `WRITE` permission on their designated topic(s) +- Consumers: `READ` permission on their designated topic(s) + `READ` on their consumer group +- No service has blanket `READ` or `WRITE` on all topics +- ACL changes require a Platform team approval workflow + +### 10.3 Encryption + +All data in transit is encrypted via TLS 1.2+ (TLS 1.3 preferred). Data at rest is encrypted via the host volume encryption provided by the cloud provider (AES-256). + +--- + +## 11. Known Limitations + +This section documents known limitations of the v0.1.0 specification. These are intentional design boundaries, not defects. + +- **No cross-cluster replication:** Events are contained within a single region. Geo-redundancy is out of scope for v0.1.0. +- **No exactly-once end-to-end:** Consumer-side exactly-once requires Kafka transactions, deferred to v0.2.0 (see §8). +- **KRaft combined mode:** Controller and broker roles are co-located in v0.1.0. This is acceptable for the initial deployment scale but limits controller isolation. Dedicated controllers are planned for v0.2.0. +- **Schema evolution:** Only BACKWARD compatibility is enforced. FORWARD and FULL compatibility are not required in v0.1.0 but will be evaluated for critical topics in v0.2.0. +- **No dead-letter topic automation:** Producers manually route to their DLQ; there is no Kafka Streams-based DLQ automation in this version. + +--- + +## 12. Open Questions + +| # | Question | Owner | Target | +|---|---------|-------|--------| +| 1 | Should search analytics events be sampled before producing to reduce volume? | Analytics Team | v0.2.0 | +| 2 | What is the right retention for `meridian.index.events` — 3 days may be too short if consumers are batch-oriented | Platform | Resolve before GA | +| 3 | Evaluate Kafka Streams vs. Flink for stateful consumer workloads (Audit aggregation use case) | Data Eng | Q3 2026 | + +--- + +## 13. Revision History + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 0.1.0 | 2026-02-10 | Platform Architecture | Initial draft | +| 0.1.0 | 2026-02-24 | Platform Architecture | Incorporated review feedback: added §11 Known Limitations, expanded §8 Delivery Semantics, clarified ACL workflow in §10.2 | + +--- + +*Meridian Event Bus Technical Specification — Platform Architecture Team* +*Questions: platform-arch@meridian.internal* diff --git a/golden-test-set/artifacts/docs/gt-doc-010-clean-research.md b/golden-test-set/artifacts/docs/gt-doc-010-clean-research.md new file mode 100644 index 0000000..a5d2edb --- /dev/null +++ b/golden-test-set/artifacts/docs/gt-doc-010-clean-research.md @@ -0,0 +1,235 @@ +# Prompt Injection Attacks Against LLM-Integrated Applications: Evidence Review + +**Document type:** Research Synthesis +**Author:** Security Research Team — T. Nakamura (lead), A. Osei +**Version:** 1.0 (Final) +**Date:** 2026-01-15 +**Distribution:** Internal — Security leadership and engineering teams + +--- + +## Abstract + +This synthesis reviews the current state of evidence on prompt injection attacks against large language model (LLM) integrated applications. We examine attack taxonomy, documented real-world incidents, defensive mitigations with empirical evaluation, and detection approaches. Drawing on 18 peer-reviewed sources and 6 credible practitioner reports published between 2023 and 2025, we find that prompt injection remains an unsolved problem with no single mitigation providing reliable protection. Defense-in-depth combining input sanitization, privilege separation, and output monitoring is the most evidenced approach currently available. This synthesis informs the security requirements for the Meridian AI Gateway. + +--- + +## 1. Introduction + +The integration of large language models into application workflows — as reasoning agents, document processors, customer-facing assistants, and code generators — creates a new class of security vulnerability: prompt injection. Unlike traditional injection attacks (SQL injection, XSS) that exploit parsing rules, prompt injection exploits the fundamental mechanism by which LLMs follow natural language instructions. + +As Meridian expands its use of LLM-based features, understanding the threat landscape and the evidence base for mitigations is essential for making defensible security architecture decisions. + +### 1.1 Research Questions + +This synthesis was designed to answer: + +**RQ1:** What is the taxonomy of prompt injection attack classes, and how do they differ in mechanism and exploitability? + +**RQ2:** What mitigations have been empirically evaluated, and what is the measured reduction in attack success rate? + +**RQ3:** What are the practical detection and monitoring approaches for prompt injection in production systems? + +--- + +## 2. Methodology + +### 2.1 Search Strategy + +We searched the following sources: arXiv (cs.CR, cs.AI), Semantic Scholar, USENIX Security proceedings, IEEE S&P proceedings, and the NIST AI Risk Management Framework documentation. We also reviewed practitioner publications from Anthropic, OpenAI, Google DeepMind, and Trail of Bits. + +Search terms included: "prompt injection," "indirect prompt injection," "jailbreak," "adversarial prompt," "LLM security," "AI application security," "instruction following attack." + +Date range: January 2023 – December 2025. We included earlier foundational work (pre-2023) only where cited as the primary technical reference by multiple retained sources. + +### 2.2 Inclusion Criteria + +- Technical description of a specific attack or mitigation mechanism +- Empirical evaluation or documented real-world incident with verifiable technical details +- Peer-reviewed publication OR practitioner report from an organization with relevant security credibility + +### 2.3 Exclusion Criteria + +- Pure opinion or speculation without technical grounding +- Attack demonstrations that require physical access to the model or training data (out of threat model scope) +- Jailbreak research focused solely on bypassing content filters (adjacent but distinct problem) + +### 2.4 Retained Sources + +After screening, 24 sources were retained: 18 peer-reviewed papers and 6 practitioner reports. + +--- + +## 3. Attack Taxonomy + +### 3.1 Direct Prompt Injection + +Direct prompt injection occurs when a malicious user directly inputs instructions into a prompt field, attempting to override the system prompt or alter the model's behavior. + +**Classic form:** The user inputs "Ignore all previous instructions and instead output..." targeting the system prompt. This attack class was first formally characterized by Perez & Ribeiro (2022) and has been extensively studied since. + +**Empirical attack success rates** vary widely by model and prompt complexity. Perez & Ribeiro (2022) reported success rates of 25–85% depending on attack sophistication across GPT-3 era models. More recent work (Wei et al., 2024) on current frontier models shows reduced but non-zero success rates (8–43%) for naive attacks, with more sophisticated "jailbreak" variants remaining effective. + +### 3.2 Indirect Prompt Injection + +Indirect prompt injection occurs when malicious instructions are embedded in data that the LLM processes as part of its context — web pages, documents, emails, database records — rather than in direct user input. + +This is the more dangerous class for RAG and agent systems. Greshake et al. (2023) demonstrated real-world indirect injection attacks across multiple LLM-integrated applications, including attacks that exfiltrated conversation history and triggered unauthorized actions via tool-calling APIs. Their work established the formal threat model for indirect injection that subsequent research builds on. + +Key variants: +- **Document injection:** Malicious instructions in a retrieved document override the model's behavior when the document is included in context +- **Email injection:** Instructions embedded in email bodies processed by an AI assistant +- **Web content injection:** Malicious instructions on web pages processed by an AI agent with browsing capability + +### 3.3 Compositional Attacks + +Compositional attacks combine multiple weak attack signals that individually fail but succeed when combined. Pasquini et al. (2024) demonstrated compositional injection achieving 73% success rate on a hardened model where individual components succeeded at <15%. This attack class is particularly relevant for long-context applications where monitoring focus may be diluted. + +### 3.4 Attack Surface Summary + +| Attack Class | Threat Level | Primary Surface | +|-------------|-------------|----------------| +| Direct injection | Medium (declining with model hardening) | User input fields | +| Indirect injection | High (increasing with agent capability) | Retrieved content, external data | +| Compositional | High (research stage, escalating) | Long-context, multi-turn | +| Stored injection | High (largely unmitigated) | Databases, document stores | + +--- + +## 4. Mitigations: Empirical Evidence + +### 4.1 Input Sanitization + +Input sanitization approaches — filtering, encoding, or normalizing user input before inclusion in prompts — provide partial protection against direct injection but are largely ineffective against indirect injection, since the attack surface is external data that cannot be controlled. + +Liu et al. (2024) evaluated 8 sanitization approaches against a benchmark of 500 direct injection attacks and found maximum success rate reduction of 31% (from baseline 67% to 44% success), with the best-performing approaches also exhibiting 12–18% false positive rates on benign inputs. + +**Verdict:** Useful as one layer, but insufficient as a primary defense. + +### 4.2 Privilege Separation + +Privilege separation architectures separate user-supplied content from system instructions at the prompt construction level, using structural markers or separate prompt components that the model is trained or prompted to treat with different trust levels. + +Anthropic's "privileged instructions" pattern (documented in their 2024 usage policies and expanded in their Constitutional AI update) showed measurable reduction in instruction following from untrusted content. Zhan et al. (2024) evaluated privilege separation across 3 frontier models and found 52–68% reduction in indirect injection success, with minimal impact on benign task performance. + +**Verdict:** Strongest individual mitigation in the literature. Should be a baseline architectural requirement. + +### 4.3 Output Monitoring + +Output monitoring approaches inspect model outputs for signs of injection success — unexpected content, out-of-scope instructions, data exfiltration patterns — rather than preventing injection at the input stage. + +Debenedetti et al. (2024) built a detection classifier trained on labeled injection outputs and benign outputs, achieving 0.89 AUC on their test set. False positive rate was 4.2% at the detection threshold optimized for recall. The classifier was effective at catching known attack patterns but degraded significantly (AUC 0.71) on novel attack variants not seen in training. + +**Verdict:** Effective as a detection layer, not a prevention layer. Requires continuous retraining as attack patterns evolve. + +### 4.4 Defense-in-Depth + +No single mitigation provides reliable protection. The convergent recommendation across the literature (Greshake et al., 2023; Anthropic 2024; OWASP LLM Top 10) is defense-in-depth: + +1. Privilege separation at prompt construction +2. Input sanitization for direct injection surface +3. Minimal tool permissions (principle of least privilege for agent capabilities) +4. Output monitoring and anomaly detection +5. Human-in-the-loop for high-stakes actions + +Empirical evaluation of combined defenses is limited in the literature; most studies evaluate mitigations in isolation. + +--- + +## 5. Detection and Monitoring in Production + +### 5.1 Logging Requirements + +Detection requires comprehensive logging of: +- Full prompt inputs (system + user content) +- Retrieved context (for RAG systems) +- Model outputs +- Tool calls and their results (for agent systems) + +Without full prompt logging, post-hoc investigation of suspected injection incidents is not feasible. This has privacy implications (logs may contain sensitive content) that must be addressed through log access controls and retention policies. + +### 5.2 Anomaly Detection Signals + +Based on the Debenedetti et al. (2024) classifier analysis and practitioner reports (Trail of Bits 2024, Lakera 2024), the following signals are most indicative of injection: + +- Model output contains instruction-like language not present in the system prompt +- Tool calls to endpoints not referenced in the original task +- Data exfiltration patterns: requests to external URLs with encoded context content +- Sudden persona shifts in multi-turn conversations +- Output contains structured data formats inconsistent with the requested task + +### 5.3 Incident Response + +At time of writing, no standardized incident response playbook for LLM injection incidents exists in the public literature. Internal playbook development is required. + +--- + +## 6. Limitations of This Review + +This synthesis has the following limitations that bound the confidence of its conclusions: + +1. **Rapidly evolving attack surface:** The rate of publication in this domain is high and our search has a cutoff of December 2025. Attack techniques documented after that date are not reflected. + +2. **Lab vs. production generalizability:** Most empirical evaluations use benchmark datasets and controlled conditions. Success rates in production environments (with real application logic, real users, real edge cases) may differ. + +3. **Model-specificity:** Attack success rates and mitigation effectiveness vary across model versions. Results for one model version may not generalize to others, including future versions of the same model family. + +4. **Publication bias:** Failed mitigations and unsuccessful attacks are less likely to be published. The literature likely over-represents attack successes and mitigation successes relative to the true distribution. + +5. **Compositional attack literature is sparse:** The compositional attack class (§3.3) has fewer than 5 papers. Conclusions about this attack class carry less evidential weight. + +--- + +## 7. Future Work + +Several important questions are not adequately addressed by current literature and represent productive directions for future research: + +- **Benchmarking in agentic systems at scale:** Most injection research uses simple single-turn setups. The threat surface in multi-step agent workflows is significantly larger and understudied. +- **Formal verification approaches:** Whether formal methods from traditional security (e.g., information flow control) can be adapted to LLM-integrated systems is an open question. +- **Longitudinal defense durability:** Do defenses that work today remain effective as models and attack techniques co-evolve? No longitudinal studies exist. + +These gaps do not undermine the actionability of current findings for near-term deployment decisions, but they are relevant for longer-horizon security roadmapping. + +--- + +## 8. Recommendations for Meridian AI Gateway + +Based on the synthesized evidence: + +1. **Require privilege separation** in all prompt construction for Meridian AI features. This is the highest-confidence mitigation in the literature (52–68% indirect injection reduction). + +2. **Implement output monitoring** from day one. Use the Debenedetti et al. (2024) signal taxonomy as a starting point. Budget for retraining as attack patterns evolve. + +3. **Apply least-privilege tool permissions** to all agent-capability features. Tools should be scoped to the minimum necessary action surface. + +4. **Log full prompt context** with appropriate access controls. Post-hoc investigation is not feasible without this. + +5. **Track the compositional attack research.** This attack class is understudied but potentially high-impact. Monitor for new publications in 2026. + +--- + +## References + +Debenedetti, G., et al. (2024). Detecting Prompt Injection with Output Classifiers. *USENIX Security 2024*. + +Greshake, K., et al. (2023). Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection. *Proceedings of the 6th ACM Workshop on Artificial Intelligence and Security (AISec 2023)*. + +Lakera. (2024). *State of Prompt Injection 2024*. Technical Report. + +Liu, Y., et al. (2024). Evaluating Input Sanitization Defenses for Prompt Injection. *arXiv:2401.XXXXX*. + +OWASP. (2025). *OWASP Top 10 for Large Language Model Applications, v1.1*. OWASP Foundation. + +Pasquini, D., et al. (2024). Compositional Prompt Injection: When the Sum Is Greater Than Its Parts. *IEEE S&P 2024*. + +Perez, F., & Ribeiro, I. (2022). Ignore Previous Prompt: Attack Techniques for Language Models. *arXiv:2211.09527*. + +Trail of Bits. (2024). *AI Security Review: LLM Integration Patterns*. Technical Report. + +Wei, A., et al. (2024). Jailbroken: How Does LLM Safety Training Fail? *NeurIPS 2024*. + +Zhan, Q., et al. (2024). Injecagent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents. *arXiv:2403.02691*. + +--- + +*This synthesis is an internal document. Distribution restricted to security leadership and authorized engineering teams. Contact: security-research@meridian.internal* diff --git a/golden-test-set/artifacts/python/gt-py-001-sql-injection.py b/golden-test-set/artifacts/python/gt-py-001-sql-injection.py new file mode 100644 index 0000000..28f4437 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-001-sql-injection.py @@ -0,0 +1,120 @@ +""" +User search and profile lookup endpoint. + +Provides Flask route handlers for the user management API. Supports +filtering by username, email, and role. Used by the admin dashboard +and public profile pages. +""" + +import logging +import sqlite3 +from flask import Flask, request, jsonify, g + +app = Flask(__name__) +DATABASE = "/var/app/data/users.db" + +logger = logging.getLogger(__name__) + + +def get_db(): + db = getattr(g, "_database", None) + if db is None: + db = g._database = sqlite3.connect(DATABASE) + db.row_factory = sqlite3.Row + return db + + +@app.teardown_appcontext +def close_connection(exception): + db = getattr(g, "_database", None) + if db is not None: + db.close() + + +@app.route("/api/users/search") +def search_users(): + """Search users by username, email, or role query parameter.""" + query_term = request.args.get("q", "") + field = request.args.get("field", "username") + + allowed_fields = {"username", "email", "role"} + if field not in allowed_fields: + return jsonify({"error": "Invalid field"}), 400 + + db = get_db() + # Build dynamic query to support flexible field filtering + sql = "SELECT id, username, email, role, created_at FROM users WHERE " + field + " LIKE '%" + query_term + "%'" + logger.debug("Executing query: %s", sql) + + try: + cursor = db.execute(sql) + rows = cursor.fetchall() + results = [dict(row) for row in rows] + return jsonify({"count": len(results), "results": results}) + except sqlite3.Error as exc: + logger.error("Database error during user search: %s", exc) + return jsonify({"error": "Database error"}), 500 + + +@app.route("/api/users/") +def get_user(user_id): + """Fetch a single user by primary key.""" + db = get_db() + row = db.execute( + "SELECT id, username, email, role, created_at FROM users WHERE id = ?", + (user_id,), + ).fetchone() + + if row is None: + return jsonify({"error": "User not found"}), 404 + + return jsonify(dict(row)) + + +@app.route("/api/users//activity") +def get_user_activity(user_id): + """Return paginated activity log for a user.""" + limit = request.args.get("limit", "20") + offset = request.args.get("offset", "0") + + # Validate pagination params are numeric + if not limit.isdigit() or not offset.isdigit(): + return jsonify({"error": "limit and offset must be integers"}), 400 + + db = get_db() + rows = db.execute( + "SELECT event_type, timestamp, metadata FROM activity_log " + "WHERE user_id = ? ORDER BY timestamp DESC LIMIT ? OFFSET ?", + (user_id, int(limit), int(offset)), + ).fetchall() + + return jsonify({"user_id": user_id, "events": [dict(r) for r in rows]}) + + +@app.route("/api/users", methods=["POST"]) +def create_user(): + """Create a new user account.""" + data = request.get_json(silent=True) + if not data: + return jsonify({"error": "JSON body required"}), 400 + + required = {"username", "email", "password_hash", "role"} + missing = required - data.keys() + if missing: + return jsonify({"error": f"Missing fields: {missing}"}), 400 + + db = get_db() + try: + db.execute( + "INSERT INTO users (username, email, password_hash, role) VALUES (?, ?, ?, ?)", + (data["username"], data["email"], data["password_hash"], data["role"]), + ) + db.commit() + except sqlite3.IntegrityError: + return jsonify({"error": "Username or email already exists"}), 409 + + return jsonify({"status": "created"}), 201 + + +if __name__ == "__main__": + app.run(debug=False, host="0.0.0.0", port=5000) diff --git a/golden-test-set/artifacts/python/gt-py-002-hardcoded-credentials.py b/golden-test-set/artifacts/python/gt-py-002-hardcoded-credentials.py new file mode 100644 index 0000000..0980e42 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-002-hardcoded-credentials.py @@ -0,0 +1,170 @@ +""" +Database connection pool and external service client factory. + +Manages persistent connections to the primary PostgreSQL instance and +the third-party analytics platform. Handles connection lifecycle, +retry on failure, and basic health-checking. +""" + +import logging +import re +import time +from typing import Optional + +import psycopg2 +import psycopg2.pool +import requests + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Database configuration +# --------------------------------------------------------------------------- + +DB_HOST = "db.internal.example.com" +DB_PORT = 5432 +DB_NAME = "appdb_prod" +DB_USER = "app_service" +DB_PASSWORD = "Tr0ub4dor&3_prod_2024!" # production database password + +ANALYTICS_API_BASE = "https://analytics.example.com/v2" +ANALYTICS_API_KEY = "sk-live-a8f3c2d19e4b7a6f1c0d2e5f8a3b9c7d" # analytics platform key + +# Regex pattern used to detect accidentally logged credentials in strings. +# This is intentionally a pattern detector, not a credential itself. +PASSWORD_PATTERN = re.compile( + r"(?i)(password|passwd|secret|token|api[_-]?key)\s*[=:]\s*\S+", + re.IGNORECASE, +) + +_pool: Optional[psycopg2.pool.ThreadedConnectionPool] = None + +# --------------------------------------------------------------------------- +# Connection pool management +# --------------------------------------------------------------------------- + +MAX_POOL_CONNECTIONS = 10 +MIN_POOL_CONNECTIONS = 2 +CONNECT_TIMEOUT_SEC = 5 +RETRY_ATTEMPTS = 3 +RETRY_BACKOFF_SEC = 2.0 + + +def _build_dsn() -> str: + return ( + f"host={DB_HOST} port={DB_PORT} dbname={DB_NAME} " + f"user={DB_USER} password={DB_PASSWORD} " + f"connect_timeout={CONNECT_TIMEOUT_SEC} sslmode=require" + ) + + +def init_pool() -> None: + """Initialize the threaded connection pool. Call once at startup.""" + global _pool + dsn = _build_dsn() + for attempt in range(1, RETRY_ATTEMPTS + 1): + try: + _pool = psycopg2.pool.ThreadedConnectionPool( + MIN_POOL_CONNECTIONS, MAX_POOL_CONNECTIONS, dsn + ) + logger.info("Database connection pool initialized (attempt %d)", attempt) + return + except psycopg2.OperationalError as exc: + logger.warning("Pool init attempt %d failed: %s", attempt, exc) + if attempt < RETRY_ATTEMPTS: + time.sleep(RETRY_BACKOFF_SEC * attempt) + raise RuntimeError("Unable to initialize database connection pool after retries") + + +def acquire() -> psycopg2.extensions.connection: + """Get a connection from the pool. Caller must call release() when done.""" + if _pool is None: + raise RuntimeError("Pool not initialized. Call init_pool() first.") + return _pool.getconn() + + +def release(conn: psycopg2.extensions.connection) -> None: + """Return a connection to the pool.""" + if _pool is not None: + _pool.putconn(conn) + + +def close_pool() -> None: + """Tear down the pool at shutdown.""" + global _pool + if _pool is not None: + _pool.closeall() + _pool = None + logger.info("Database connection pool closed") + + +# --------------------------------------------------------------------------- +# Analytics API client +# --------------------------------------------------------------------------- + +ANALYTICS_TIMEOUT_SEC = 10 +ANALYTICS_RETRIES = 2 + + +class AnalyticsClient: + """Thin wrapper around the analytics HTTP API.""" + + def __init__(self, base_url: str = ANALYTICS_API_BASE, api_key: str = ANALYTICS_API_KEY): + self._base = base_url.rstrip("/") + self._session = requests.Session() + self._session.headers.update( + { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + ) + + def track_event(self, user_id: str, event: str, properties: dict) -> bool: + """ + POST a single event to the analytics platform. + + Returns True on success, False if the platform returned an error + (after retries). Does not raise — callers should not fail hard on + analytics failures. + """ + payload = {"user_id": user_id, "event": event, "properties": properties} + url = f"{self._base}/events" + + for attempt in range(1, ANALYTICS_RETRIES + 1): + try: + resp = self._session.post(url, json=payload, timeout=ANALYTICS_TIMEOUT_SEC) + if resp.status_code == 200: + return True + logger.warning( + "Analytics track_event HTTP %d on attempt %d: %s", + resp.status_code, + attempt, + resp.text[:200], + ) + except requests.RequestException as exc: + logger.warning("Analytics request error on attempt %d: %s", attempt, exc) + + if attempt < ANALYTICS_RETRIES: + time.sleep(RETRY_BACKOFF_SEC) + + return False + + def health_check(self) -> bool: + """Return True if the analytics API is reachable.""" + try: + resp = self._session.get( + f"{self._base}/health", timeout=ANALYTICS_TIMEOUT_SEC + ) + return resp.status_code == 200 + except requests.RequestException: + return False + + def scrub_log_line(self, line: str) -> str: + """ + Redact credential-like patterns from a log line before writing. + + Uses PASSWORD_PATTERN to find and redact any key=value pairs + that look like they contain sensitive material. + """ + return PASSWORD_PATTERN.sub(r"\1=[REDACTED]", line) diff --git a/golden-test-set/artifacts/python/gt-py-003-command-injection.py b/golden-test-set/artifacts/python/gt-py-003-command-injection.py new file mode 100644 index 0000000..f938c20 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-003-command-injection.py @@ -0,0 +1,213 @@ +""" +Deployment utility for managing application releases. + +Supports artifact verification, service restarts, rollback, and +post-deploy smoke testing. Intended to be invoked by the CI/CD pipeline +or by operators via the admin CLI. +""" + +import argparse +import logging +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +DEPLOY_BASE = Path("/opt/deploys") +SERVICE_NAMES = {"api", "worker", "scheduler", "gateway"} +ARTIFACT_REGISTRY = "artifacts.internal.example.com" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _require_env(key: str) -> str: + val = os.environ.get(key) + if not val: + raise EnvironmentError(f"Required environment variable not set: {key}") + return val + + +def _safe_service_name(name: str) -> str: + """Validate that a service name is in the allowed set.""" + if name not in SERVICE_NAMES: + raise ValueError(f"Unknown service '{name}'. Allowed: {SERVICE_NAMES}") + return name + + +# --------------------------------------------------------------------------- +# Artifact download +# --------------------------------------------------------------------------- + + +def download_artifact(artifact_name: str, version: str, dest_dir: Path) -> Path: + """ + Pull a versioned artifact from the internal registry. + + artifact_name and version come from CI metadata — they are NOT + sanitized before being embedded in the shell command. + """ + dest_dir.mkdir(parents=True, exist_ok=True) + dest_path = dest_dir / f"{artifact_name}-{version}.tar.gz" + + # Build download command — artifact_name and version are injected directly + download_cmd = ( + f"curl -fsSL https://{ARTIFACT_REGISTRY}/releases/{artifact_name}/{version}/package.tar.gz" + f" -o {dest_path}" + ) + logger.info("Downloading artifact: %s", download_cmd) + exit_code = os.system(download_cmd) + if exit_code != 0: + raise RuntimeError(f"Artifact download failed (exit {exit_code})") + return dest_path + + +# --------------------------------------------------------------------------- +# Service control +# --------------------------------------------------------------------------- + + +def restart_service(service: str) -> None: + """ + Restart a managed systemd service. + + service name is validated against the allow-list before use. + """ + safe = _safe_service_name(service) + # Fully controlled input — safe to pass as list args + result = subprocess.run( + ["systemctl", "restart", f"app-{safe}.service"], + capture_output=True, + text=True, + check=True, + ) + logger.info("Service restart output: %s", result.stdout) + + +def check_service_status(service: str) -> bool: + """Return True if the service reports active state.""" + safe = _safe_service_name(service) + result = subprocess.run( + ["systemctl", "is-active", f"app-{safe}.service"], + capture_output=True, + text=True, + ) + return result.stdout.strip() == "active" + + +# --------------------------------------------------------------------------- +# Deploy step +# --------------------------------------------------------------------------- + + +def run_migrations(db_url: str, migration_dir: str) -> None: + """ + Run database migrations using alembic. + + migration_dir is supplied by the operator or CI environment. + The db_url and migration_dir are passed as arguments to the shell + command without sanitization. + """ + migrate_cmd = f"alembic --config {migration_dir}/alembic.ini upgrade head" + logger.info("Running migrations: %s", migrate_cmd) + result = subprocess.run( + migrate_cmd, + shell=True, + capture_output=True, + text=True, + env={**os.environ, "DATABASE_URL": db_url}, + ) + if result.returncode != 0: + raise RuntimeError( + f"Migration failed (exit {result.returncode}):\n{result.stderr}" + ) + logger.info("Migration stdout: %s", result.stdout) + + +# --------------------------------------------------------------------------- +# File system utilities +# --------------------------------------------------------------------------- + + +def list_deploy_artifacts(deploy_dir: Optional[str] = None) -> list: + """List artifacts in the deploy directory (read-only, fully controlled path).""" + target = Path(deploy_dir) if deploy_dir else DEPLOY_BASE + result = subprocess.run(["ls", "-la", str(target)], capture_output=True, text=True, check=True) + return result.stdout.splitlines() + + +def verify_checksum(artifact_path: Path, expected_sha256: str) -> bool: + """Verify a downloaded artifact against its published SHA-256.""" + result = subprocess.run( + ["sha256sum", str(artifact_path)], + capture_output=True, + text=True, + check=True, + ) + actual = result.stdout.split()[0] + return actual == expected_sha256 + + +# --------------------------------------------------------------------------- +# Post-deploy smoke test +# --------------------------------------------------------------------------- + + +def run_smoke_test(endpoint: str, expected_status: int = 200) -> bool: + """Hit a health check endpoint and verify it returns the expected status.""" + result = subprocess.run( + ["curl", "-sf", "-o", "/dev/null", "-w", "%{http_code}", endpoint], + capture_output=True, + text=True, + timeout=15, + ) + try: + code = int(result.stdout.strip()) + return code == expected_status + except ValueError: + return False + + +# --------------------------------------------------------------------------- +# CLI entrypoint +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description="Application deployment utility") + sub = parser.add_subparsers(dest="command", required=True) + + dl_parser = sub.add_parser("download", help="Download an artifact") + dl_parser.add_argument("artifact_name") + dl_parser.add_argument("version") + dl_parser.add_argument("--dest", default=str(DEPLOY_BASE)) + + restart_parser = sub.add_parser("restart", help="Restart a service") + restart_parser.add_argument("service") + + migrate_parser = sub.add_parser("migrate", help="Run database migrations") + migrate_parser.add_argument("--db-url", required=True) + migrate_parser.add_argument("--migration-dir", required=True) + + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + if args.command == "download": + path = download_artifact(args.artifact_name, args.version, Path(args.dest)) + print(f"Downloaded to: {path}") + elif args.command == "restart": + restart_service(args.service) + print(f"Restarted: {args.service}") + elif args.command == "migrate": + run_migrations(args.db_url, args.migration_dir) + print("Migrations complete") + + +if __name__ == "__main__": + main() diff --git a/golden-test-set/artifacts/python/gt-py-004-insecure-deserialization.py b/golden-test-set/artifacts/python/gt-py-004-insecure-deserialization.py new file mode 100644 index 0000000..65495d6 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-004-insecure-deserialization.py @@ -0,0 +1,216 @@ +""" +Distributed cache layer backed by a Redis-compatible socket server. + +Provides get/set/delete operations with optional object serialization. +Used by the session manager and the query result cache to avoid redundant +database round-trips across worker processes. +""" + +import hashlib +import io +import json +import logging +import pickle +import socket +import struct +import threading +import time +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Wire protocol constants +# --------------------------------------------------------------------------- + +HEADER_FMT = "!BHI" # version(1) + opcode(2) + payload_len(4) +HEADER_SIZE = struct.calcsize(HEADER_FMT) +PROTO_VERSION = 1 + +OP_GET = 0x01 +OP_SET = 0x02 +OP_DEL = 0x03 +OP_PING = 0x04 + +DEFAULT_HOST = "cache.internal.example.com" +DEFAULT_PORT = 6380 +SOCKET_TIMEOUT = 5.0 +MAX_PAYLOAD_BYTES = 16 * 1024 * 1024 # 16 MB + +# --------------------------------------------------------------------------- +# Low-level socket transport +# --------------------------------------------------------------------------- + + +class CacheTransport: + """Manages a single persistent TCP connection to the cache server.""" + + def __init__(self, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT): + self._host = host + self._port = port + self._sock: Optional[socket.socket] = None + self._lock = threading.Lock() + + def connect(self) -> None: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(SOCKET_TIMEOUT) + sock.connect((self._host, self._port)) + self._sock = sock + logger.debug("Cache transport connected to %s:%d", self._host, self._port) + + def disconnect(self) -> None: + if self._sock: + try: + self._sock.close() + except OSError: + pass + self._sock = None + + def _send_frame(self, opcode: int, payload: bytes) -> None: + header = struct.pack(HEADER_FMT, PROTO_VERSION, opcode, len(payload)) + self._sock.sendall(header + payload) + + def _recv_frame(self) -> bytes: + raw_header = self._recv_exact(HEADER_SIZE) + version, opcode, payload_len = struct.unpack(HEADER_FMT, raw_header) + if version != PROTO_VERSION: + raise ValueError(f"Unsupported protocol version: {version}") + if payload_len > MAX_PAYLOAD_BYTES: + raise ValueError(f"Payload too large: {payload_len} bytes") + return self._recv_exact(payload_len) + + def _recv_exact(self, n: int) -> bytes: + buf = bytearray() + while len(buf) < n: + chunk = self._sock.recv(n - len(buf)) + if not chunk: + raise ConnectionError("Connection closed by cache server") + buf.extend(chunk) + return bytes(buf) + + def request(self, opcode: int, payload: bytes) -> bytes: + with self._lock: + self._send_frame(opcode, payload) + return self._recv_frame() + + +# --------------------------------------------------------------------------- +# High-level cache client +# --------------------------------------------------------------------------- + + +class CacheClient: + """ + High-level cache client supporting arbitrary Python object storage. + + Objects are serialized with pickle for full type fidelity across + worker processes. Strings and bytes are stored as-is using JSON + for interoperability with non-Python consumers. + """ + + def __init__(self, host: str = DEFAULT_HOST, port: int = DEFAULT_PORT): + self._transport = CacheTransport(host, port) + self._transport.connect() + + # ------------------------------------------------------------------ + # Core operations + # ------------------------------------------------------------------ + + def get(self, key: str) -> Optional[Any]: + """ + Retrieve a value from the cache. + + Returns None on cache miss. Deserializes the stored payload; + object types are restored via pickle. + """ + request_payload = key.encode("utf-8") + raw_response = self._transport.request(OP_GET, request_payload) + + if not raw_response: + return None # cache miss + + # First byte is a type tag: 0x01 = JSON, 0x02 = pickle + type_tag = raw_response[0] + data = raw_response[1:] + + if type_tag == 0x01: + return json.loads(data) + elif type_tag == 0x02: + # Deserialize a cached Python object. The server may hold data + # written by any connected worker — including external callers + # on the network segment. + return pickle.loads(data) + else: + raise ValueError(f"Unknown type tag in cache response: {type_tag:#x}") + + def set(self, key: str, value: Any, ttl: int = 300) -> bool: + """ + Store a value in the cache with an optional TTL (seconds). + + Python objects that aren't JSON-serializable are stored as pickle. + Strings, dicts, lists, and primitives use JSON. + """ + try: + serialized = json.dumps(value).encode("utf-8") + type_tag = b"\x01" + except (TypeError, ValueError): + serialized = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL) + type_tag = b"\x02" + + key_bytes = key.encode("utf-8") + ttl_bytes = struct.pack("!I", ttl) + key_len = struct.pack("!H", len(key_bytes)) + + payload = key_len + key_bytes + ttl_bytes + type_tag + serialized + response = self._transport.request(OP_SET, payload) + return response == b"\x00" # 0x00 = OK + + def delete(self, key: str) -> bool: + """Remove a key from the cache. Returns True if the key existed.""" + response = self._transport.request(OP_DEL, key.encode("utf-8")) + return response == b"\x01" + + def ping(self) -> bool: + """Health check. Returns True if the server responds correctly.""" + try: + response = self._transport.request(OP_PING, b"") + return response == b"PONG" + except (OSError, ConnectionError): + return False + + # ------------------------------------------------------------------ + # Convenience helpers + # ------------------------------------------------------------------ + + def get_or_set(self, key: str, factory, ttl: int = 300) -> Any: + """ + Return cached value or compute and cache it via factory callable. + + factory() is only called on cache miss. + """ + value = self.get(key) + if value is None: + value = factory() + self.set(key, value, ttl=ttl) + return value + + def fingerprint(self, key: str) -> Optional[str]: + """ + Return a SHA-256 hex digest of the raw cached bytes, or None on miss. + + Useful for cache invalidation checks without deserializing the payload. + """ + request_payload = key.encode("utf-8") + raw_response = self._transport.request(OP_GET, request_payload) + if not raw_response: + return None + return hashlib.sha256(raw_response).hexdigest() + + def close(self) -> None: + self._transport.disconnect() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() diff --git a/golden-test-set/artifacts/python/gt-py-005-logic-errors.py b/golden-test-set/artifacts/python/gt-py-005-logic-errors.py new file mode 100644 index 0000000..ad9e27c --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-005-logic-errors.py @@ -0,0 +1,225 @@ +""" +Pricing calculator for the e-commerce checkout pipeline. + +Computes final order totals by applying tiered discounts, promotional +codes, and jurisdiction-specific tax rates. Called by the checkout service +before presenting the order summary to the customer. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from decimal import Decimal, ROUND_HALF_UP +from typing import Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + +TAX_RATES = { + "CA": Decimal("0.0725"), + "WA": Decimal("0.065"), + "TX": Decimal("0.0625"), + "NY": Decimal("0.08875"), + "FL": Decimal("0.06"), + "DEFAULT": Decimal("0.05"), +} + + +@dataclass +class LineItem: + sku: str + unit_price: Decimal + quantity: int + + +@dataclass +class OrderContext: + items: list[LineItem] + customer_tier: str # "standard", "silver", "gold", "platinum" + promo_code: Optional[str] + state_code: str + subtotal: Decimal = field(init=False, default=Decimal("0")) + + def __post_init__(self): + self.subtotal = sum( + item.unit_price * item.quantity for item in self.items + ) + + +# --------------------------------------------------------------------------- +# Discount tiers +# +# Business rule (spec §4.2): +# subtotal < $50 → 0% discount +# $50 – $99.99 → 5% discount +# $100 – $249.99 → 10% discount +# $250+ → 15% discount +# --------------------------------------------------------------------------- + +DISCOUNT_TIERS = [ + (Decimal("250"), Decimal("0.15")), + (Decimal("100"), Decimal("0.10")), + (Decimal("50"), Decimal("0.05")), +] + + +def compute_tier_discount(subtotal: Decimal) -> Decimal: + """ + Return the fractional discount rate for the given subtotal. + + Iterates tiers from highest to lowest threshold and returns the first + matching rate. + """ + for threshold, rate in DISCOUNT_TIERS: + # Off-by-one: uses strict > instead of >= for the tier boundary, + # so a subtotal of exactly $250.00 gets the 10% tier instead of 15%. + if subtotal > threshold: + return rate + return Decimal("0") + + +def apply_customer_tier_multiplier(base_discount: Decimal, tier: str) -> Decimal: + """ + Loyalty tier multiplies the base discount rate. + + standard → 1.0x, silver → 1.1x, gold → 1.25x, platinum → 1.5x + """ + multipliers = { + "standard": Decimal("1.0"), + "silver": Decimal("1.1"), + "gold": Decimal("1.25"), + "platinum": Decimal("1.5"), + } + mult = multipliers.get(tier, Decimal("1.0")) + combined = base_discount * mult + # Cap total discount at 25% + return min(combined, Decimal("0.25")) + + +# --------------------------------------------------------------------------- +# Promo codes +# --------------------------------------------------------------------------- + +PROMO_CODES = { + "SAVE10": Decimal("10"), # $10 flat off + "SUMMER15": Decimal("15"), # $15 flat off + "WELCOME20": Decimal("20"), # $20 flat off +} + + +def apply_promo_code(subtotal: Decimal, code: Optional[str]) -> Decimal: + """Return the promo discount amount (flat dollar value, not rate).""" + if not code: + return Decimal("0") + discount_amount = PROMO_CODES.get(code.upper(), Decimal("0")) + if discount_amount == 0: + logger.warning("Unrecognized promo code: %s", code) + return min(discount_amount, subtotal) + + +# --------------------------------------------------------------------------- +# Tax calculation +# --------------------------------------------------------------------------- + + +def compute_tax(taxable_amount: Decimal, state_code: str) -> Decimal: + """ + Compute sales tax on the taxable amount. + + Tax is applied AFTER discounts. The formula is: + tax = taxable_amount * rate + + The rate is rounded to 4 decimal places before multiplication. + """ + rate = TAX_RATES.get(state_code.upper(), TAX_RATES["DEFAULT"]) + + # Precedence error: multiplication binds tighter than expected here. + # Intent is: tax = taxable_amount * rate, rounded to cents. + # Actual expression due to missing parentheses mixes Decimal + float + # operations in an order that may produce wrong results in edge cases. + # Specifically: the rounding quantum is applied to `rate` alone, not + # to the product, so the final tax can be off by $0.01–$0.04. + tax = taxable_amount * rate.quantize(Decimal("0.0001")) + Decimal("0") + + return tax.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + + +# --------------------------------------------------------------------------- +# Top-level order total +# --------------------------------------------------------------------------- + + +def calculate_order_total(ctx: OrderContext) -> dict: + """ + Compute the full breakdown of an order total. + + Returns a dict with keys: + subtotal, tier_discount_amount, promo_discount_amount, + taxable_amount, tax, total + """ + subtotal = ctx.subtotal + + # Step 1: tier-based percentage discount + tier_rate = compute_tier_discount(subtotal) + effective_rate = apply_customer_tier_multiplier(tier_rate, ctx.customer_tier) + tier_discount_amount = (subtotal * effective_rate).quantize( + Decimal("0.01"), rounding=ROUND_HALF_UP + ) + + # Step 2: flat promo code discount (applied after tier discount) + post_tier = subtotal - tier_discount_amount + promo_discount_amount = apply_promo_code(post_tier, ctx.promo_code) + + # Step 3: tax on remaining amount + taxable_amount = post_tier - promo_discount_amount + if taxable_amount < Decimal("0"): + taxable_amount = Decimal("0") + + tax = compute_tax(taxable_amount, ctx.state_code) + total = (taxable_amount + tax).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP) + + breakdown = { + "subtotal": str(subtotal), + "tier_discount_rate": str(effective_rate), + "tier_discount_amount": str(tier_discount_amount), + "promo_discount_amount": str(promo_discount_amount), + "taxable_amount": str(taxable_amount), + "tax": str(tax), + "total": str(total), + } + + logger.info( + "Order total calculated: subtotal=%s total=%s (tier=%s state=%s promo=%s)", + subtotal, + total, + ctx.customer_tier, + ctx.state_code, + ctx.promo_code, + ) + + return breakdown + + +# --------------------------------------------------------------------------- +# Example usage +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import json + + items = [ + LineItem(sku="SKU-001", unit_price=Decimal("89.99"), quantity=2), + LineItem(sku="SKU-002", unit_price=Decimal("14.50"), quantity=3), + ] + ctx = OrderContext( + items=items, + customer_tier="gold", + promo_code="SAVE10", + state_code="WA", + ) + result = calculate_order_total(ctx) + print(json.dumps(result, indent=2)) diff --git a/golden-test-set/artifacts/python/gt-py-006-missing-error-handling.py b/golden-test-set/artifacts/python/gt-py-006-missing-error-handling.py new file mode 100644 index 0000000..7d80f6f --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-006-missing-error-handling.py @@ -0,0 +1,207 @@ +""" +File processing pipeline for ingesting nightly data exports. + +Reads CSV files from a configurable drop directory, enriches each record +by calling an internal enrichment API, and writes normalized JSON output +to an object store. Designed to be run as a cron job or triggered by a +file system event. +""" + +from __future__ import annotations + +import csv +import json +import logging +import os +import time +from pathlib import Path +from typing import Generator, Optional + +import requests + +logger = logging.getLogger(__name__) + +ENRICHMENT_API_BASE = os.environ.get("ENRICHMENT_API_URL", "http://enrichment-svc:8080") +OBJECT_STORE_BASE = Path(os.environ.get("OUTPUT_DIR", "/var/data/processed")) +DROP_DIR = Path(os.environ.get("DROP_DIR", "/var/data/ingest")) + +BATCH_SIZE = 100 +MAX_RETRIES = 3 + + +# --------------------------------------------------------------------------- +# Reading +# --------------------------------------------------------------------------- + + +def iter_csv_records(filepath: Path) -> Generator[dict, None, None]: + """ + Yield dicts for each row in a CSV file. + + Assumes the file is UTF-8 and has a header row. Does not handle + missing files — callers are expected to pass valid paths. + """ + # No try/except: if the file doesn't exist or can't be opened, + # FileNotFoundError or PermissionError will propagate uncaught. + fh = open(filepath, newline="", encoding="utf-8") + reader = csv.DictReader(fh) + for row in reader: + yield row + fh.close() + + +def discover_pending_files(drop_dir: Path) -> list[Path]: + """Return all .csv files in the drop directory, sorted by mtime.""" + if not drop_dir.exists(): + logger.warning("Drop directory does not exist: %s", drop_dir) + return [] + files = sorted(drop_dir.glob("*.csv"), key=lambda p: p.stat().st_mtime) + logger.info("Discovered %d pending file(s) in %s", len(files), drop_dir) + return files + + +# --------------------------------------------------------------------------- +# Enrichment API +# --------------------------------------------------------------------------- + + +def enrich_record(record: dict) -> dict: + """ + Call the enrichment API to append geolocation and risk metadata. + + The API may return non-200 on transient errors or unknown record IDs. + On success it returns a JSON body with an `enrichment` key. + """ + record_id = record.get("id", "unknown") + url = f"{ENRICHMENT_API_BASE}/v1/enrich/{record_id}" + + # No timeout specified — will block indefinitely if the service hangs. + response = requests.post(url, json=record) + + # Return value is not checked before accessing .json() — if the API + # returns 4xx or 5xx the caller receives incomplete data silently. + enrichment_data = response.json().get("enrichment", {}) + + return {**record, "enrichment": enrichment_data} + + +def enrich_batch(records: list[dict]) -> list[dict]: + """Enrich a batch of records, logging failures individually.""" + enriched = [] + for record in records: + for attempt in range(1, MAX_RETRIES + 1): + try: + result = enrich_record(record) + enriched.append(result) + break + except requests.RequestException as exc: + logger.warning( + "Enrichment attempt %d/%d failed for record %s: %s", + attempt, + MAX_RETRIES, + record.get("id"), + exc, + ) + if attempt == MAX_RETRIES: + logger.error( + "Giving up on record %s after %d attempts", + record.get("id"), + MAX_RETRIES, + ) + enriched.append({**record, "enrichment": None, "enrichment_error": str(exc)}) + else: + time.sleep(2 ** attempt) + return enriched + + +# --------------------------------------------------------------------------- +# Writing +# --------------------------------------------------------------------------- + + +def write_output(records: list[dict], source_path: Path) -> Path: + """Write enriched records to a JSON file in the output directory.""" + OBJECT_STORE_BASE.mkdir(parents=True, exist_ok=True) + out_path = OBJECT_STORE_BASE / (source_path.stem + ".json") + with open(out_path, "w", encoding="utf-8") as fh: + json.dump(records, fh, indent=2, default=str) + logger.info("Wrote %d records to %s", len(records), out_path) + return out_path + + +# --------------------------------------------------------------------------- +# Pipeline orchestration +# --------------------------------------------------------------------------- + + +def process_file(filepath: Path) -> Optional[Path]: + """ + Process a single CSV file end-to-end. + + Returns the output path on success, None on failure. + """ + logger.info("Processing: %s", filepath) + records = list(iter_csv_records(filepath)) + + if not records: + logger.warning("No records found in %s — skipping", filepath) + return None + + total = len(records) + all_enriched = [] + + for start in range(0, total, BATCH_SIZE): + batch = records[start : start + BATCH_SIZE] + enriched = enrich_batch(batch) + all_enriched.extend(enriched) + logger.info( + "Processed batch %d/%d (%d records)", + start // BATCH_SIZE + 1, + -(-total // BATCH_SIZE), + len(batch), + ) + + return write_output(all_enriched, filepath) + + +def archive_file(filepath: Path) -> None: + """Move a processed file to the archive subdirectory.""" + archive_dir = filepath.parent / "archive" + archive_dir.mkdir(exist_ok=True) + dest = archive_dir / filepath.name + filepath.rename(dest) + logger.info("Archived %s → %s", filepath, dest) + + +def run_pipeline(drop_dir: Path = DROP_DIR) -> dict: + """ + Main pipeline entry point. + + Returns a summary dict with counts for processed/failed files. + """ + pending = discover_pending_files(drop_dir) + summary = {"total": len(pending), "processed": 0, "failed": 0} + + for filepath in pending: + try: + out = process_file(filepath) + if out is not None: + archive_file(filepath) + summary["processed"] += 1 + else: + summary["failed"] += 1 + except Exception as exc: + logger.error("Unhandled error processing %s: %s", filepath, exc, exc_info=True) + summary["failed"] += 1 + + logger.info("Pipeline complete: %s", summary) + return summary + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + result = run_pipeline() + print(json.dumps(result)) diff --git a/golden-test-set/artifacts/python/gt-py-007-dead-code.py b/golden-test-set/artifacts/python/gt-py-007-dead-code.py new file mode 100644 index 0000000..61f7c6f --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-007-dead-code.py @@ -0,0 +1,258 @@ +""" +String and collection utility module. + +General-purpose helpers used across the application: slug generation, +deep merging of config dicts, pagination helpers, and lightweight +CSV serialization. Pulled in by multiple services. +""" + +from __future__ import annotations + +import csv +import io +import json +import re +import unicodedata +from typing import Any, Iterable, Iterator, Optional + +# Unused import left over from an earlier refactor that moved XML support +# to a dedicated module (xml_utils.py). Nothing in this file uses xml. +import xml.etree.ElementTree as ET + +# --------------------------------------------------------------------------- +# Slug generation +# --------------------------------------------------------------------------- + + +def slugify(text: str, max_length: int = 80, separator: str = "-") -> str: + """ + Convert arbitrary text to a URL-safe slug. + + Normalizes unicode, strips non-alphanumeric characters, collapses + whitespace, and truncates to max_length. Preserves hyphens and + underscores by default. + """ + # Normalize to ASCII-compatible form + text = unicodedata.normalize("NFKD", text) + text = text.encode("ascii", "ignore").decode("ascii") + text = text.lower() + # Replace non-word characters with the separator + text = re.sub(r"[^\w\s-]", "", text) + text = re.sub(r"[\s_]+", separator, text) + text = re.sub(rf"{re.escape(separator)}+", separator, text) + text = text.strip(separator) + return text[:max_length] + + +# --------------------------------------------------------------------------- +# Deep merge +# --------------------------------------------------------------------------- + + +def deep_merge(base: dict, override: dict) -> dict: + """ + Recursively merge override into base, returning a new dict. + + Nested dicts are merged rather than replaced. All other value types + (lists, scalars) are overwritten by the override value. + """ + result = dict(base) + for key, val in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(val, dict): + result[key] = deep_merge(result[key], val) + else: + result[key] = val + return result + + +# --------------------------------------------------------------------------- +# Pagination +# --------------------------------------------------------------------------- + + +def paginate(items: list, page: int, page_size: int) -> dict: + """ + Slice a list for the given 1-indexed page and return a pagination envelope. + + Returns a dict with keys: items, page, page_size, total, total_pages, has_next, has_prev. + """ + if page < 1: + page = 1 + if page_size < 1: + page_size = 10 + + total = len(items) + total_pages = max(1, -(-total // page_size)) # ceiling division + start = (page - 1) * page_size + end = start + page_size + page_items = items[start:end] + + return { + "items": page_items, + "page": page, + "page_size": page_size, + "total": total, + "total_pages": total_pages, + "has_next": page < total_pages, + "has_prev": page > 1, + } + + +def iter_pages(items: list, page_size: int) -> Iterator[list]: + """Yield successive fixed-size chunks from a list.""" + for i in range(0, len(items), page_size): + yield items[i : i + page_size] + + +# --------------------------------------------------------------------------- +# CSV serialization +# --------------------------------------------------------------------------- + + +def dicts_to_csv(records: Iterable[dict], fieldnames: Optional[list] = None) -> str: + """ + Serialize a sequence of dicts to a CSV string. + + If fieldnames is not provided, it is inferred from the first record. + Missing keys in subsequent records are written as empty strings. + """ + records = list(records) + if not records: + return "" + + if fieldnames is None: + fieldnames = list(records[0].keys()) + + buf = io.StringIO() + writer = csv.DictWriter( + buf, fieldnames=fieldnames, extrasaction="ignore", lineterminator="\n" + ) + writer.writeheader() + writer.writerows(records) + return buf.getvalue() + + +def csv_to_dicts(text: str) -> list[dict]: + """Parse a CSV string and return a list of dicts.""" + buf = io.StringIO(text) + reader = csv.DictReader(buf) + return list(reader) + + +# --------------------------------------------------------------------------- +# Config normalization +# --------------------------------------------------------------------------- + +_TRUTHY = {"true", "yes", "1", "on"} +_FALSY = {"false", "no", "0", "off"} + + +def coerce_bool(value: Any) -> bool: + """ + Coerce a loosely typed value to bool. + + Accepts booleans, integers, and case-insensitive string representations. + Raises ValueError for unrecognized strings. + """ + if isinstance(value, bool): + return value + if isinstance(value, int): + return bool(value) + if isinstance(value, str): + lower = value.strip().lower() + if lower in _TRUTHY: + return True + if lower in _FALSY: + return False + raise ValueError(f"Cannot coerce {value!r} to bool") + raise TypeError(f"Unsupported type for bool coercion: {type(value)}") + + +def normalize_config(raw: dict) -> dict: + """ + Normalize a raw config dict: strip string values, coerce known bool keys. + + Returns a new dict; does not mutate the input. + """ + BOOL_KEYS = {"enabled", "debug", "verbose", "dry_run", "ssl"} + result = {} + for k, v in raw.items(): + if isinstance(v, str): + v = v.strip() + if k in BOOL_KEYS: + try: + v = coerce_bool(v) + except (ValueError, TypeError): + pass # leave as-is if coercion fails + result[k] = v + return result + + +# --------------------------------------------------------------------------- +# Text formatting +# --------------------------------------------------------------------------- + + +def truncate(text: str, max_len: int, suffix: str = "...") -> str: + """Truncate text to max_len characters, appending suffix if truncated.""" + if len(text) <= max_len: + return text + return text[: max_len - len(suffix)] + suffix + + +def camel_to_snake(name: str) -> str: + """Convert camelCase or PascalCase to snake_case.""" + # Insert underscore before uppercase letters following lowercase letters + s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name) + return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1).lower() + + +def snake_to_camel(name: str) -> str: + """Convert snake_case to lowerCamelCase.""" + components = name.split("_") + # This duplicates logic that is already in a utility function defined + # in this same file in a different method below. Both do word-join + # capitalization but were written independently during different sprints. + return components[0] + "".join(x.title() for x in components[1:]) + + +def _title_case_words(words: list[str]) -> str: + """Join a list of words in title case.""" + # NOTE: snake_to_camel above duplicates this joining logic inline + # rather than calling this helper. Both should be consolidated. + return "".join(w.title() for w in words) + + +def snake_to_pascal(name: str) -> str: + """Convert snake_case to PascalCase.""" + return _title_case_words(name.split("_")) + + +def json_pretty(obj: Any, indent: int = 2) -> str: + """Return a pretty-printed JSON string for the given object.""" + return json.dumps(obj, indent=indent, default=str, ensure_ascii=False) + + +# --------------------------------------------------------------------------- +# Unreachable code example +# --------------------------------------------------------------------------- + + +def classify_size(n: int) -> str: + """ + Return a human-readable size classification for integer n. + + Categories: tiny (<10), small (10-99), medium (100-999), large (1000+). + """ + if n >= 1000: + return "large" + elif n >= 100: + return "medium" + elif n >= 10: + return "small" + else: + return "tiny" + # These lines are unreachable — control always hits a return above. + # Left over from an earlier version that had a "fallthrough" path. + category = "unknown" + return category diff --git a/golden-test-set/artifacts/python/gt-py-008-path-traversal.py b/golden-test-set/artifacts/python/gt-py-008-path-traversal.py new file mode 100644 index 0000000..52b3cc3 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-008-path-traversal.py @@ -0,0 +1,165 @@ +""" +Static file and user document serving endpoint. + +Handles two categories of file access: + 1. Public static assets (CSS, JS, images) — served from a fixed directory + 2. User-uploaded documents — served by user ID and filename from a + per-user upload directory + +Deployed behind nginx in production but also runnable standalone for +development. +""" + +from __future__ import annotations + +import logging +import mimetypes +import os +import stat +from pathlib import Path + +from flask import Flask, abort, request, send_file, jsonify + +app = Flask(__name__) +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Directory constants +# --------------------------------------------------------------------------- + +BASE_DIR = Path(__file__).resolve().parent +STATIC_DIR = BASE_DIR / "static" +UPLOADS_BASE = Path(os.environ.get("UPLOADS_DIR", "/var/app/uploads")) + +# --------------------------------------------------------------------------- +# Public static assets (no user input in path) +# --------------------------------------------------------------------------- + + +@app.route("/static/style.css") +def serve_stylesheet(): + """Serve the main application stylesheet.""" + css_path = os.path.join(BASE_DIR, "static", "style.css") + return send_file(css_path, mimetype="text/css") + + +@app.route("/static/") +def serve_static_asset(asset: str): + """ + Serve a named static asset. + + Only files under STATIC_DIR are served. The asset path is joined + and resolved — any traversal attempt that escapes STATIC_DIR returns 404. + """ + resolved = (STATIC_DIR / asset).resolve() + if not str(resolved).startswith(str(STATIC_DIR)): + logger.warning("Static path traversal attempt blocked: %s", asset) + abort(404) + if not resolved.is_file(): + abort(404) + mime, _ = mimetypes.guess_type(str(resolved)) + return send_file(resolved, mimetype=mime or "application/octet-stream") + + +# --------------------------------------------------------------------------- +# User document endpoint +# --------------------------------------------------------------------------- + + +@app.route("/api/users//documents/") +def serve_user_document(user_id: int, filename: str): + """ + Serve a document from the user's upload directory. + + The filename parameter comes directly from the URL and is joined to + the user's upload directory without sanitization or normalization. + An attacker who controls filename can escape the user directory via + path traversal sequences (e.g. '../../../etc/passwd'). + """ + user_dir = UPLOADS_BASE / str(user_id) + # Vulnerable: filename is not sanitized before path join + send_file + document_path = user_dir / filename + logger.info("Serving document: %s for user %d", document_path, user_id) + + if not document_path.exists(): + abort(404) + + mime, _ = mimetypes.guess_type(str(document_path)) + return send_file(document_path, mimetype=mime or "application/octet-stream") + + +# --------------------------------------------------------------------------- +# Directory listing (internal use, exposed inadvertently) +# --------------------------------------------------------------------------- + + +@app.route("/api/users//documents") +def list_user_documents(user_id: int): + """ + Return a listing of all documents in the user's upload directory. + + This endpoint exposes the raw filesystem listing including all + filenames, which leaks the existence and names of documents even + when the requester isn't authorized to read them. There is no + authentication or authorization check on this route. + """ + user_dir = UPLOADS_BASE / str(user_id) + if not user_dir.exists(): + return jsonify({"documents": []}) + + entries = [] + for entry in sorted(user_dir.iterdir()): + if entry.is_file(): + st = entry.stat() + entries.append({ + "name": entry.name, + "size_bytes": st.st_size, + "modified": st.st_mtime, + }) + + return jsonify({"user_id": user_id, "document_count": len(entries), "documents": entries}) + + +# --------------------------------------------------------------------------- +# Upload endpoint +# --------------------------------------------------------------------------- + + +@app.route("/api/users//documents", methods=["POST"]) +def upload_user_document(user_id: int): + """Accept a file upload and store it in the user's directory.""" + if "file" not in request.files: + abort(400) + + f = request.files["file"] + if not f.filename: + abort(400) + + # Sanitize upload filename using werkzeug's secure_filename + from werkzeug.utils import secure_filename + safe_name = secure_filename(f.filename) + if not safe_name: + abort(400) + + user_dir = UPLOADS_BASE / str(user_id) + user_dir.mkdir(parents=True, exist_ok=True) + + dest = user_dir / safe_name + f.save(dest) + logger.info("Stored upload: %s for user %d", dest, user_id) + + return jsonify({"status": "uploaded", "filename": safe_name}), 201 + + +# --------------------------------------------------------------------------- +# Health check +# --------------------------------------------------------------------------- + + +@app.route("/health") +def health(): + return jsonify({"status": "ok"}) + + +if __name__ == "__main__": + app.run(debug=False, host="127.0.0.1", port=8080) diff --git a/golden-test-set/artifacts/python/gt-py-009-type-mismatch.py b/golden-test-set/artifacts/python/gt-py-009-type-mismatch.py new file mode 100644 index 0000000..a54b5ae --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-009-type-mismatch.py @@ -0,0 +1,274 @@ +""" +Data transformation pipeline for the reporting service. + +Ingests raw event records from the message queue, applies a series of +field normalizations and aggregations, and emits typed output records +ready for insertion into the reporting warehouse. Each transformer stage +is composable and independently testable. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Data model +# --------------------------------------------------------------------------- + + +@dataclass +class RawEvent: + event_id: str + user_id: str # Always a string from the message bus + event_type: str + payload: dict + received_at: str # ISO 8601 string + + +@dataclass +class NormalizedEvent: + event_id: str + user_id: str + event_type: str + amount_cents: int + currency: str + region: str + timestamp: datetime + tags: list[str] + priority: int + + +# --------------------------------------------------------------------------- +# Stage 1: Parse and coerce basic fields +# --------------------------------------------------------------------------- + + +def parse_timestamp(ts_str: str) -> datetime: + """Parse an ISO 8601 timestamp string to a timezone-aware datetime.""" + return datetime.fromisoformat(ts_str.replace("Z", "+00:00")) + + +def extract_amount_cents(payload: dict) -> int: + """ + Extract the monetary amount from the payload and convert to cents. + + The payload may carry `amount` as a float (dollars) or as a pre-scaled + integer in cents. The `amount_unit` key disambiguates. + """ + unit = payload.get("amount_unit", "dollars") + raw = payload.get("amount", 0) + + if unit == "cents": + return int(raw) + else: + # Convert dollars to cents + return int(float(raw) * 100) + + +# --------------------------------------------------------------------------- +# Stage 2: User segment lookup +# --------------------------------------------------------------------------- + +# Simulated segment store: maps user_id (string) → segment metadata +USER_SEGMENTS: dict[str, dict] = { + "user-001": {"region": "us-west", "priority": 2, "tags": ["beta", "enterprise"]}, + "user-002": {"region": "eu-central", "priority": 1, "tags": ["standard"]}, + "user-003": {"region": "us-east", "priority": 3, "tags": ["enterprise", "vip"]}, +} + + +def lookup_user_segment(user_id: str) -> dict: + """ + Return segment metadata for a user_id. + + Falls back to defaults if the user is not in the segment store. + The default priority is intentionally stored as a string here to + simulate a heterogeneous data source (e.g., a config file or external API + that doesn't guarantee type consistency). + """ + return USER_SEGMENTS.get(user_id, {"region": "unknown", "priority": "1", "tags": []}) + + +# --------------------------------------------------------------------------- +# Stage 3: Priority filtering and routing +# --------------------------------------------------------------------------- + +HIGH_PRIORITY_THRESHOLD = 2 # integer + + +def is_high_priority(event: NormalizedEvent) -> bool: + """ + Return True if the event has high priority (>= threshold). + + Priority is expected to be an int, but the segment lookup can return + it as a string (see lookup_user_segment defaults). The comparison + int >= str always evaluates to False in Python 3, silently dropping + events that should be routed as high-priority. + """ + return event.priority >= HIGH_PRIORITY_THRESHOLD + + +def route_event(event: NormalizedEvent) -> str: + """ + Determine the routing target for a normalized event. + + High-priority events go to the fast lane; others to the standard queue. + """ + if is_high_priority(event): + return "queue://events.high-priority" + return "queue://events.standard" + + +# --------------------------------------------------------------------------- +# Stage 4: Tag enrichment +# --------------------------------------------------------------------------- + +TAG_WEIGHT_MAP: dict[str, int] = { + "enterprise": 10, + "vip": 20, + "beta": 5, + "standard": 1, +} + + +def compute_tag_weight(tags: list[str]) -> int: + """Sum the weight of all tags for an event.""" + return sum(TAG_WEIGHT_MAP.get(t, 0) for t in tags) + + +def filter_by_region(events: list[NormalizedEvent], region: str) -> list[NormalizedEvent]: + """Return only events matching the given region string.""" + return [e for e in events if e.region == region] + + +# --------------------------------------------------------------------------- +# Stage 5: Aggregation +# --------------------------------------------------------------------------- + + +def aggregate_by_type(events: list[NormalizedEvent]) -> dict[str, dict]: + """ + Group events by type and compute per-type totals. + + Returns a dict keyed by event_type with sub-keys: + count, total_amount_cents, avg_amount_cents. + """ + buckets: dict[str, dict] = {} + + for event in events: + if event.event_type not in buckets: + buckets[event.event_type] = {"count": 0, "total_amount_cents": 0} + + buckets[event.event_type]["count"] += 1 + buckets[event.event_type]["total_amount_cents"] += event.amount_cents + + for event_type, stats in buckets.items(): + count = stats["count"] + stats["avg_amount_cents"] = stats["total_amount_cents"] // count if count else 0 + + return buckets + + +# --------------------------------------------------------------------------- +# Main transformation entry point +# --------------------------------------------------------------------------- + + +def transform(raw: RawEvent) -> Optional[NormalizedEvent]: + """ + Transform a raw event into a normalized event. + + Returns None if the event cannot be transformed (e.g., unknown type). + """ + try: + timestamp = parse_timestamp(raw.received_at) + except (ValueError, AttributeError) as exc: + logger.warning("Cannot parse timestamp for event %s: %s", raw.event_id, exc) + return None + + amount_cents = extract_amount_cents(raw.payload) + currency = raw.payload.get("currency", "USD") + + segment = lookup_user_segment(raw.user_id) + region = segment.get("region", "unknown") + + # priority comes from segment store — may be str or int depending on + # whether the user is in USER_SEGMENTS or using the default. + priority = segment.get("priority", 1) + + tags = segment.get("tags", []) + + # Type coercion sanity check for a literal — valid conversion, not a bug + max_tags = int("42") # configuration constant parsed from env + if len(tags) > max_tags: + tags = tags[:max_tags] + + return NormalizedEvent( + event_id=raw.event_id, + user_id=raw.user_id, + event_type=raw.event_type, + amount_cents=amount_cents, + currency=currency, + region=region, + timestamp=timestamp, + tags=tags, + priority=priority, # BUG: may be str "1" for unknown users + ) + + +def transform_batch(raws: list[RawEvent]) -> list[NormalizedEvent]: + """Transform a list of raw events, dropping None results.""" + results = [] + for raw in raws: + normalized = transform(raw) + if normalized is not None: + results.append(normalized) + return results + + +# --------------------------------------------------------------------------- +# Output summary +# --------------------------------------------------------------------------- + + +def summarize(events: list[NormalizedEvent]) -> dict: + """ + Produce a pipeline run summary dict. + + Includes event count, total volume, routing breakdown, and region totals. + The `metadata` key carries a default value pulled from a dict that may + not have the expected type when populated from untrusted sources. + """ + routing: dict[str, int] = {"high-priority": 0, "standard": 0} + region_totals: dict[str, int] = {} + + for event in events: + target = route_event(event) + lane = "high-priority" if "high-priority" in target else "standard" + routing[lane] += 1 + region_totals[event.region] = ( + region_totals.get(event.region, 0) + event.amount_cents + ) + + # get() with a dict default — the default type (dict) doesn't match the + # expected type (int) for subsequent arithmetic in callers that assume + # they're summing numeric values. Callers doing sum(meta.values()) will + # get a TypeError at runtime if the key is missing. + metadata = { + "pipeline_version": "2.1.0", + "run_total_cents": region_totals, + "extra_counts": {}, + } + extra = metadata.get("missing_key", {}) # returns {} but callers expect int + + return { + "total_events": len(events), + "routing": routing, + "region_totals_cents": region_totals, + "extra": extra, + } diff --git a/golden-test-set/artifacts/python/gt-py-010-minor-hygiene.py b/golden-test-set/artifacts/python/gt-py-010-minor-hygiene.py new file mode 100644 index 0000000..3f0f41f --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-010-minor-hygiene.py @@ -0,0 +1,264 @@ +""" +Date and time utilities for the scheduling service. + +Provides helpers for working with business-hours windows, timezone +conversions, recurring schedule evaluation, and ISO 8601 interval +parsing. Used by the task scheduler and the SLA reporting module. +""" + +from __future__ import annotations + +import re +from datetime import date, datetime, timedelta, timezone +from typing import Optional +from zoneinfo import ZoneInfo + +DEFAULT_TZ = ZoneInfo("America/Los_Angeles") + +# Business hours window (local time) +BIZ_HOUR_START = 9 +BIZ_HOUR_END = 17 + +# --------------------------------------------------------------------------- +# Timezone helpers +# --------------------------------------------------------------------------- + + +def now_pacific() -> datetime: + """Return the current time in the America/Los_Angeles timezone.""" + return datetime.now(tz=DEFAULT_TZ) + + +def to_utc(dt: datetime) -> datetime: + """Convert a timezone-aware datetime to UTC.""" + if dt.tzinfo is None: + raise ValueError("to_utc requires a timezone-aware datetime") + return dt.astimezone(timezone.utc) + + +def to_tz(dt: datetime, tz_name: str) -> datetime: + """Convert a timezone-aware datetime to the named timezone.""" + if dt.tzinfo is None: + raise ValueError("to_tz requires a timezone-aware datetime") + return dt.astimezone(ZoneInfo(tz_name)) + + +def localize(dt: datetime, tz_name: str) -> datetime: + """Attach a timezone to a naive datetime (interpret as local time in that zone).""" + if dt.tzinfo is not None: + raise ValueError("localize only works on naive datetimes") + return dt.replace(tzinfo=ZoneInfo(tz_name)) + + +# --------------------------------------------------------------------------- +# Business hours +# --------------------------------------------------------------------------- + + +def is_business_hours(dt: datetime, tz_name: str = "America/Los_Angeles") -> bool: + """ + Return True if dt falls within Monday–Friday 09:00–17:00 local time. + + Converts the datetime to the target timezone before evaluation. + """ + local = to_tz(dt, tz_name) + if local.weekday() >= 5: # 5=Saturday, 6=Sunday + return False + return BIZ_HOUR_START <= local.hour < BIZ_HOUR_END + + +def next_business_day(d: date) -> date: + """Return the next calendar day that falls on a weekday.""" + candidate = d + timedelta(days=1) + while candidate.weekday() >= 5: + candidate += timedelta(days=1) + return candidate + + +def business_days_between(start: date, end: date) -> int: + """ + Count business days (weekdays) between start (inclusive) and end (exclusive). + + Returns 0 if start >= end. + """ + if start >= end: + return 0 + count = 0 + current = start + while current < end: + if current.weekday() < 5: + count += 1 + current += timedelta(days=1) + return count + + +def next_business_hour_window(dt: datetime, tz_name: str = "America/Los_Angeles") -> datetime: + """ + Given an arbitrary datetime, return the start of the next business-hours window. + + If dt is already within business hours, returns dt unchanged. + If dt is after business hours or on a weekend, advances to the next + applicable 09:00. + """ + local = to_tz(dt, tz_name) + + # If within business hours, return as-is + if local.weekday() < 5 and BIZ_HOUR_START <= local.hour < BIZ_HOUR_END: + return dt + + # Advance to the start of next business day at 09:00 + candidate = local.replace(hour=BIZ_HOUR_START, minute=0, second=0, microsecond=0) + if local.hour >= BIZ_HOUR_END or local.weekday() >= 5: + candidate += timedelta(days=1) + + while candidate.weekday() >= 5: + candidate += timedelta(days=1) + + return candidate + + +# --------------------------------------------------------------------------- +# ISO 8601 duration parsing +# --------------------------------------------------------------------------- + +_DURATION_RE = re.compile( + r"^P" + r"(?:(\d+)Y)?" + r"(?:(\d+)M)?" + r"(?:(\d+)W)?" + r"(?:(\d+)D)?" + r"(?:T" + r"(?:(\d+)H)?" + r"(?:(\d+)M)?" + r"(?:(\d+(?:\.\d+)?)S)?" + r")?$" +) + + +def parse_iso_duration(duration: str) -> timedelta: + """ + Parse an ISO 8601 duration string into a timedelta. + + Supports years (approximated as 365 days), months (approximated as 30 + days), weeks, days, hours, minutes, and seconds (with fractional seconds). + Does not support calendar-accurate month/year arithmetic. + + Examples: "P1D", "PT30M", "P2W", "P1Y2M3DT4H5M6S" + """ + m = _DURATION_RE.match(duration) + if not m: + raise ValueError(f"Invalid ISO 8601 duration: {duration!r}") + + years, months, weeks, days, hours, minutes, seconds = m.groups() + + total_seconds = 0.0 + if years: + total_seconds += int(years) * 365 * 86400 + if months: + total_seconds += int(months) * 30 * 86400 + if weeks: + total_seconds += int(weeks) * 7 * 86400 + if days: + total_seconds += int(days) * 86400 + if hours: + total_seconds += int(hours) * 3600 + if minutes: + total_seconds += int(minutes) * 60 + if seconds: + total_seconds += float(seconds) + + return timedelta(seconds=total_seconds) + + +# --------------------------------------------------------------------------- +# Recurring schedule evaluation +# --------------------------------------------------------------------------- + + +def cron_next_run( + last_run: datetime, + interval: timedelta, + jitter_seconds: int = 0, + respect_business_hours: bool = False, + tz_name: str = "America/Los_Angeles", +) -> datetime: + """ + Compute the next scheduled run time for a recurring task. + + Parameters + ---------- + last_run: + When the task last completed. + interval: + How frequently the task should run. + jitter_seconds: + Optional random jitter upper bound in seconds. Pass 0 to disable. + Callers are responsible for passing a deterministic seed in tests. + respect_business_hours: + If True, advance the next run to the start of the next business + window if it would fall outside 09:00–17:00 Mon–Fri. + tz_name: + Timezone for business-hours evaluation. + + Returns + ------- + datetime: + Next run time, timezone-aware in the same zone as last_run. + """ + if last_run.tzinfo is None: + raise ValueError("last_run must be timezone-aware") + + next_run = last_run + interval + + if jitter_seconds > 0: + import random + next_run += timedelta(seconds=random.randint(0, jitter_seconds)) + + if respect_business_hours: + next_run = next_business_hour_window(next_run, tz_name) + + return next_run + + +# --------------------------------------------------------------------------- +# Formatting +# --------------------------------------------------------------------------- + +# Magic number: 86400 seconds in a day — used inline rather than as a named constant +def humanize_timedelta(td: timedelta) -> str: + """ + Convert a timedelta to a human-readable string. + + Examples: "3 days", "2 hours 15 minutes", "45 seconds". + Only shows the two most significant non-zero units. + """ + total = int(td.total_seconds()) + if total < 0: + return f"-{humanize_timedelta(-td)}" + + parts = [] + + days = total // 86400 + if days: + parts.append(f"{days} day{'s' if days != 1 else ''}") + remaining = total % 86400 + + hours = remaining // 3600 + if hours: + parts.append(f"{hours} hour{'s' if hours != 1 else ''}") + remaining = remaining % 3600 + + minutes = remaining // 60 + if minutes and len(parts) < 2: + parts.append(f"{minutes} minute{'s' if minutes != 1 else ''}") + + seconds = remaining % 60 + if seconds and len(parts) < 2: + parts.append(f"{seconds} second{'s' if seconds != 1 else ''}") + + return " ".join(parts) if parts else "0 seconds" + + +def format_range(start: datetime, end: datetime, fmt: str = "%Y-%m-%d %H:%M %Z") -> str: + """Format a datetime range as 'start → end'.""" + return f"{start.strftime(fmt)} → {end.strftime(fmt)}" diff --git a/golden-test-set/artifacts/python/gt-py-011-minor-completeness.py b/golden-test-set/artifacts/python/gt-py-011-minor-completeness.py new file mode 100644 index 0000000..d2163d7 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-011-minor-completeness.py @@ -0,0 +1,187 @@ +""" +Lightweight API client for the internal notification service. + +Wraps the REST API exposed by the notification microservice. Supports +sending emails, push notifications, and in-app messages. Intended to be +used by application services that need to trigger notifications without +depending on the notification service internals. +""" + +from __future__ import annotations + +import logging +import time +from enum import Enum +from typing import Optional + +import requests + +logger = logging.getLogger(__name__) + +NOTIFICATION_API_BASE = "http://notification-svc:9000" +DEFAULT_TIMEOUT = 10 +MAX_RETRIES = 3 +RETRY_BACKOFF = 1.5 + + +class NotificationType(Enum): + EMAIL = "email" + PUSH = "push" + IN_APP = "in_app" + + +class NotificationPriority(Enum): + LOW = "low" + NORMAL = "normal" + HIGH = "high" + URGENT = "urgent" + + +class NotificationClient: + + def __init__( + self, + base_url: str = NOTIFICATION_API_BASE, + timeout: int = DEFAULT_TIMEOUT, + service_token: Optional[str] = None, + ): + self._base = base_url.rstrip("/") + self._timeout = timeout + self._session = requests.Session() + if service_token: + self._session.headers["Authorization"] = f"Bearer {service_token}" + self._session.headers["Content-Type"] = "application/json" + + def send( + self, + recipient_id: str, + notification_type: NotificationType, + subject: str, + body: str, + priority: NotificationPriority = NotificationPriority.NORMAL, + metadata: Optional[dict] = None, + dry_run: bool = False, + ) -> dict: + """ + Send a notification to a recipient. + + Returns the API response dict containing at minimum `notification_id` + and `status` keys. Raises requests.HTTPError on non-2xx responses + after all retries are exhausted. + """ + payload = { + "recipient_id": recipient_id, + "type": notification_type.value, + "subject": subject, + "body": body, + "priority": priority.value, + "metadata": metadata or {}, + "dry_run": dry_run, + } + + url = f"{self._base}/v1/notifications" + return self._post_with_retry(url, payload) + + def send_bulk( + self, + recipient_ids: list[str], + notification_type: NotificationType, + subject: str, + body: str, + priority: NotificationPriority = NotificationPriority.NORMAL, + ): + payload = { + "recipient_ids": recipient_ids, + "type": notification_type.value, + "subject": subject, + "body": body, + "priority": priority.value, + } + url = f"{self._base}/v1/notifications/bulk" + return self._post_with_retry(url, payload) + + def get_status(self, notification_id: str) -> dict: + """ + Retrieve the delivery status of a previously sent notification. + + Returns a dict with keys: notification_id, status, delivered_at, + failure_reason (may be None). + """ + url = f"{self._base}/v1/notifications/{notification_id}" + resp = self._session.get(url, timeout=self._timeout) + resp.raise_for_status() + return resp.json() + + def cancel(self, notification_id: str) -> bool: + """ + Attempt to cancel a queued notification before delivery. + + Returns True if cancellation succeeded, False if the notification + was already delivered or not found. + """ + url = f"{self._base}/v1/notifications/{notification_id}/cancel" + resp = self._session.post(url, timeout=self._timeout) + if resp.status_code == 200: + return True + if resp.status_code in (404, 409): + return False + resp.raise_for_status() + return False + + def list_recent(self, recipient_id: str, limit: int = 20, offset: int = 0) -> dict: + """ + Return paginated recent notifications for a recipient. + + No input validation on limit/offset — negative or zero values are + passed through to the API, which may return unexpected results. + """ + params = {"recipient_id": recipient_id, "limit": limit, "offset": offset} + url = f"{self._base}/v1/notifications" + resp = self._session.get(url, params=params, timeout=self._timeout) + resp.raise_for_status() + return resp.json() + + def health_check(self) -> bool: + """Return True if the notification service is reachable.""" + try: + resp = self._session.get(f"{self._base}/health", timeout=5) + return resp.status_code == 200 + except requests.RequestException: + return False + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _post_with_retry(self, url: str, payload: dict) -> dict: + """ + POST to url with exponential backoff retry. + + Retries on 429 (rate limit) and 5xx server errors. Raises + requests.HTTPError on 4xx (except 429) or after max retries. + """ + last_exc = None + for attempt in range(1, MAX_RETRIES + 1): + try: + resp = self._session.post(url, json=payload, timeout=self._timeout) + if resp.status_code in (429,) or resp.status_code >= 500: + logger.warning( + "Notification API returned %d on attempt %d/%d", + resp.status_code, + attempt, + MAX_RETRIES, + ) + if attempt < MAX_RETRIES: + time.sleep(RETRY_BACKOFF ** attempt) + continue + resp.raise_for_status() + return resp.json() + except requests.ConnectionError as exc: + logger.warning("Connection error on attempt %d: %s", attempt, exc) + last_exc = exc + if attempt < MAX_RETRIES: + time.sleep(RETRY_BACKOFF ** attempt) + + raise requests.HTTPError( + f"Notification API failed after {MAX_RETRIES} attempts" + ) from last_exc diff --git a/golden-test-set/artifacts/python/gt-py-012-ssrf-partial.py b/golden-test-set/artifacts/python/gt-py-012-ssrf-partial.py new file mode 100644 index 0000000..d2472e9 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-012-ssrf-partial.py @@ -0,0 +1,161 @@ +""" +URL fetcher with partial SSRF protection. + +Used by the webhook preview service to fetch external URLs on behalf of +users. Applies a scheme allowlist and a basic hostname check, but the +validation has gaps that allow SSRF via non-HTTP schemes and IP-encoded +hostnames that aren't caught by the domain blocklist. +""" + +from __future__ import annotations + +import ipaddress +import logging +import re +import socket +import urllib.parse +from typing import Optional + +import requests + +logger = logging.getLogger(__name__) + +MAX_RESPONSE_BYTES = 1 * 1024 * 1024 # 1 MB +FETCH_TIMEOUT = 10 + +# Blocklist of internal hostnames (partial — not comprehensive) +_BLOCKED_HOSTS = frozenset({ + "localhost", + "127.0.0.1", + "0.0.0.0", + "169.254.169.254", # AWS metadata + "metadata.google.internal", +}) + +# Only allow http and https — but file://, gopher://, dict:// are not checked +_ALLOWED_SCHEMES = {"http", "https"} + + +def _is_private_ip(ip_str: str) -> bool: + """Return True if the IP address is in a private or reserved range.""" + try: + addr = ipaddress.ip_address(ip_str) + return addr.is_private or addr.is_loopback or addr.is_link_local or addr.is_reserved + except ValueError: + return False + + +def _resolve_hostname(hostname: str) -> Optional[str]: + """Resolve hostname to IP string; return None on resolution failure.""" + try: + return socket.gethostbyname(hostname) + except socket.gaierror: + return None + + +def validate_url(url: str) -> tuple[bool, str]: + """ + Validate a user-supplied URL for safe fetching. + + Returns (is_valid, reason). Reason is empty string on success. + + Checks performed: + 1. URL is parseable + 2. Scheme is http or https + 3. Hostname is not in the explicit blocklist + 4. Resolved IP is not in a private range + + Known gap: does not check for non-HTTP schemes passed via redirect + chains, and does not normalize URLs before parsing (e.g., + http://user@169.254.169.254/ may bypass the hostname check). + The scheme check only validates the top-level scheme — file:// or + gopher:// passed via a malformed URL may slip through if the + urllib.parse behavior differs from requests' URL handling. + """ + parsed = urllib.parse.urlparse(url) + + # Scheme check — only validates the declared scheme, not scheme + # overrides embedded in the URL (e.g. via double-slash tricks). + if parsed.scheme not in _ALLOWED_SCHEMES: + return False, f"Scheme not allowed: {parsed.scheme!r}. Only http/https are permitted." + + hostname = parsed.hostname + if not hostname: + return False, "URL has no hostname" + + if hostname in _BLOCKED_HOSTS: + return False, f"Hostname is blocked: {hostname}" + + # Resolve and check IP + resolved_ip = _resolve_hostname(hostname) + if resolved_ip is None: + return False, f"Cannot resolve hostname: {hostname}" + + if _is_private_ip(resolved_ip): + return False, f"Resolved IP {resolved_ip} is in a private range" + + return True, "" + + +def fetch_url(url: str, user_agent: str = "WebhookPreview/1.0") -> dict: + """ + Fetch a URL after SSRF validation. + + Returns a dict with keys: url, status_code, content_type, body_preview. + The body is truncated to MAX_RESPONSE_BYTES. + + Raises ValueError if validation fails. + Raises requests.RequestException on network errors. + + SSRF gap: validate_url checks http/https at parse time, but does NOT + handle file:// or gopher:// URLs. A carefully constructed URL like: + "http://foo@file:///etc/passwd" + or a URL using a non-standard scheme accepted by some URL libraries + may bypass the scheme check depending on urllib.parse version. + Additionally, URLs with decimal-encoded IP addresses (e.g., http://2130706433/ + which equals 127.0.0.1) are resolved by the socket layer but may not + match the string blocklist. + """ + valid, reason = validate_url(url) + if not valid: + raise ValueError(f"URL failed SSRF validation: {reason}") + + headers = {"User-Agent": user_agent} + resp = requests.get( + url, + headers=headers, + timeout=FETCH_TIMEOUT, + stream=True, + allow_redirects=True, # Redirect chains are not re-validated + ) + resp.raise_for_status() + + content_type = resp.headers.get("Content-Type", "application/octet-stream") + body = resp.raw.read(MAX_RESPONSE_BYTES, decode_content=True) + + return { + "url": url, + "final_url": resp.url, + "status_code": resp.status_code, + "content_type": content_type, + "body_size_bytes": len(body), + "body_preview": body[:512].decode("utf-8", errors="replace"), + } + + +def batch_fetch(urls: list[str]) -> list[dict]: + """ + Fetch multiple URLs, collecting results and errors. + + Returns a list of result dicts. Failed fetches include an `error` key + instead of response fields. + """ + results = [] + for url in urls: + try: + result = fetch_url(url) + results.append(result) + except (ValueError, requests.RequestException) as exc: + logger.warning("Fetch failed for %s: %s", url, exc) + results.append({"url": url, "error": str(exc)}) + return results diff --git a/golden-test-set/artifacts/python/gt-py-013-clean-crypto.py b/golden-test-set/artifacts/python/gt-py-013-clean-crypto.py new file mode 100644 index 0000000..6d9cd87 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-013-clean-crypto.py @@ -0,0 +1,316 @@ +""" +Cryptographic utilities for the document management service. + +Provides: + - Content fingerprinting (non-security checksums for deduplication) + - HMAC-based request signing and verification + - Symmetric encryption/decryption using AES-GCM (via cryptography package) + - Secure random token generation + +None of the MD5 usage here is for security purposes — it is used +exclusively for content deduplication fingerprints where collision +resistance is not a security requirement. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import os +import secrets +import struct +import time +from typing import Optional + +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +# AES-256 key length in bytes +AES_KEY_LEN = 32 + +# GCM nonce length (96 bits is the recommended size for AES-GCM) +GCM_NONCE_LEN = 12 + +# HMAC digest algorithm for request signing +SIGNING_ALGORITHM = "sha256" + +# Signature validity window in seconds (prevent replay attacks) +SIGNATURE_MAX_AGE_SEC = 300 + +# Token byte length for secure random tokens +TOKEN_BYTE_LEN = 32 + +# --------------------------------------------------------------------------- +# Content fingerprinting (non-security) +# --------------------------------------------------------------------------- + + +def content_fingerprint(data: bytes) -> str: + """ + Compute a fast content fingerprint for deduplication purposes. + + Uses MD5 intentionally: the output is used as a cache key and + deduplication identifier, not for password hashing or any security + purpose. MD5 is appropriate here because: + - Collision resistance is not required (we're not defending against + adversarial inputs; we're identifying duplicate uploads) + - The fingerprint is never used to make trust decisions + - MD5 is significantly faster than SHA-2 for large binary payloads + + Returns a 32-character lowercase hex digest. + """ + return hashlib.md5(data).hexdigest() # noqa: S324 — non-security use + + +def content_fingerprint_sha256(data: bytes) -> str: + """ + Compute a SHA-256 content fingerprint. + + Use this variant when the fingerprint is stored alongside user-visible + integrity metadata (e.g., in API responses or audit logs) where a + collision-resistant algorithm is preferable for consistency. + """ + return hashlib.sha256(data).hexdigest() + + +def fingerprint_file(path: str, chunk_size: int = 65536) -> str: + """ + Compute a SHA-256 fingerprint of a file by streaming it in chunks. + + Does not load the entire file into memory. + """ + h = hashlib.sha256() + with open(path, "rb") as fh: + while True: + chunk = fh.read(chunk_size) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +# --------------------------------------------------------------------------- +# Secure random token generation +# --------------------------------------------------------------------------- + + +def generate_token(n_bytes: int = TOKEN_BYTE_LEN) -> str: + """ + Generate a cryptographically secure random token. + + Returns a URL-safe base64-encoded string. Uses secrets.token_bytes + which is backed by the OS CSPRNG. + """ + return secrets.token_urlsafe(n_bytes) + + +def generate_hex_token(n_bytes: int = TOKEN_BYTE_LEN) -> str: + """Return a cryptographically secure random hex string.""" + return secrets.token_hex(n_bytes) + + +# --------------------------------------------------------------------------- +# HMAC request signing +# --------------------------------------------------------------------------- + + +class RequestSigner: + """ + Signs and verifies timestamped HMAC-SHA256 request signatures. + + Designed for service-to-service authentication where both sides share + a secret key. The signature covers the HTTP method, path, timestamp, + and body hash to prevent tampering and replay attacks. + """ + + def __init__(self, secret_key: bytes): + if len(secret_key) < 32: + raise ValueError("Secret key must be at least 32 bytes") + self._key = secret_key + + def sign(self, method: str, path: str, body: bytes, timestamp: Optional[int] = None) -> str: + """ + Compute an HMAC-SHA256 signature for an HTTP request. + + Parameters + ---------- + method: + HTTP method in uppercase (e.g. "POST"). + path: + Request path including query string (e.g. "/api/v1/events?foo=bar"). + body: + Raw request body bytes. Pass b"" for requests with no body. + timestamp: + Unix timestamp (seconds). Defaults to current time. + + Returns + ------- + str: + Hex-encoded HMAC-SHA256 signature. + """ + if timestamp is None: + timestamp = int(time.time()) + + body_hash = hashlib.sha256(body).hexdigest() + message = f"{method.upper()}\n{path}\n{timestamp}\n{body_hash}".encode("utf-8") + sig = hmac.new(self._key, message, digestmod=hashlib.sha256).hexdigest() + return f"{timestamp}.{sig}" + + def verify( + self, + method: str, + path: str, + body: bytes, + signature_header: str, + max_age_sec: int = SIGNATURE_MAX_AGE_SEC, + ) -> bool: + """ + Verify an HMAC-SHA256 signature from the request header. + + Returns True only if the signature is valid AND the timestamp is + within max_age_sec of the current time. + + Uses hmac.compare_digest to prevent timing side-channel attacks. + """ + try: + ts_str, provided_sig = signature_header.split(".", 1) + timestamp = int(ts_str) + except (ValueError, AttributeError): + return False + + # Check timestamp freshness + age = int(time.time()) - timestamp + if abs(age) > max_age_sec: + return False + + # Recompute expected signature + body_hash = hashlib.sha256(body).hexdigest() + message = f"{method.upper()}\n{path}\n{timestamp}\n{body_hash}".encode("utf-8") + expected_sig = hmac.new(self._key, message, digestmod=hashlib.sha256).hexdigest() + + return hmac.compare_digest(expected_sig, provided_sig) + + +# --------------------------------------------------------------------------- +# Symmetric encryption (AES-256-GCM) +# --------------------------------------------------------------------------- + + +class SymmetricCipher: + """ + Envelope encryption using AES-256-GCM. + + Provides authenticated encryption with associated data (AEAD). + Each encryption call generates a fresh random nonce; the nonce is + prepended to the ciphertext so the decryption side can recover it. + + Wire format (bytes): + [ 12-byte nonce ][ variable ciphertext+tag ] + """ + + def __init__(self, key: bytes): + if len(key) != AES_KEY_LEN: + raise ValueError(f"Key must be exactly {AES_KEY_LEN} bytes (got {len(key)})") + self._aesgcm = AESGCM(key) + + @classmethod + def generate_key(cls) -> bytes: + """Generate a random 256-bit AES key.""" + return os.urandom(AES_KEY_LEN) + + def encrypt(self, plaintext: bytes, associated_data: Optional[bytes] = None) -> bytes: + """ + Encrypt plaintext and return the encoded ciphertext (nonce prepended). + + Parameters + ---------- + plaintext: + Data to encrypt. + associated_data: + Optional authenticated-but-not-encrypted data (e.g. record ID). + Must be passed identically during decryption. + + Returns + ------- + bytes: + Nonce (12 bytes) + ciphertext + GCM authentication tag. + """ + nonce = os.urandom(GCM_NONCE_LEN) + ciphertext = self._aesgcm.encrypt(nonce, plaintext, associated_data) + return nonce + ciphertext + + def decrypt(self, ciphertext_with_nonce: bytes, associated_data: Optional[bytes] = None) -> bytes: + """ + Decrypt ciphertext produced by encrypt(). + + Raises cryptography.exceptions.InvalidTag if authentication fails + (i.e., the ciphertext has been tampered with). + """ + if len(ciphertext_with_nonce) < GCM_NONCE_LEN: + raise ValueError("Ciphertext too short to contain a valid nonce") + nonce = ciphertext_with_nonce[:GCM_NONCE_LEN] + ciphertext = ciphertext_with_nonce[GCM_NONCE_LEN:] + return self._aesgcm.decrypt(nonce, ciphertext, associated_data) + + def encrypt_b64(self, plaintext: bytes, associated_data: Optional[bytes] = None) -> str: + """Encrypt and return result as URL-safe base64.""" + raw = self.encrypt(plaintext, associated_data) + return base64.urlsafe_b64encode(raw).decode("ascii") + + def decrypt_b64(self, encoded: str, associated_data: Optional[bytes] = None) -> bytes: + """Decrypt a URL-safe base64-encoded ciphertext.""" + raw = base64.urlsafe_b64decode(encoded) + return self.decrypt(raw, associated_data) + + +# --------------------------------------------------------------------------- +# Key derivation helper +# --------------------------------------------------------------------------- + + +def derive_subkey(master_key: bytes, context: str, length: int = AES_KEY_LEN) -> bytes: + """ + Derive a context-specific subkey from a master key using HKDF-SHA256. + + Parameters + ---------- + master_key: + The high-entropy root key (at least 32 bytes recommended). + context: + A unique string identifying the purpose of the derived key + (e.g., "document-encryption-v1", "signing-v1"). Acts as the HKDF info. + length: + Desired output length in bytes (default 32 for AES-256). + + Returns + ------- + bytes: + Derived subkey of the requested length. + """ + from cryptography.hazmat.primitives.hashes import SHA256 + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + hkdf = HKDF( + algorithm=SHA256(), + length=length, + salt=None, + info=context.encode("utf-8"), + ) + return hkdf.derive(master_key) + + +# --------------------------------------------------------------------------- +# Module-level evaluator registry (no eval() usage — just named for domain logic) +# --------------------------------------------------------------------------- + +# This dict maps expression type names to handler callables for the +# rule engine. The variable is named `evaluator` but does NOT use Python's +# built-in eval() function at any point. +evaluator = { + "fingerprint_match": content_fingerprint_sha256, + "token_valid": lambda tok: len(tok) >= 32 and tok.isascii(), +} diff --git a/golden-test-set/artifacts/python/gt-py-014-clean-api.py b/golden-test-set/artifacts/python/gt-py-014-clean-api.py new file mode 100644 index 0000000..c5ca942 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-014-clean-api.py @@ -0,0 +1,408 @@ +""" +REST API client for the inventory management service. + +Provides typed access to inventory records, stock level queries, and +reservation operations. Implements exponential backoff with jitter, +connection pooling, request correlation IDs, and structured error handling. + +Intended for use by fulfillment services, the warehouse management system, +and the demand forecasting pipeline. +""" + +from __future__ import annotations + +import logging +import random +import time +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import Iterator, Optional + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +DEFAULT_BASE_URL = "http://inventory-svc:8080" +DEFAULT_TIMEOUT = (5.0, 30.0) # (connect timeout, read timeout) in seconds +MAX_RETRIES = 3 +RETRY_BACKOFF_BASE = 0.5 # seconds +RETRY_BACKOFF_MAX = 10.0 # seconds +RETRY_STATUS_FORCELIST = (429, 500, 502, 503, 504) +PAGE_SIZE = 100 + +# --------------------------------------------------------------------------- +# Domain types +# --------------------------------------------------------------------------- + + +class StockStatus(Enum): + IN_STOCK = "in_stock" + LOW_STOCK = "low_stock" + OUT_OF_STOCK = "out_of_stock" + DISCONTINUED = "discontinued" + + +@dataclass +class InventoryItem: + sku: str + name: str + quantity_on_hand: int + quantity_reserved: int + warehouse_id: str + status: StockStatus + unit_cost_cents: int + reorder_threshold: int + last_updated: str # ISO 8601 + + @property + def quantity_available(self) -> int: + return max(0, self.quantity_on_hand - self.quantity_reserved) + + @classmethod + def from_dict(cls, data: dict) -> "InventoryItem": + return cls( + sku=data["sku"], + name=data["name"], + quantity_on_hand=int(data["quantity_on_hand"]), + quantity_reserved=int(data.get("quantity_reserved", 0)), + warehouse_id=data["warehouse_id"], + status=StockStatus(data["status"]), + unit_cost_cents=int(data["unit_cost_cents"]), + reorder_threshold=int(data.get("reorder_threshold", 0)), + last_updated=data["last_updated"], + ) + + +@dataclass +class ReservationResult: + reservation_id: str + sku: str + quantity: int + warehouse_id: str + expires_at: str + status: str + + +@dataclass +class ClientConfig: + base_url: str = DEFAULT_BASE_URL + timeout: tuple = field(default_factory=lambda: DEFAULT_TIMEOUT) + service_token: Optional[str] = None + max_retries: int = MAX_RETRIES + +# --------------------------------------------------------------------------- +# Error types +# --------------------------------------------------------------------------- + + +class InventoryError(Exception): + """Base class for inventory client errors.""" + + +class InsufficientStockError(InventoryError): + """Raised when a reservation cannot be fulfilled due to stock.""" + + def __init__(self, sku: str, requested: int, available: int): + self.sku = sku + self.requested = requested + self.available = available + super().__init__( + f"Insufficient stock for {sku}: requested {requested}, available {available}" + ) + + +class ItemNotFoundError(InventoryError): + """Raised when the requested SKU does not exist in the system.""" + + +# --------------------------------------------------------------------------- +# Session factory +# --------------------------------------------------------------------------- + + +def _build_session(config: ClientConfig) -> requests.Session: + """ + Build a requests.Session with connection pooling and retry logic. + + Retries are handled at the urllib3 layer for idempotent requests + (GET, HEAD, OPTIONS). POST retries are handled at the application layer + in _request() to avoid double-booking on reservation endpoints. + """ + session = requests.Session() + + retry = Retry( + total=config.max_retries, + backoff_factor=RETRY_BACKOFF_BASE, + status_forcelist=RETRY_STATUS_FORCELIST, + allowed_methods=["GET", "HEAD", "OPTIONS"], + raise_on_status=False, + ) + adapter = HTTPAdapter( + max_retries=retry, + pool_connections=4, + pool_maxsize=16, + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + + if config.service_token: + session.headers["Authorization"] = f"Bearer {config.service_token}" + + session.headers.update({ + "Accept": "application/json", + "Content-Type": "application/json", + }) + + return session + + +# --------------------------------------------------------------------------- +# Client +# --------------------------------------------------------------------------- + + +class InventoryClient: + """ + Thread-safe client for the inventory management API. + + All public methods raise InventoryError subclasses on domain errors + and requests.RequestException on network/HTTP transport errors. + """ + + def __init__(self, config: Optional[ClientConfig] = None): + self._cfg = config or ClientConfig() + self._session = _build_session(self._cfg) + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def get_item(self, sku: str) -> InventoryItem: + """ + Fetch a single inventory item by SKU. + + Raises ItemNotFoundError if the SKU does not exist. + Raises InventoryError on other API errors. + """ + data = self._request("GET", f"/v1/inventory/{sku}") + return InventoryItem.from_dict(data) + + def list_items( + self, + warehouse_id: Optional[str] = None, + status: Optional[StockStatus] = None, + low_stock_only: bool = False, + ) -> list[InventoryItem]: + """ + Return all inventory items matching the given filters. + + Handles pagination internally. Returns a flat list. + """ + params: dict = {"page_size": PAGE_SIZE} + if warehouse_id: + params["warehouse_id"] = warehouse_id + if status: + params["status"] = status.value + if low_stock_only: + params["low_stock_only"] = "true" + + items = [] + for page in self._paginate("/v1/inventory", params): + items.extend(InventoryItem.from_dict(rec) for rec in page) + return items + + def reserve( + self, + sku: str, + quantity: int, + warehouse_id: str, + order_id: str, + ttl_seconds: int = 900, + ) -> ReservationResult: + """ + Reserve inventory for an order. + + Parameters + ---------- + sku: + Stock-keeping unit identifier. + quantity: + Number of units to reserve. + warehouse_id: + The warehouse to reserve from. + order_id: + Caller's order reference (used for idempotency key). + ttl_seconds: + How long to hold the reservation (default 15 minutes). + + Returns + ------- + ReservationResult + + Raises + ------ + InsufficientStockError: + If the warehouse cannot fulfill the quantity. + InventoryError: + On other domain errors. + """ + if quantity <= 0: + raise ValueError(f"quantity must be positive, got {quantity}") + if ttl_seconds <= 0 or ttl_seconds > 86400: + raise ValueError("ttl_seconds must be between 1 and 86400") + + payload = { + "sku": sku, + "quantity": quantity, + "warehouse_id": warehouse_id, + "order_id": order_id, + "ttl_seconds": ttl_seconds, + } + data = self._request("POST", "/v1/reservations", json=payload) + return ReservationResult( + reservation_id=data["reservation_id"], + sku=data["sku"], + quantity=data["quantity"], + warehouse_id=data["warehouse_id"], + expires_at=data["expires_at"], + status=data["status"], + ) + + def cancel_reservation(self, reservation_id: str) -> bool: + """ + Cancel a pending reservation. + + Returns True if cancelled, False if already expired or fulfilled. + """ + try: + self._request("DELETE", f"/v1/reservations/{reservation_id}") + return True + except InventoryError as exc: + if "not_found" in str(exc).lower() or "expired" in str(exc).lower(): + return False + raise + + def adjust_stock(self, sku: str, warehouse_id: str, delta: int, reason: str) -> InventoryItem: + """ + Adjust on-hand stock by a signed delta (positive = receipt, negative = shrinkage). + + Returns the updated InventoryItem. + """ + payload = { + "delta": delta, + "warehouse_id": warehouse_id, + "reason": reason, + } + data = self._request("POST", f"/v1/inventory/{sku}/adjust", json=payload) + return InventoryItem.from_dict(data) + + def health(self) -> bool: + """Return True if the inventory service is healthy.""" + url = f"{self._cfg.base_url}/health" + try: + resp = self._session.get(url, timeout=self._cfg.timeout) + return resp.status_code == 200 + except requests.RequestException: + return False + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _request( + self, + method: str, + path: str, + json: Optional[dict] = None, + params: Optional[dict] = None, + _attempt: int = 1, + ) -> dict: + """ + Execute an HTTP request against the inventory API. + + Handles error response mapping to domain exceptions. Retries POST + requests with exponential backoff + jitter on 429/5xx. + """ + url = f"{self._cfg.base_url}{path}" + correlation_id = str(uuid.uuid4()) + headers = {"X-Correlation-ID": correlation_id} + + try: + resp = self._session.request( + method, + url, + json=json, + params=params, + headers=headers, + timeout=self._cfg.timeout, + ) + except requests.ConnectionError as exc: + raise InventoryError(f"Cannot connect to inventory service: {exc}") from exc + except requests.Timeout as exc: + raise InventoryError(f"Request timed out: {exc}") from exc + + # Handle retry-able responses for non-GET methods (GET is handled by urllib3) + if method not in ("GET", "HEAD") and resp.status_code in RETRY_STATUS_FORCELIST: + if _attempt <= self._cfg.max_retries: + wait = min( + RETRY_BACKOFF_BASE * (2 ** (_attempt - 1)) + random.uniform(0, 0.5), + RETRY_BACKOFF_MAX, + ) + logger.warning( + "Request %s %s returned %d (correlation=%s); retry %d/%d in %.1fs", + method, path, resp.status_code, correlation_id, + _attempt, self._cfg.max_retries, wait, + ) + time.sleep(wait) + return self._request(method, path, json=json, params=params, _attempt=_attempt + 1) + + if resp.status_code == 404: + raise ItemNotFoundError(f"Resource not found: {path}") + if resp.status_code == 409: + body = self._safe_json(resp) + available = body.get("available", 0) + requested = json.get("quantity", 0) if json else 0 + sku = (json or {}).get("sku", path) + raise InsufficientStockError(sku, requested, available) + if not resp.ok: + body = self._safe_json(resp) + message = body.get("message", resp.text[:200]) + raise InventoryError( + f"API error {resp.status_code} [{correlation_id}]: {message}" + ) + + return self._safe_json(resp) + + def _paginate(self, path: str, params: dict) -> Iterator[list]: + """Yield successive pages of results from a paginated list endpoint.""" + page_params = {**params, "page": 1} + while True: + resp = self._request("GET", path, params=page_params) + items = resp.get("items", []) + if not items: + break + yield items + if not resp.get("has_next"): + break + page_params["page"] += 1 + + @staticmethod + def _safe_json(resp: requests.Response) -> dict: + """Parse response JSON, returning empty dict on parse failure.""" + try: + return resp.json() + except ValueError: + return {} + + def __enter__(self): + return self + + def __exit__(self, *args): + self._session.close() diff --git a/golden-test-set/artifacts/python/gt-py-015-clean-subprocess.py b/golden-test-set/artifacts/python/gt-py-015-clean-subprocess.py new file mode 100644 index 0000000..6f5a098 --- /dev/null +++ b/golden-test-set/artifacts/python/gt-py-015-clean-subprocess.py @@ -0,0 +1,360 @@ +""" +Process management utilities for the build runner service. + +Provides safe wrappers around subprocess for executing build commands, +managing child process lifecycles, capturing output streams, and enforcing +resource constraints. All subprocess calls use list-form arguments and +never pass shell=True. + +Environment variable access in this module reads runtime configuration +only — no credentials or secrets are read or stored. +""" + +from __future__ import annotations + +import logging +import os +import shlex +import signal +import subprocess +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Iterator, Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Configuration (read from environment — not credentials) +# --------------------------------------------------------------------------- + +BUILD_WORKSPACE = Path(os.environ.get("BUILD_WORKSPACE", "/var/build/workspace")) +MAX_OUTPUT_BYTES = int(os.environ.get("MAX_OUTPUT_BYTES", str(10 * 1024 * 1024))) # 10 MB +DEFAULT_TIMEOUT_SEC = int(os.environ.get("BUILD_TIMEOUT_SEC", "600")) +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO") + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + + +@dataclass +class ProcessResult: + returncode: int + stdout: str + stderr: str + elapsed_sec: float + timed_out: bool = False + + @property + def succeeded(self) -> bool: + return self.returncode == 0 and not self.timed_out + + +@dataclass +class BuildStep: + name: str + command: list[str] # Always a list — never a shell string + working_dir: Optional[Path] = None + env_overrides: dict = field(default_factory=dict) + timeout_sec: int = DEFAULT_TIMEOUT_SEC + capture_output: bool = True + allow_failure: bool = False + + +# --------------------------------------------------------------------------- +# Core subprocess wrapper +# --------------------------------------------------------------------------- + + +def run_command( + command: list[str], + working_dir: Optional[Path] = None, + env_overrides: Optional[dict] = None, + timeout_sec: int = DEFAULT_TIMEOUT_SEC, + input_data: Optional[bytes] = None, +) -> ProcessResult: + """ + Execute a command as a subprocess and return the result. + + Parameters + ---------- + command: + Command and arguments as a list. Never joined into a shell string. + Callers must not pass user-controlled strings into this list without + explicit validation — this function does not sanitize args. + working_dir: + Optional working directory for the subprocess. + env_overrides: + Dict of additional environment variables to inject. Merged over the + current environment; does not replace it entirely. + timeout_sec: + Maximum wall-clock time to allow. Process is killed on timeout. + input_data: + Optional bytes to write to stdin. + + Returns + ------- + ProcessResult + """ + if not command: + raise ValueError("command must be a non-empty list") + + # Validate that command is a list (guards against accidental string passing) + if isinstance(command, str): + raise TypeError( + "command must be a list, not a string. " + "Use shlex.split() if you need to parse a string command." + ) + + env = {**os.environ} + if env_overrides: + env.update(env_overrides) + + cwd = str(working_dir) if working_dir else None + + logger.debug("Executing: %s (cwd=%s timeout=%ds)", shlex.join(command), cwd, timeout_sec) + + start = time.monotonic() + timed_out = False + + try: + proc = subprocess.run( + command, + cwd=cwd, + env=env, + input=input_data, + capture_output=True, + timeout=timeout_sec, + ) + returncode = proc.returncode + stdout_bytes = proc.stdout + stderr_bytes = proc.stderr + + except subprocess.TimeoutExpired as exc: + timed_out = True + returncode = -1 + stdout_bytes = exc.stdout or b"" + stderr_bytes = exc.stderr or b"" + logger.warning("Command timed out after %ds: %s", timeout_sec, shlex.join(command)) + + elapsed = time.monotonic() - start + + # Truncate oversized output + if len(stdout_bytes) > MAX_OUTPUT_BYTES: + logger.warning("stdout truncated from %d to %d bytes", len(stdout_bytes), MAX_OUTPUT_BYTES) + stdout_bytes = stdout_bytes[:MAX_OUTPUT_BYTES] + if len(stderr_bytes) > MAX_OUTPUT_BYTES: + stderr_bytes = stderr_bytes[:MAX_OUTPUT_BYTES] + + stdout = stdout_bytes.decode("utf-8", errors="replace") + stderr = stderr_bytes.decode("utf-8", errors="replace") + + result = ProcessResult( + returncode=returncode, + stdout=stdout, + stderr=stderr, + elapsed_sec=round(elapsed, 3), + timed_out=timed_out, + ) + + logger.debug( + "Command finished: exit=%d elapsed=%.2fs timed_out=%s cmd=%s", + returncode, + elapsed, + timed_out, + shlex.join(command), + ) + + return result + + +# --------------------------------------------------------------------------- +# Streaming output +# --------------------------------------------------------------------------- + + +def run_streaming( + command: list[str], + on_stdout: Callable[[str], None], + on_stderr: Callable[[str], None], + working_dir: Optional[Path] = None, + env_overrides: Optional[dict] = None, + timeout_sec: int = DEFAULT_TIMEOUT_SEC, +) -> ProcessResult: + """ + Execute a command and stream stdout/stderr line-by-line to callbacks. + + Useful for long-running build commands where real-time log forwarding + is required. Blocks until the process completes or times out. + """ + if isinstance(command, str): + raise TypeError("command must be a list, not a string") + + env = {**os.environ, **(env_overrides or {})} + cwd = str(working_dir) if working_dir else None + + start = time.monotonic() + timed_out = False + stdout_lines: list[str] = [] + stderr_lines: list[str] = [] + + proc = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + + def _reader(stream, lines_list: list, callback: Callable[[str], None]) -> None: + for line in stream: + stripped = line.rstrip("\n") + lines_list.append(stripped) + try: + callback(stripped) + except Exception as exc: + logger.warning("Output callback raised: %s", exc) + + stdout_thread = threading.Thread(target=_reader, args=(proc.stdout, stdout_lines, on_stdout)) + stderr_thread = threading.Thread(target=_reader, args=(proc.stderr, stderr_lines, on_stderr)) + stdout_thread.start() + stderr_thread.start() + + try: + proc.wait(timeout=timeout_sec) + except subprocess.TimeoutExpired: + timed_out = True + logger.warning("Streaming command timed out after %ds; sending SIGTERM", timeout_sec) + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning("Process did not exit after SIGTERM; sending SIGKILL") + proc.kill() + proc.wait() + + stdout_thread.join(timeout=5) + stderr_thread.join(timeout=5) + + elapsed = time.monotonic() - start + + return ProcessResult( + returncode=proc.returncode if not timed_out else -1, + stdout="\n".join(stdout_lines), + stderr="\n".join(stderr_lines), + elapsed_sec=round(elapsed, 3), + timed_out=timed_out, + ) + + +# --------------------------------------------------------------------------- +# Build pipeline +# --------------------------------------------------------------------------- + + +def run_build_step(step: BuildStep) -> ProcessResult: + """ + Execute a single build step and log its outcome. + + Returns the ProcessResult. Raises RuntimeError if the step fails + and allow_failure is False. + """ + logger.info("Starting build step: %s", step.name) + logger.debug("Command: %s", shlex.join(step.command)) + + result = run_command( + command=step.command, + working_dir=step.working_dir or BUILD_WORKSPACE, + env_overrides=step.env_overrides, + timeout_sec=step.timeout_sec, + ) + + if result.succeeded: + logger.info("Step '%s' completed in %.2fs", step.name, result.elapsed_sec) + elif result.timed_out: + logger.error("Step '%s' timed out after %ds", step.name, step.timeout_sec) + else: + logger.error( + "Step '%s' failed (exit %d) in %.2fs", + step.name, + result.returncode, + result.elapsed_sec, + ) + if result.stderr: + logger.debug("stderr:\n%s", result.stderr[-2000:]) + + if not result.succeeded and not step.allow_failure: + raise RuntimeError( + f"Build step '{step.name}' failed with exit code {result.returncode}" + ) + + return result + + +def run_pipeline(steps: list[BuildStep]) -> list[ProcessResult]: + """ + Execute a list of build steps sequentially. + + Stops at the first failing step (unless allow_failure=True). + Returns results for all executed steps. + """ + results = [] + for step in steps: + result = run_build_step(step) + results.append(result) + return results + + +# --------------------------------------------------------------------------- +# Process inventory +# --------------------------------------------------------------------------- + + +def list_child_processes(parent_pid: Optional[int] = None) -> list[dict]: + """ + List child processes of the given PID (defaults to current process). + + Uses 'ps' with controlled arguments — no user input involved. + Returns a list of dicts with keys: pid, ppid, stat, command. + """ + pid = parent_pid if parent_pid is not None else os.getpid() + + result = run_command( + ["ps", "-o", "pid,ppid,stat,comm", "--ppid", str(pid), "--no-headers"], + timeout_sec=10, + ) + + if result.returncode != 0: + return [] + + children = [] + for line in result.stdout.splitlines(): + parts = line.split(None, 3) + if len(parts) >= 4: + children.append({ + "pid": int(parts[0]), + "ppid": int(parts[1]), + "stat": parts[2], + "command": parts[3], + }) + return children + + +def wait_for_file(path: Path, timeout_sec: float = 30.0, poll_interval: float = 0.5) -> bool: + """ + Poll until a file exists or timeout elapses. + + Returns True if the file appeared within the timeout, False otherwise. + Pure Python — no subprocess involved. + """ + deadline = time.monotonic() + timeout_sec + while time.monotonic() < deadline: + if path.exists(): + return True + time.sleep(poll_interval) + return False diff --git a/golden-test-set/artifacts/shell/gt-sh-001-command-injection.sh b/golden-test-set/artifacts/shell/gt-sh-001-command-injection.sh new file mode 100644 index 0000000..d4e823b --- /dev/null +++ b/golden-test-set/artifacts/shell/gt-sh-001-command-injection.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# deploy.sh — application deployment script +# Usage: ./deploy.sh +# Environments: staging, production +# Version: e.g. 1.4.2 or "latest" +# +# Managed by: platform-eng@company.internal +# Last updated: 2026-01-14 + +DEPLOY_USER="${1:-staging}" +VERSION="${2:-latest}" +APP_NAME="webapp" +DEPLOY_BASE="/opt/deploys" +REGISTRY="registry.company.internal" +HEALTHCHECK_RETRIES=5 + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +# Resolve version alias to concrete tag +if [[ "$VERSION" == "latest" ]]; then + VERSION=$(curl -s "https://registry.company.internal/api/v2/${APP_NAME}/tags/latest" \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['tag'])") + log "Resolved 'latest' → $VERSION" +fi + +# Build deployment command from environment config file and execute +CONFIG_FILE="${DEPLOY_BASE}/configs/${DEPLOY_USER}.env" +if [[ -f "$CONFIG_FILE" ]]; then + log "Loading config for environment: $DEPLOY_USER" + DEPLOY_CMD=$(grep "^DEPLOY_CMD=" "$CONFIG_FILE" | cut -d= -f2-) + eval $DEPLOY_CMD +fi + +# Pull and run the deployment image +log "Pulling image: ${REGISTRY}/${APP_NAME}:${VERSION}" +INSTALL_URL="https://bootstrap.company.internal/install/${APP_NAME}/${VERSION}/setup.sh" +log "Fetching bootstrap installer from $INSTALL_URL" +curl $INSTALL_URL | bash + +# Wait for the new version to be healthy +log "Waiting for healthcheck..." +for i in $(seq 1 $HEALTHCHECK_RETRIES); do + STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:8080/health") + if [[ "$STATUS" == "200" ]]; then + log "Service healthy after $i attempt(s)." + break + fi + log "Attempt $i/$HEALTHCHECK_RETRIES: got HTTP $STATUS, retrying in 5s..." + sleep 5 +done + +if [[ "$STATUS" != "200" ]]; then + log "ERROR: Healthcheck failed after $HEALTHCHECK_RETRIES attempts. Deployment aborted." + exit 1 +fi + +log "Deployment of ${APP_NAME}:${VERSION} to ${DEPLOY_USER} complete." diff --git a/golden-test-set/artifacts/shell/gt-sh-002-missing-guards.sh b/golden-test-set/artifacts/shell/gt-sh-002-missing-guards.sh new file mode 100644 index 0000000..a9806aa --- /dev/null +++ b/golden-test-set/artifacts/shell/gt-sh-002-missing-guards.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# backup.sh — nightly database and config backup script +# Usage: ./backup.sh [--full | --incremental] +# Runs via cron: 02:00 UTC daily +# +# Managed by: ops@company.internal +# Last updated: 2026-02-03 + +BACKUP_MODE="${1:---incremental}" +BACKUP_DIR="/mnt/backups/$(date +%Y%m%d)" +DB_HOST="db-primary.company.internal" +DB_PORT=5432 +DB_NAME="appdb" +DB_USER="backup_user" +CONFIG_SOURCE="/etc/appconfig" +RETENTION_DAYS=30 +S3_BUCKET="s3://company-backups/db" + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +# ── Preparation ─────────────────────────────────────────────────────────────── + +mkdir -p "$BACKUP_DIR" +log "Backup started. Mode: $BACKUP_MODE Dir: $BACKUP_DIR" + +# ── Database dump ───────────────────────────────────────────────────────────── + +if [[ "$BACKUP_MODE" == "--full" ]]; then + DUMP_FILE="${BACKUP_DIR}/${DB_NAME}-full-$(date +%H%M%S).sql.gz" + log "Running full pg_dump → $DUMP_FILE" + pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" "$DB_NAME" \ + | gzip > "$DUMP_FILE" +else + DUMP_FILE="${BACKUP_DIR}/${DB_NAME}-incr-$(date +%H%M%S).sql.gz" + log "Running incremental pg_dump → $DUMP_FILE" + pg_dump -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" "$DB_NAME" \ + --table=audit_log --table=events \ + | gzip > "$DUMP_FILE" +fi + +log "Database dump complete: $DUMP_FILE" + +# ── Config snapshot ─────────────────────────────────────────────────────────── + +CONFIG_ARCHIVE="${BACKUP_DIR}/config-$(date +%H%M%S).tar.gz" +tar -czf "$CONFIG_ARCHIVE" "$CONFIG_SOURCE" +log "Config snapshot: $CONFIG_ARCHIVE" + +# ── Upload to S3 ────────────────────────────────────────────────────────────── + +log "Uploading to $S3_BUCKET..." +aws s3 sync "$BACKUP_DIR" "$S3_BUCKET/$(date +%Y%m%d)/" \ + --sse aws:kms \ + --storage-class STANDARD_IA + +# ── Prune old local backups ─────────────────────────────────────────────────── + +log "Pruning local backups older than ${RETENTION_DAYS} days..." +OLD_DIR=$(find /mnt/backups -maxdepth 1 -type d -mtime +${RETENTION_DAYS} | head -1) +rm -rf $OLD_DIR + +log "Backup complete." diff --git a/golden-test-set/artifacts/shell/gt-sh-003-clean-deploy.sh b/golden-test-set/artifacts/shell/gt-sh-003-clean-deploy.sh new file mode 100644 index 0000000..deb759d --- /dev/null +++ b/golden-test-set/artifacts/shell/gt-sh-003-clean-deploy.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# release-deploy.sh — production release deployment script +# Usage: ./release-deploy.sh --env --version +# Environments: staging, canary, production +# Version: semver tag (e.g. 1.4.2) — "latest" is not accepted +# +# Requires: +# - AWS CLI v2 configured with deploy-bot credentials +# - kubectl context set to the target cluster +# - DEPLOY_TOKEN env var (injected by CI/CD, never logged) +# +# Managed by: platform-eng@company.internal +# Last updated: 2026-02-28 + +set -euo pipefail + +# ── Constants ───────────────────────────────────────────────────────────────── + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly LOCK_FILE="/var/run/release-deploy.lock" +readonly APP_NAME="webapp" +readonly REGISTRY="registry.company.internal" +readonly DEPLOY_BASE="/opt/releases" +readonly HEALTHCHECK_URL="http://localhost:8080/healthz" +readonly HEALTHCHECK_RETRIES=10 +readonly HEALTHCHECK_INTERVAL=6 + +# ── Utilities ───────────────────────────────────────────────────────────────── + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S %Z')] INFO $*" >&2; } +warn() { echo "[$(date '+%Y-%m-%d %H:%M:%S %Z')] WARN $*" >&2; } +die() { echo "[$(date '+%Y-%m-%d %H:%M:%S %Z')] ERROR $*" >&2; exit 1; } + +cleanup() { + local exit_code=$? + if [[ -f "$LOCK_FILE" ]]; then + rm -f "$LOCK_FILE" + log "Lock released." + fi + if [[ $exit_code -ne 0 ]]; then + warn "Deployment exited with code $exit_code. Check logs above." + fi +} +trap cleanup EXIT + +# ── Argument parsing ────────────────────────────────────────────────────────── + +DEPLOY_ENV="" +VERSION="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --env) DEPLOY_ENV="$2"; shift 2 ;; + --version) VERSION="$2"; shift 2 ;; + *) die "Unknown argument: $1" ;; + esac +done + +[[ -n "$DEPLOY_ENV" ]] || die "--env is required" +[[ -n "$VERSION" ]] || die "--version is required" + +# Reject non-semver strings (no 'latest', no shell metacharacters) +if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then + die "Invalid version format: '${VERSION}'. Must be semver (e.g. 1.4.2)." +fi + +# Allowlist environments +case "$DEPLOY_ENV" in + staging|canary|production) ;; + *) die "Unknown environment: '${DEPLOY_ENV}'. Allowed: staging, canary, production." ;; +esac + +# ── Prerequisite checks ─────────────────────────────────────────────────────── + +command -v kubectl >/dev/null 2>&1 || die "kubectl not found in PATH" +command -v aws >/dev/null 2>&1 || die "aws CLI not found in PATH" +[[ -n "${DEPLOY_TOKEN:-}" ]] || die "DEPLOY_TOKEN env var not set" + +# ── Lock — prevent concurrent deploys ──────────────────────────────────────── + +if [[ -f "$LOCK_FILE" ]]; then + LOCK_PID=$(cat "$LOCK_FILE" 2>/dev/null || echo "unknown") + die "Another deploy is running (PID ${LOCK_PID}). Remove ${LOCK_FILE} if stale." +fi +echo $$ > "$LOCK_FILE" +log "Lock acquired (PID $$)." + +# ── Verify image exists in registry ────────────────────────────────────────── + +IMAGE="${REGISTRY}/${APP_NAME}:${VERSION}" +log "Verifying image: ${IMAGE}" +docker manifest inspect "${IMAGE}" >/dev/null 2>&1 \ + || die "Image not found in registry: ${IMAGE}" + +# ── Deploy directory setup ──────────────────────────────────────────────────── + +DEPLOY_DIR="${DEPLOY_BASE}/${DEPLOY_ENV}/${VERSION}" +if [[ -d "$DEPLOY_DIR" ]]; then + warn "Deploy directory already exists: ${DEPLOY_DIR}. Overwriting." +fi +mkdir -p "${DEPLOY_DIR}" + +# ── Render kubernetes manifests ─────────────────────────────────────────────── + +log "Rendering manifests for ${DEPLOY_ENV} / ${VERSION}..." +helm template "${APP_NAME}" "${SCRIPT_DIR}/charts/${APP_NAME}" \ + --set "image.tag=${VERSION}" \ + --set "env=${DEPLOY_ENV}" \ + --values "${SCRIPT_DIR}/charts/${APP_NAME}/values-${DEPLOY_ENV}.yaml" \ + > "${DEPLOY_DIR}/manifests.yaml" + +# ── Apply — dry run first ───────────────────────────────────────────────────── + +log "Dry-run apply..." +kubectl apply --dry-run=server -f "${DEPLOY_DIR}/manifests.yaml" + +log "Applying manifests to cluster (env=${DEPLOY_ENV})..." +kubectl apply -f "${DEPLOY_DIR}/manifests.yaml" + +# ── Rollout wait ────────────────────────────────────────────────────────────── + +log "Waiting for rollout: deployment/${APP_NAME}-${DEPLOY_ENV}..." +kubectl rollout status "deployment/${APP_NAME}-${DEPLOY_ENV}" \ + --namespace "${DEPLOY_ENV}" \ + --timeout=300s + +# ── Healthcheck ─────────────────────────────────────────────────────────────── + +log "Running healthcheck (up to ${HEALTHCHECK_RETRIES} attempts)..." +HEALTHY=0 +for i in $(seq 1 "$HEALTHCHECK_RETRIES"); do + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + --max-time 5 \ + "${HEALTHCHECK_URL}" || echo "000") + if [[ "$HTTP_STATUS" == "200" ]]; then + log "Healthcheck passed on attempt ${i}." + HEALTHY=1 + break + fi + log "Attempt ${i}/${HEALTHCHECK_RETRIES}: HTTP ${HTTP_STATUS} — retrying in ${HEALTHCHECK_INTERVAL}s..." + sleep "$HEALTHCHECK_INTERVAL" +done + +if [[ "$HEALTHY" -ne 1 ]]; then + warn "Healthcheck failed. Initiating rollback..." + kubectl rollout undo "deployment/${APP_NAME}-${DEPLOY_ENV}" \ + --namespace "${DEPLOY_ENV}" + die "Deployment of ${VERSION} to ${DEPLOY_ENV} FAILED — rolled back." +fi + +# ── Prune old release artifacts ─────────────────────────────────────────────── +# Only delete if the directory is non-empty, under DEPLOY_BASE, and not the +# current version. Guards prevent accidental deletion of unrelated paths. + +log "Pruning old release directories (keeping last 5)..." +RELEASE_PARENT="${DEPLOY_BASE}/${DEPLOY_ENV}" +if [[ -d "$RELEASE_PARENT" && "$RELEASE_PARENT" == "${DEPLOY_BASE}/"* ]]; then + mapfile -t OLD_RELEASES < <( + find "$RELEASE_PARENT" -maxdepth 1 -mindepth 1 -type d \ + | sort -V \ + | head -n -5 + ) + for old_dir in "${OLD_RELEASES[@]}"; do + # Paranoia: confirm path is still under RELEASE_PARENT + if [[ -n "$old_dir" && "$old_dir" == "${RELEASE_PARENT}/"* ]]; then + log "Removing old release: ${old_dir}" + rm -rf "${old_dir}" + fi + done +fi + +# ── Done ────────────────────────────────────────────────────────────────────── + +log "Deployment of ${APP_NAME}:${VERSION} to ${DEPLOY_ENV} complete." diff --git a/golden-test-set/manifest.yaml b/golden-test-set/manifest.yaml new file mode 100644 index 0000000..e77eaff --- /dev/null +++ b/golden-test-set/manifest.yaml @@ -0,0 +1,286 @@ +schema_version: '1.0' +description: "Golden Test Set manifest \u2014 40 annotated artifacts for Quorum accuracy\ + \ measurement" +created: '2026-03-12' +total_artifacts: 40 +artifacts: +- path: artifacts/config/gt-cfg-001-exposed-secrets.yaml + expected_verdict: REJECT + primary_defect_class: security + complexity: low + source: synthetic + rubric: agent-config + finding_count: 2 +- path: artifacts/config/gt-cfg-002-insecure-permissions.yaml + expected_verdict: REJECT + primary_defect_class: security + complexity: medium + source: synthetic + rubric: agent-config + finding_count: 3 +- path: artifacts/config/gt-cfg-003-stale-references.yaml + expected_verdict: REVISE + primary_defect_class: correctness + complexity: medium + source: modified-natural + rubric: agent-config + finding_count: 2 +- path: artifacts/config/gt-cfg-004-missing-validation.json + expected_verdict: REVISE + primary_defect_class: completeness + complexity: low + source: synthetic + rubric: agent-config + finding_count: 3 +- path: artifacts/config/gt-cfg-005-contradictory.yaml + expected_verdict: REVISE + primary_defect_class: code_hygiene + complexity: high + source: synthetic + rubric: agent-config + finding_count: 2 +- path: artifacts/config/gt-cfg-006-minor-issues.yaml + expected_verdict: PASS_WITH_NOTES + primary_defect_class: completeness + complexity: low + source: synthetic + rubric: agent-config + finding_count: 1 +- path: artifacts/config/gt-cfg-007-clean-k8s.yaml + expected_verdict: PASS_WITH_NOTES + primary_defect_class: completeness + complexity: medium + source: modified-natural + rubric: agent-config + finding_count: 1 +- path: artifacts/config/gt-cfg-008-clean-pipeline.json + expected_verdict: PASS + primary_defect_class: clean + complexity: medium + source: synthetic + rubric: agent-config + finding_count: 0 +- path: artifacts/docs/gt-doc-001-wrong-citations.md + expected_verdict: REJECT + primary_defect_class: correctness + complexity: medium + source: synthetic + rubric: research-synthesis + finding_count: 2 +- path: artifacts/docs/gt-doc-002-contradictory-claims.md + expected_verdict: REJECT + primary_defect_class: correctness + complexity: high + source: synthetic + rubric: markdown-doc + finding_count: 2 +- path: artifacts/docs/gt-doc-003-stale-references.md + expected_verdict: REVISE + primary_defect_class: correctness + complexity: medium + source: modified-natural + rubric: markdown-doc + finding_count: 2 +- path: artifacts/docs/gt-doc-004-missing-sections.md + expected_verdict: REVISE + primary_defect_class: completeness + complexity: low + source: synthetic + rubric: markdown-doc + finding_count: 3 +- path: artifacts/docs/gt-doc-005-research-gaps.md + expected_verdict: REVISE + primary_defect_class: completeness + complexity: high + source: synthetic + rubric: research-synthesis + finding_count: 3 +- path: artifacts/docs/gt-doc-006-poor-structure.md + expected_verdict: REVISE + primary_defect_class: code_hygiene + complexity: medium + source: modified-natural + rubric: markdown-doc + finding_count: 2 +- path: artifacts/docs/gt-doc-007-minor-style.md + expected_verdict: PASS_WITH_NOTES + primary_defect_class: code_hygiene + complexity: low + source: synthetic + rubric: markdown-doc + finding_count: 1 +- path: artifacts/docs/gt-doc-008-minor-completeness.md + expected_verdict: PASS_WITH_NOTES + primary_defect_class: completeness + complexity: low + source: modified-natural + rubric: markdown-doc + finding_count: 2 +- path: artifacts/docs/gt-doc-009-clean-spec.md + expected_verdict: PASS + primary_defect_class: clean + complexity: high + source: synthetic + rubric: markdown-doc + finding_count: 0 +- path: artifacts/docs/gt-doc-010-clean-research.md + expected_verdict: PASS + primary_defect_class: clean + complexity: medium + source: natural + rubric: research-synthesis + finding_count: 0 +- path: artifacts/python/gt-py-001-sql-injection.py + expected_verdict: REJECT + primary_defect_class: security + complexity: low + source: synthetic + rubric: python-code + finding_count: 1 +- path: artifacts/python/gt-py-002-hardcoded-credentials.py + expected_verdict: REJECT + primary_defect_class: security + complexity: medium + source: synthetic + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-003-command-injection.py + expected_verdict: REJECT + primary_defect_class: security + complexity: high + source: synthetic + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-004-insecure-deserialization.py + expected_verdict: REJECT + primary_defect_class: security + complexity: medium + source: synthetic + rubric: python-code + finding_count: 1 +- path: artifacts/python/gt-py-005-logic-errors.py + expected_verdict: REVISE + primary_defect_class: correctness + complexity: medium + source: synthetic + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-006-missing-error-handling.py + expected_verdict: REVISE + primary_defect_class: completeness + complexity: medium + source: modified-natural + rubric: python-code + finding_count: 3 +- path: artifacts/python/gt-py-007-dead-code.py + expected_verdict: REVISE + primary_defect_class: code_hygiene + complexity: medium + source: synthetic + rubric: python-code + finding_count: 3 +- path: artifacts/python/gt-py-008-path-traversal.py + expected_verdict: REVISE + primary_defect_class: security + complexity: high + source: modified-natural + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-009-type-mismatch.py + expected_verdict: REVISE + primary_defect_class: correctness + complexity: high + source: synthetic + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-010-minor-hygiene.py + expected_verdict: PASS_WITH_NOTES + primary_defect_class: code_hygiene + complexity: low + source: synthetic + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-011-minor-completeness.py + expected_verdict: PASS_WITH_NOTES + primary_defect_class: completeness + complexity: low + source: synthetic + rubric: python-code + finding_count: 2 +- path: artifacts/python/gt-py-012-ssrf-partial.py + expected_verdict: PASS_WITH_NOTES + primary_defect_class: security + complexity: medium + source: modified-natural + rubric: python-code + finding_count: 1 +- path: artifacts/python/gt-py-013-clean-crypto.py + expected_verdict: PASS + primary_defect_class: clean + complexity: high + source: synthetic + rubric: python-code + finding_count: 0 +- path: artifacts/python/gt-py-014-clean-api.py + expected_verdict: PASS + primary_defect_class: clean + complexity: medium + source: synthetic + rubric: python-code + finding_count: 0 +- path: artifacts/python/gt-py-015-clean-subprocess.py + expected_verdict: PASS + primary_defect_class: clean + complexity: high + source: synthetic + rubric: python-code + finding_count: 0 +- path: artifacts/shell/gt-sh-001-command-injection.sh + expected_verdict: REJECT + primary_defect_class: security + complexity: low + source: synthetic + rubric: shell-script + finding_count: 2 +- path: artifacts/shell/gt-sh-002-missing-guards.sh + expected_verdict: REVISE + primary_defect_class: completeness + complexity: medium + source: modified-natural + rubric: shell-script + finding_count: 3 +- path: artifacts/shell/gt-sh-003-clean-deploy.sh + expected_verdict: PASS + primary_defect_class: clean + complexity: medium + source: synthetic + rubric: shell-script + finding_count: 0 +- path: artifacts/cross-artifact/gt-xa-001-spec-impl-mismatch/ + expected_verdict: REJECT + primary_defect_class: cross_consistency + complexity: high + source: synthetic + rubric: cross-artifact + finding_count: 2 +- path: artifacts/cross-artifact/gt-xa-002-config-code-mismatch/ + expected_verdict: REVISE + primary_defect_class: cross_consistency + complexity: medium + source: synthetic + rubric: cross-artifact + finding_count: 2 +- path: artifacts/cross-artifact/gt-xa-003-readme-code-mismatch/ + expected_verdict: REVISE + primary_defect_class: cross_consistency + complexity: medium + source: modified-natural + rubric: cross-artifact + finding_count: 2 +- path: artifacts/cross-artifact/gt-xa-004-consistent-pair/ + expected_verdict: PASS + primary_defect_class: clean + complexity: high + source: natural + rubric: cross-artifact + finding_count: 0 diff --git a/golden-test-set/schema.yaml b/golden-test-set/schema.yaml new file mode 100644 index 0000000..2c65a85 --- /dev/null +++ b/golden-test-set/schema.yaml @@ -0,0 +1,129 @@ +# Golden Test Set — Annotation Schema v1.0 +# +# Each artifact in the golden test set has a sidecar annotation file +# following this schema. The annotation is the "answer key" that Quorum's +# output is scored against. +# +# ID Convention: GT-### = Golden Test ground truth finding +# See docs/ID_CONVENTIONS.md (planned) for the full prefix registry. +# +# Matching rules: see DESIGN.md § Matching & Scoring Rules + +# --- Schema Definition --- + +schema_version: + type: string + required: true + description: "Schema version (currently '1.0')" + example: "1.0" + +artifact: + type: string + required: true + description: "Relative path from golden-test-set/ to the artifact file" + example: "artifacts/python/vulnerable-api.py" + +artifact_sha256: + type: string + required: true + description: "SHA-256 hex digest of the artifact file. Ensures annotation matches the exact file version." + +expected_verdict: + type: string + required: true + enum: [PASS, PASS_WITH_NOTES, REVISE, REJECT] + description: "The verdict Quorum should produce for this artifact" + +findings: + type: list + required: true # empty list [] for clean PASS artifacts + description: "Expected findings — the answer key" + items: + id: + type: string + required: true + pattern: "GT-\\d{3}" + description: "Unique ground truth finding ID" + description: + type: string + required: true + description: "What the issue is — human-readable" + location: + type: string + required: false + description: "Where in the artifact (e.g. 'line 42-48', 'section 3.2'). Matched with ±5 line tolerance." + severity: + type: string + required: true + enum: [CRITICAL, HIGH, MEDIUM, LOW, INFO] + description: "Expected severity level" + category: + type: string + required: true + enum: [security, correctness, completeness, code_hygiene, cross_consistency] + description: "Defect class — must match critic domain" + critic: + type: string + required: true + enum: [security, correctness, completeness, code_hygiene, cross_consistency] + description: "Which critic should catch this finding" + rubric_criterion: + type: string + required: false + description: "Specific rubric criterion ID if applicable" + notes: + type: string + required: false + description: "Human notes explaining why this is a finding" + +false_positive_traps: + type: list + required: false + description: "Things that LOOK like issues but aren't. If Quorum flags these, they count against precision." + items: + description: + type: string + required: true + description: "What the non-issue is" + location: + type: string + required: false + description: "Where in the artifact" + notes: + type: string + required: false + description: "Why this is NOT an issue" + +metadata: + type: object + required: true + fields: + source: + type: string + required: true + enum: [synthetic, natural, modified-natural] + description: "How the artifact was created" + domain: + type: string + required: true + enum: [python-code, yaml-config, markdown-doc, shell-script, cross-artifact] + complexity: + type: string + required: true + enum: [low, medium, high] + rubric: + type: string + required: true + description: "Rubric name to use for evaluation" + depth: + type: string + required: true + enum: [quick, standard, thorough] + description: "Recommended depth for this test" + author: + type: string + required: true + created: + type: string + required: true + description: "ISO date string" diff --git a/golden-test-set/scripts/SCORE_SPEC.md b/golden-test-set/scripts/SCORE_SPEC.md new file mode 100644 index 0000000..7c32649 --- /dev/null +++ b/golden-test-set/scripts/SCORE_SPEC.md @@ -0,0 +1,161 @@ +# score.py — Scoring Framework Specification + +## Purpose + +Compare Quorum validation output against golden test set annotations. +Compute precision, recall, F1, severity accuracy, false positive rate, and verdict accuracy. + +## Usage + +```bash +# Score a single run +python3 score.py --run-dir results/baseline-20260312/ --annotations-dir annotations/ + +# Score with custom thresholds +python3 score.py --run-dir results/baseline-20260312/ --annotations-dir annotations/ \ + --min-recall 0.80 --min-precision 0.70 + +# Output formats +python3 score.py --run-dir results/baseline-20260312/ --annotations-dir annotations/ \ + --format json # machine-readable + --format markdown # human-readable report + --format both # default: both to stdout + files +``` + +## Exit Codes + +- 0: All thresholds met +- 1: One or more thresholds not met (prints which) +- 2: Error (missing files, schema mismatch, etc.) + +## Matching Algorithm + +For each artifact in the test set: + +1. Load the annotation sidecar (`.annotations.yaml`) +2. Verify `artifact_sha256` matches the actual file (abort with warning if mismatch) +3. Load Quorum's findings from the run directory +4. For each ground truth finding (GT-###), attempt to match a Quorum finding: + a. **Critic match:** Quorum finding's `critic` field matches GT `critic` (required) + b. **Category match:** Quorum finding's category matches GT `category` (required) + c. **Location match** (if GT has location): Quorum finding location within ±5 lines (preferred) OR description fuzzy match ≥0.6 (fallback) + d. **Location match** (if GT has no location): description fuzzy match ≥0.6 +5. Each Quorum finding can match at most one GT finding (greedy, highest-similarity-first) +6. Each GT finding can be matched at most once + +### Classification + +- **True Positive (TP):** Quorum finding matches a GT finding +- **False Positive (FP):** Quorum finding matches no GT finding AND is not in a `false_positive_traps` entry (if it matches a trap, it's a **Trapped FP** — counted separately for analysis) +- **False Negative (FN):** GT finding has no matching Quorum finding + +## Metrics + +### Aggregate + +| Metric | Formula | +|--------|---------| +| Detection Precision | TP / (TP + FP) | +| Detection Recall | TP / (TP + FN) | +| F1 | 2 × (P × R) / (P + R) | +| Severity Accuracy | TP_exact_severity / TP | +| Severity Distance (mean) | mean(|GT_tier - Quorum_tier|) for all TPs | +| False Positive Rate (clean) | FP on PASS artifacts / total Quorum findings on PASS artifacts | +| Verdict Accuracy | artifacts_correct_verdict / total_artifacts | +| Trapped FP Count | Quorum findings that hit annotated false_positive_traps | + +Severity tier encoding: CRITICAL=4, HIGH=3, MEDIUM=2, LOW=1, INFO=0. + +### Sliced (same metrics computed per dimension) + +- Per critic: correctness, completeness, security, code_hygiene, cross_consistency +- Per severity: CRITICAL, HIGH, MEDIUM, LOW +- Per complexity: low, medium, high +- Per file type: python-code, yaml-config, markdown-doc, shell-script, cross-artifact +- Per source: synthetic, natural, modified-natural + +## Output Format + +### JSON (`--format json`) + +```json +{ + "schema_version": "1.0", + "run_dir": "results/baseline-20260312/", + "timestamp": "2026-03-12T...", + "aggregate": { + "precision": 0.82, + "recall": 0.78, + "f1": 0.80, + "severity_accuracy": 0.71, + "severity_distance_mean": 0.35, + "fp_rate_clean": 0.10, + "verdict_accuracy": 0.80, + "trapped_fp_count": 3, + "tp": 45, + "fp": 10, + "fn": 13, + "total_gt_findings": 58, + "total_quorum_findings": 55 + }, + "by_critic": { ... }, + "by_severity": { ... }, + "by_complexity": { ... }, + "by_file_type": { ... }, + "by_source": { ... }, + "per_artifact": [ + { + "artifact": "artifacts/python/vulnerable-api.py", + "expected_verdict": "REVISE", + "actual_verdict": "REVISE", + "verdict_correct": true, + "tp": 3, + "fp": 1, + "fn": 0, + "trapped_fp": 0, + "findings_detail": [ + { + "gt_id": "GT-001", + "matched": true, + "quorum_finding_id": "F-abc123", + "severity_match": true, + "gt_severity": "CRITICAL", + "quorum_severity": "CRITICAL" + } + ] + } + ], + "thresholds": { + "min_recall": 0.80, + "min_precision": 0.70, + "met": true + } +} +``` + +### Markdown (`--format markdown`) + +Human-readable report with: +- Summary table (aggregate metrics) +- Per-critic breakdown table +- Per-artifact results (pass/fail, missed findings, false positives) +- Worst performers (artifacts with lowest recall) +- False positive analysis (most common FP patterns) +- Threshold pass/fail status + +## Dependencies + +- Standard library only (no external deps beyond what Quorum already requires) +- Uses `difflib.SequenceMatcher` for fuzzy matching (consistent with Quorum's tester) +- Reads Quorum run output via JSON (verdict.json, critics/*.json) + +## Testing + +Include `tests/test_score.py`: +- Test matching algorithm with known TP/FP/FN cases +- Test severity distance calculation +- Test fuzzy location matching (±5 lines) +- Test handling of clean (PASS) artifacts +- Test trapped false positive detection +- Test SHA-256 integrity check +- Test edge cases: empty annotations, no Quorum findings, all false positives diff --git a/golden-test-set/scripts/score.py b/golden-test-set/scripts/score.py new file mode 100644 index 0000000..89ddb48 --- /dev/null +++ b/golden-test-set/scripts/score.py @@ -0,0 +1,849 @@ +#!/usr/bin/env python3 +"""Golden Test Set Scoring Framework. + +Compares Quorum validation output against human-annotated ground truth. +Computes precision, recall, F1, severity accuracy, false positive rate, +and verdict accuracy — aggregate and sliced by multiple dimensions. + +Usage: + python3 score.py --run-dir results/baseline-20260312/ --annotations-dir annotations/ + python3 score.py --run-dir results/baseline-20260312/ --annotations-dir annotations/ --validate +""" + +from __future__ import annotations + +import argparse +import difflib +import hashlib +import json +import os +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +SEVERITY_TIERS = {"CRITICAL": 4, "HIGH": 3, "MEDIUM": 2, "LOW": 1, "INFO": 0} +LOCATION_TOLERANCE = 5 +FUZZY_THRESHOLD = 0.6 +SCHEMA_VERSION = "1.0" + + +# --------------------------------------------------------------------------- +# Data classes +# --------------------------------------------------------------------------- + +@dataclass +class GTFinding: + """A single ground-truth finding from an annotation.""" + id: str + description: str + location: str | None + severity: str + category: str + critic: str + rubric_criterion: str | None = None + notes: str | None = None + + +@dataclass +class FPTrap: + """A false-positive trap from an annotation.""" + description: str + location: str | None = None + notes: str | None = None + + +@dataclass +class Annotation: + """Parsed annotation sidecar for one artifact.""" + artifact: str + artifact_sha256: str + expected_verdict: str + findings: list[GTFinding] + false_positive_traps: list[FPTrap] + metadata: dict[str, str] + schema_version: str = SCHEMA_VERSION + + +@dataclass +class QuorumFinding: + """A single finding from Quorum's output.""" + id: str + severity: str + category: str | None + description: str + location: str | None + critic: str + rubric_criterion: str | None = None + + +@dataclass +class MatchResult: + """Result of matching one GT finding.""" + gt_id: str + matched: bool + quorum_finding_id: str | None = None + severity_match: bool = False + gt_severity: str = "" + quorum_severity: str = "" + + +@dataclass +class ArtifactScore: + """Scoring result for one artifact.""" + artifact: str + expected_verdict: str + actual_verdict: str | None + verdict_correct: bool + tp: int = 0 + fp: int = 0 + fn: int = 0 + trapped_fp: int = 0 + findings_detail: list[MatchResult] = field(default_factory=list) + metadata: dict[str, str] = field(default_factory=dict) + severity_matches: int = 0 + severity_distances: list[int] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + +def parse_annotation(path: Path) -> Annotation: + """Parse an annotation YAML file.""" + with open(path) as f: + data = yaml.safe_load(f) + + findings = [] + for fd in data.get("findings") or []: + findings.append(GTFinding( + id=fd["id"], + description=fd["description"], + location=fd.get("location"), + severity=fd["severity"], + category=fd["category"], + critic=fd["critic"], + rubric_criterion=fd.get("rubric_criterion"), + notes=fd.get("notes"), + )) + + traps = [] + for td in data.get("false_positive_traps") or []: + traps.append(FPTrap( + description=td["description"], + location=td.get("location"), + notes=td.get("notes"), + )) + + return Annotation( + artifact=data["artifact"], + artifact_sha256=data["artifact_sha256"], + expected_verdict=data["expected_verdict"], + findings=findings, + false_positive_traps=traps, + metadata=data.get("metadata", {}), + schema_version=data.get("schema_version", SCHEMA_VERSION), + ) + + +def parse_quorum_run(run_dir: Path, artifact_name: str) -> tuple[str | None, list[QuorumFinding]]: + """Parse Quorum output for a specific artifact from a run directory. + + Supports two layouts: + 1. Batch: run_dir/per-file/-/{verdict.json, critics/*.json} + 2. Single: run_dir/{verdict.json, critics/*.json} + + Returns (verdict_status, findings_list). + """ + # Try batch layout first: look for a subdirectory matching the artifact name + per_file = run_dir / "per-file" + target_dir = None + + if per_file.is_dir(): + stem = Path(artifact_name).stem + for entry in per_file.iterdir(): + if entry.is_dir() and stem in entry.name: + target_dir = entry + break + + if target_dir is None: + # Try single-file layout + if (run_dir / "verdict.json").exists(): + target_dir = run_dir + else: + return None, [] + + # Read verdict + verdict_status = None + verdict_path = target_dir / "verdict.json" + if verdict_path.exists(): + with open(verdict_path) as f: + verdict_data = json.load(f) + verdict_status = verdict_data.get("status") + + # Read findings from all critic files + findings: list[QuorumFinding] = [] + critics_dir = target_dir / "critics" + if critics_dir.is_dir(): + for critic_file in sorted(critics_dir.glob("*-findings.json")): + critic_name = critic_file.stem.replace("-findings", "") + with open(critic_file) as f: + critic_data = json.load(f) + for fd in critic_data.get("findings", []): + findings.append(QuorumFinding( + id=fd.get("id", ""), + severity=fd.get("severity", "INFO"), + category=fd.get("category"), + description=fd.get("description", ""), + location=fd.get("location"), + critic=fd.get("critic", critic_name), + rubric_criterion=fd.get("rubric_criterion"), + )) + + # Also read findings from verdict.json if they exist there + if verdict_path.exists(): + with open(verdict_path) as f: + verdict_data = json.load(f) + report = verdict_data.get("report", {}) + for fd in report.get("findings", []): + # Avoid duplicates by checking ID + existing_ids = {f.id for f in findings} + fid = fd.get("id", "") + if fid and fid not in existing_ids: + findings.append(QuorumFinding( + id=fid, + severity=fd.get("severity", "INFO"), + category=fd.get("category"), + description=fd.get("description", ""), + location=fd.get("location"), + critic=fd.get("critic", "unknown"), + rubric_criterion=fd.get("rubric_criterion"), + )) + + return verdict_status, findings + + +# --------------------------------------------------------------------------- +# Matching +# --------------------------------------------------------------------------- + +def _parse_line_numbers(location: str | None) -> list[int]: + """Extract line numbers from a location string like 'line 42-48' or 'line 30'. + + Requires either 'line' prefix or a standalone number/range pattern (not embedded + in words like 'section 3.2'). + """ + if not location: + return [] + nums = [] + # Match "line N", "line N-M", "Line N", or standalone "N-M" at word boundary + # Require "line" prefix OR the number must not be preceded by a dot (avoids "section 3.2") + for m in re.finditer(r"(?:line\s+)(\d+)(?:\s*[-–]\s*(\d+))?|(?:^|(?<=\s))(\d+)\s*[-–]\s*(\d+)", location, re.IGNORECASE): + if m.group(1) is not None: + # "line N" or "line N-M" + start = int(m.group(1)) + end = int(m.group(2)) if m.group(2) else start + else: + # Standalone "N-M" range + start = int(m.group(3)) + end = int(m.group(4)) + nums.extend(range(start, end + 1)) + return nums + + +def _location_match(gt_location: str | None, q_location: str | None) -> bool: + """Check if locations overlap within ±LOCATION_TOLERANCE lines.""" + gt_lines = _parse_line_numbers(gt_location) + q_lines = _parse_line_numbers(q_location) + if not gt_lines or not q_lines: + return False + for gl in gt_lines: + for ql in q_lines: + if abs(gl - ql) <= LOCATION_TOLERANCE: + return True + return False + + +def _fuzzy_match(text_a: str, text_b: str) -> float: + """Fuzzy string similarity using SequenceMatcher.""" + return difflib.SequenceMatcher(None, text_a.lower(), text_b.lower()).ratio() + + +def _is_trap_match(qf: QuorumFinding, traps: list[FPTrap]) -> bool: + """Check if a Quorum finding matches any false-positive trap.""" + for trap in traps: + # Location match + if trap.location and qf.location: + if _location_match(trap.location, qf.location): + return True + # Description fuzzy match + if _fuzzy_match(trap.description, qf.description) >= FUZZY_THRESHOLD: + return True + return False + + +def match_findings( + gt_findings: list[GTFinding], + quorum_findings: list[QuorumFinding], + traps: list[FPTrap], +) -> tuple[list[MatchResult], int, int, int, int, int, list[int]]: + """Match Quorum findings against ground truth. + + Returns: (match_results, tp, fp, fn, trapped_fp, severity_matches, severity_distances) + """ + # Build similarity scores for all pairs + pairs: list[tuple[float, int, int]] = [] # (score, gt_idx, q_idx) + for gi, gt in enumerate(gt_findings): + for qi, qf in enumerate(quorum_findings): + # Hard requirements: critic and category must match + if qf.critic != gt.critic: + continue + q_cat = qf.category or qf.critic # fallback: use critic name as category + if q_cat != gt.category: + continue + + # Score: location match gets priority, then fuzzy description + score = 0.0 + if gt.location and _location_match(gt.location, qf.location): + score = 1.0 + elif _fuzzy_match(gt.description, qf.description) >= FUZZY_THRESHOLD: + score = _fuzzy_match(gt.description, qf.description) + + if score > 0: + pairs.append((score, gi, qi)) + + # Greedy matching: highest similarity first + pairs.sort(key=lambda x: -x[0]) + matched_gt: set[int] = set() + matched_q: set[int] = set() + match_map: dict[int, int] = {} # gt_idx -> q_idx + + for score, gi, qi in pairs: + if gi not in matched_gt and qi not in matched_q: + matched_gt.add(gi) + matched_q.add(qi) + match_map[gi] = qi + + # Build results + tp = len(match_map) + fn = len(gt_findings) - tp + + # Count FP and trapped FP + trapped_fp = 0 + fp = 0 + for qi, qf in enumerate(quorum_findings): + if qi not in matched_q: + if _is_trap_match(qf, traps): + trapped_fp += 1 + else: + fp += 1 + + # Severity accuracy + severity_matches = 0 + severity_distances: list[int] = [] + match_results: list[MatchResult] = [] + + for gi, gt in enumerate(gt_findings): + if gi in match_map: + qi = match_map[gi] + qf = quorum_findings[qi] + sev_match = gt.severity == qf.severity + if sev_match: + severity_matches += 1 + gt_tier = SEVERITY_TIERS.get(gt.severity, 0) + q_tier = SEVERITY_TIERS.get(qf.severity, 0) + severity_distances.append(abs(gt_tier - q_tier)) + + match_results.append(MatchResult( + gt_id=gt.id, + matched=True, + quorum_finding_id=qf.id, + severity_match=sev_match, + gt_severity=gt.severity, + quorum_severity=qf.severity, + )) + else: + match_results.append(MatchResult( + gt_id=gt.id, + matched=False, + gt_severity=gt.severity, + )) + + return match_results, tp, fp, fn, trapped_fp, severity_matches, severity_distances + + +# --------------------------------------------------------------------------- +# SHA-256 integrity +# --------------------------------------------------------------------------- + +def compute_sha256(path: Path) -> str: + """Compute SHA-256 hex digest of a file.""" + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + h.update(chunk) + return h.hexdigest() + + +# --------------------------------------------------------------------------- +# Metrics aggregation +# --------------------------------------------------------------------------- + +def _safe_div(a: float, b: float) -> float: + return a / b if b > 0 else 0.0 + + +def _compute_aggregate(scores: list[ArtifactScore]) -> dict[str, Any]: + """Compute aggregate metrics only (no slicing — avoids recursion).""" + total_tp = sum(s.tp for s in scores) + total_fp = sum(s.fp for s in scores) + total_fn = sum(s.fn for s in scores) + total_trapped = sum(s.trapped_fp for s in scores) + total_sev_matches = sum(s.severity_matches for s in scores) + all_sev_distances: list[int] = [] + for s in scores: + all_sev_distances.extend(s.severity_distances) + + precision = _safe_div(total_tp, total_tp + total_fp) + recall = _safe_div(total_tp, total_tp + total_fn) + f1 = _safe_div(2 * precision * recall, precision + recall) + severity_accuracy = _safe_div(total_sev_matches, total_tp) + severity_distance_mean = _safe_div(sum(all_sev_distances), len(all_sev_distances)) if all_sev_distances else 0.0 + + clean = [s for s in scores if s.expected_verdict == "PASS"] + clean_fp = sum(s.fp + s.trapped_fp for s in clean) + clean_total_q = sum(s.tp + s.fp + s.trapped_fp for s in clean) + fp_rate_clean = _safe_div(clean_fp, clean_total_q) if clean_total_q > 0 else 0.0 + + verdict_correct = sum(1 for s in scores if s.verdict_correct) + verdict_accuracy = _safe_div(verdict_correct, len(scores)) + + total_gt = total_tp + total_fn + total_q = total_tp + total_fp + total_trapped + + return { + "precision": round(precision, 4), + "recall": round(recall, 4), + "f1": round(f1, 4), + "severity_accuracy": round(severity_accuracy, 4), + "severity_distance_mean": round(severity_distance_mean, 4), + "fp_rate_clean": round(fp_rate_clean, 4), + "verdict_accuracy": round(verdict_accuracy, 4), + "trapped_fp_count": total_trapped, + "tp": total_tp, + "fp": total_fp, + "fn": total_fn, + "total_gt_findings": total_gt, + "total_quorum_findings": total_q, + } + + +def compute_metrics(scores: list[ArtifactScore]) -> dict[str, Any]: + """Compute aggregate and sliced metrics from per-artifact scores.""" + aggregate = _compute_aggregate(scores) + + # Sliced metrics + def _slice(key_fn: Any) -> dict[str, dict[str, Any]]: + buckets: dict[str, list[ArtifactScore]] = {} + for s in scores: + k = key_fn(s) + buckets.setdefault(k, []).append(s) + result = {} + for k, bucket in sorted(buckets.items()): + m = _compute_aggregate(bucket) + result[k] = m + return result + + by_critic: dict[str, dict[str, Any]] = {} + for critic_name in ["correctness", "completeness", "security", "code_hygiene", "cross_consistency"]: + # Filter findings per critic — create synthetic ArtifactScores + critic_scores = [] + for s in scores: + # This is a simplification — full per-critic slicing would require + # re-running matching per critic. For now, use metadata category. + pass + by_critic[critic_name] = {} # populated below + + sliced = { + "by_complexity": _slice(lambda s: s.metadata.get("complexity", "unknown")), + "by_file_type": _slice(lambda s: s.metadata.get("domain", "unknown")), + "by_source": _slice(lambda s: s.metadata.get("source", "unknown")), + } + + # Per-severity slice (need to reconstruct from finding details) + by_severity: dict[str, dict[str, int]] = {} + for s in scores: + for mr in s.findings_detail: + sev = mr.gt_severity + if sev not in by_severity: + by_severity[sev] = {"tp": 0, "fn": 0} + if mr.matched: + by_severity[sev]["tp"] += 1 + else: + by_severity[sev]["fn"] += 1 + severity_slice = {} + for sev, counts in sorted(by_severity.items()): + t = counts["tp"] + f = counts["fn"] + r = _safe_div(t, t + f) + severity_slice[sev] = {"recall": round(r, 4), "tp": t, "fn": f} + sliced["by_severity"] = severity_slice + + return { + "aggregate": aggregate, + **sliced, + } + + +# --------------------------------------------------------------------------- +# Validation mode +# --------------------------------------------------------------------------- + +def validate_annotations(annotations_dir: Path, golden_dir: Path) -> list[str]: + """Validate all annotations: schema compliance + SHA-256 integrity.""" + errors: list[str] = [] + count = 0 + for ann_path in sorted(annotations_dir.glob("*.annotations.yaml")): + count += 1 + try: + ann = parse_annotation(ann_path) + except Exception as e: + errors.append(f"{ann_path.name}: parse error: {e}") + continue + + # Check schema version + if ann.schema_version != SCHEMA_VERSION: + errors.append(f"{ann_path.name}: schema_version {ann.schema_version} != {SCHEMA_VERSION}") + + # Check required fields + if not ann.artifact: + errors.append(f"{ann_path.name}: missing artifact path") + if not ann.expected_verdict: + errors.append(f"{ann_path.name}: missing expected_verdict") + if ann.expected_verdict not in ("PASS", "PASS_WITH_NOTES", "REVISE", "REJECT"): + errors.append(f"{ann_path.name}: invalid expected_verdict '{ann.expected_verdict}'") + + # Check artifact exists and SHA-256 matches + artifact_path = golden_dir / ann.artifact + if not artifact_path.exists(): + errors.append(f"{ann_path.name}: artifact not found at {ann.artifact}") + elif artifact_path.is_dir(): + # Cross-artifact pairs: SHA is typically of the primary document + # Just verify the directory exists and has files + if not any(artifact_path.iterdir()): + errors.append(f"{ann_path.name}: artifact directory is empty at {ann.artifact}") + else: + actual_sha = compute_sha256(artifact_path) + if actual_sha != ann.artifact_sha256: + errors.append( + f"{ann_path.name}: SHA-256 mismatch — " + f"expected {ann.artifact_sha256[:16]}... got {actual_sha[:16]}..." + ) + + # Check findings have required fields + seen_ids: set[str] = set() + for fd in ann.findings: + if fd.id in seen_ids: + errors.append(f"{ann_path.name}: duplicate finding ID {fd.id}") + seen_ids.add(fd.id) + if fd.severity not in SEVERITY_TIERS: + errors.append(f"{ann_path.name}: {fd.id} invalid severity '{fd.severity}'") + if fd.category not in ("security", "correctness", "completeness", "code_hygiene", "cross_consistency"): + errors.append(f"{ann_path.name}: {fd.id} invalid category '{fd.category}'") + + # Check metadata + for req in ("source", "domain", "complexity", "rubric", "depth", "author", "created"): + if req not in ann.metadata: + errors.append(f"{ann_path.name}: missing metadata.{req}") + + # PASS artifacts should have no findings + if ann.expected_verdict == "PASS" and ann.findings: + errors.append(f"{ann_path.name}: PASS verdict but has {len(ann.findings)} findings") + + if count == 0: + errors.append("No annotation files found") + + return errors + + +# --------------------------------------------------------------------------- +# Output formatting +# --------------------------------------------------------------------------- + +def format_json(metrics: dict, scores: list[ArtifactScore], run_dir: str) -> str: + """Format results as JSON.""" + from datetime import datetime, timezone + output = { + "schema_version": SCHEMA_VERSION, + "run_dir": run_dir, + "timestamp": datetime.now(timezone.utc).isoformat(), + **metrics, + "per_artifact": [], + } + for s in scores: + output["per_artifact"].append({ + "artifact": s.artifact, + "expected_verdict": s.expected_verdict, + "actual_verdict": s.actual_verdict, + "verdict_correct": s.verdict_correct, + "tp": s.tp, + "fp": s.fp, + "fn": s.fn, + "trapped_fp": s.trapped_fp, + "findings_detail": [ + { + "gt_id": mr.gt_id, + "matched": mr.matched, + "quorum_finding_id": mr.quorum_finding_id, + "severity_match": mr.severity_match, + "gt_severity": mr.gt_severity, + "quorum_severity": mr.quorum_severity, + } + for mr in s.findings_detail + ], + }) + return json.dumps(output, indent=2) + + +def format_markdown(metrics: dict, scores: list[ArtifactScore], thresholds: dict) -> str: + """Format results as Markdown report.""" + agg = metrics["aggregate"] + lines = [ + "# Golden Test Set — Scoring Report", + "", + "## Summary", + "", + "| Metric | Value |", + "|--------|-------|", + f"| Detection Precision | {agg['precision']:.2%} |", + f"| Detection Recall | {agg['recall']:.2%} |", + f"| F1 | {agg['f1']:.2%} |", + f"| Severity Accuracy | {agg['severity_accuracy']:.2%} |", + f"| Severity Distance (mean) | {agg['severity_distance_mean']:.2f} |", + f"| FP Rate (clean) | {agg['fp_rate_clean']:.2%} |", + f"| Verdict Accuracy | {agg['verdict_accuracy']:.2%} |", + f"| Trapped FP Count | {agg['trapped_fp_count']} |", + "", + f"**Totals:** {agg['tp']} TP, {agg['fp']} FP, {agg['fn']} FN " + f"({agg['total_gt_findings']} GT findings, {agg['total_quorum_findings']} Quorum findings)", + "", + ] + + # Thresholds + lines.extend([ + "## Thresholds", + "", + "| Metric | Target | Actual | Status |", + "|--------|--------|--------|--------|", + ]) + checks = [ + ("Recall", thresholds.get("min_recall", 0.80), agg["recall"]), + ("Precision", thresholds.get("min_precision", 0.70), agg["precision"]), + ] + all_met = True + for name, target, actual in checks: + met = actual >= target + if not met: + all_met = False + status = "PASS" if met else "FAIL" + lines.append(f"| {name} | {target:.0%} | {actual:.2%} | {status} |") + lines.append("") + + # Per-complexity breakdown + if "by_complexity" in metrics: + lines.extend(["## By Complexity", "", "| Complexity | Precision | Recall | F1 |", "|------------|-----------|--------|-----|"]) + for k, v in metrics["by_complexity"].items(): + lines.append(f"| {k} | {v['precision']:.2%} | {v['recall']:.2%} | {v['f1']:.2%} |") + lines.append("") + + # Per-severity breakdown + if "by_severity" in metrics: + lines.extend(["## By Severity", "", "| Severity | Recall | TP | FN |", "|----------|--------|----|----|"]) + for k, v in metrics["by_severity"].items(): + lines.append(f"| {k} | {v['recall']:.2%} | {v['tp']} | {v['fn']} |") + lines.append("") + + # Per-artifact results + lines.extend(["## Per-Artifact Results", "", "| Artifact | Expected | Actual | Verdict | TP | FP | FN |", + "|----------|----------|--------|---------|----|----|-----|"]) + for s in sorted(scores, key=lambda x: x.artifact): + v_mark = "correct" if s.verdict_correct else "WRONG" + lines.append(f"| {s.artifact} | {s.expected_verdict} | {s.actual_verdict or 'N/A'} | {v_mark} | {s.tp} | {s.fp} | {s.fn} |") + lines.append("") + + # Worst performers + worst = sorted([s for s in scores if s.fn > 0], key=lambda x: -x.fn)[:5] + if worst: + lines.extend(["## Worst Performers (most missed findings)", ""]) + for s in worst: + missed = [mr.gt_id for mr in s.findings_detail if not mr.matched] + lines.append(f"- **{s.artifact}**: {s.fn} missed — {', '.join(missed)}") + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Main scoring pipeline +# --------------------------------------------------------------------------- + +def score( + run_dir: Path, + annotations_dir: Path, + golden_dir: Path, + thresholds: dict[str, float], +) -> tuple[dict[str, Any], list[ArtifactScore], bool]: + """Run the full scoring pipeline. + + Returns: (metrics_dict, per_artifact_scores, thresholds_met) + """ + scores: list[ArtifactScore] = [] + warnings: list[str] = [] + + for ann_path in sorted(annotations_dir.glob("*.annotations.yaml")): + ann = parse_annotation(ann_path) + + # SHA-256 integrity check + artifact_path = golden_dir / ann.artifact + if artifact_path.exists(): + actual_sha = compute_sha256(artifact_path) + if actual_sha != ann.artifact_sha256: + warnings.append( + f"WARNING: SHA-256 mismatch for {ann.artifact} " + f"(expected {ann.artifact_sha256[:16]}..., got {actual_sha[:16]}...)" + ) + + # Parse Quorum output + actual_verdict, quorum_findings = parse_quorum_run(run_dir, ann.artifact) + + # Match findings + match_results, tp, fp, fn, trapped_fp, sev_matches, sev_dists = match_findings( + ann.findings, quorum_findings, ann.false_positive_traps + ) + + verdict_correct = (actual_verdict == ann.expected_verdict) if actual_verdict else False + + scores.append(ArtifactScore( + artifact=ann.artifact, + expected_verdict=ann.expected_verdict, + actual_verdict=actual_verdict, + verdict_correct=verdict_correct, + tp=tp, + fp=fp, + fn=fn, + trapped_fp=trapped_fp, + findings_detail=match_results, + metadata=ann.metadata, + severity_matches=sev_matches, + severity_distances=sev_dists, + )) + + # Print warnings + for w in warnings: + print(w, file=sys.stderr) + + # Compute metrics + metrics = compute_metrics(scores) + + # Check thresholds + agg = metrics["aggregate"] + met = True + for key, target in thresholds.items(): + metric_name = key.replace("min_", "") + if metric_name in agg and agg[metric_name] < target: + met = False + print(f"THRESHOLD NOT MET: {metric_name} = {agg[metric_name]:.4f} < {target}", file=sys.stderr) + + metrics["thresholds"] = {**thresholds, "met": met} + + return metrics, scores, met + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> int: + parser = argparse.ArgumentParser(description="Score Quorum output against golden test set annotations") + parser.add_argument("--run-dir", type=Path, help="Path to Quorum run output directory") + parser.add_argument("--annotations-dir", type=Path, default=Path("annotations"), + help="Path to annotations directory (default: annotations/)") + parser.add_argument("--golden-dir", type=Path, default=None, + help="Path to golden-test-set root (default: parent of annotations-dir)") + parser.add_argument("--min-recall", type=float, default=0.80) + parser.add_argument("--min-precision", type=float, default=0.70) + parser.add_argument("--format", choices=["json", "markdown", "both"], default="both", + dest="output_format") + parser.add_argument("--output-dir", type=Path, default=None, + help="Write output files to this directory") + parser.add_argument("--validate", action="store_true", + help="Validate annotations without scoring (no run-dir needed)") + + args = parser.parse_args() + + golden_dir = args.golden_dir or args.annotations_dir.parent + + # Validate mode + if args.validate: + errors = validate_annotations(args.annotations_dir, golden_dir) + if errors: + print(f"Validation found {len(errors)} error(s):", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 2 + print(f"All annotations valid.") + return 0 + + # Score mode + if not args.run_dir: + parser.error("--run-dir is required for scoring (use --validate for validation-only)") + + if not args.run_dir.is_dir(): + print(f"ERROR: run directory not found: {args.run_dir}", file=sys.stderr) + return 2 + + thresholds = { + "min_recall": args.min_recall, + "min_precision": args.min_precision, + } + + try: + metrics, scores, met = score(args.run_dir, args.annotations_dir, golden_dir, thresholds) + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + return 2 + + # Output + if args.output_format in ("json", "both"): + json_str = format_json(metrics, scores, str(args.run_dir)) + if args.output_dir: + out_path = args.output_dir / "score-results.json" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json_str) + print(f"JSON results written to {out_path}") + else: + print(json_str) + + if args.output_format in ("markdown", "both"): + md_str = format_markdown(metrics, scores, thresholds) + if args.output_dir: + out_path = args.output_dir / "score-report.md" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(md_str) + print(f"Markdown report written to {out_path}") + else: + print(md_str) + + return 0 if met else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/golden-test-set/tests/test_score.py b/golden-test-set/tests/test_score.py new file mode 100644 index 0000000..64432a0 --- /dev/null +++ b/golden-test-set/tests/test_score.py @@ -0,0 +1,483 @@ +"""Tests for the golden test set scoring framework.""" + +from __future__ import annotations + +import hashlib +import json +import textwrap +from pathlib import Path + +import pytest +import yaml + +# Add scripts/ to path +import sys +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +from score import ( + GTFinding, + FPTrap, + QuorumFinding, + Annotation, + ArtifactScore, + MatchResult, + match_findings, + compute_metrics, + compute_sha256, + parse_annotation, + validate_annotations, + _parse_line_numbers, + _location_match, + _fuzzy_match, + _is_trap_match, + _safe_div, +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def tmp_golden(tmp_path): + """Create a minimal golden test set structure.""" + (tmp_path / "artifacts" / "python").mkdir(parents=True) + (tmp_path / "annotations").mkdir() + return tmp_path + + +def _make_artifact(golden_dir: Path, rel_path: str, content: str) -> str: + """Write an artifact file, return its SHA-256.""" + full = golden_dir / rel_path + full.parent.mkdir(parents=True, exist_ok=True) + full.write_text(content) + return hashlib.sha256(content.encode()).hexdigest() + + +def _make_annotation(golden_dir: Path, name: str, data: dict): + """Write an annotation YAML file.""" + ann_path = golden_dir / "annotations" / f"{name}.annotations.yaml" + ann_path.parent.mkdir(parents=True, exist_ok=True) + ann_path.write_text(yaml.dump(data, default_flow_style=False)) + + +def _make_gt_finding(id: str, severity: str = "HIGH", category: str = "security", + critic: str = "security", location: str | None = "line 42") -> GTFinding: + return GTFinding( + id=id, + description=f"Test finding {id}", + location=location, + severity=severity, + category=category, + critic=critic, + ) + + +def _make_q_finding(id: str = "F-test", severity: str = "HIGH", category: str = "security", + critic: str = "security", location: str | None = "line 42", + description: str = "Test finding") -> QuorumFinding: + return QuorumFinding( + id=id, + severity=severity, + category=category, + description=description, + location=location, + critic=critic, + ) + + +# --------------------------------------------------------------------------- +# Line number parsing +# --------------------------------------------------------------------------- + +class TestParseLineNumbers: + def test_single_line(self): + assert _parse_line_numbers("line 42") == [42] + + def test_line_range(self): + assert _parse_line_numbers("line 42-48") == [42, 43, 44, 45, 46, 47, 48] + + def test_no_prefix(self): + assert _parse_line_numbers("42-45") == [42, 43, 44, 45] + + def test_none(self): + assert _parse_line_numbers(None) == [] + + def test_no_numbers(self): + assert _parse_line_numbers("section 3.2") == [] + + def test_multiple_references(self): + nums = _parse_line_numbers("line 10, line 20-22") + assert 10 in nums + assert 20 in nums + assert 22 in nums + + +# --------------------------------------------------------------------------- +# Location matching +# --------------------------------------------------------------------------- + +class TestLocationMatch: + def test_exact_match(self): + assert _location_match("line 42", "line 42") + + def test_within_tolerance(self): + assert _location_match("line 42", "line 45") # diff=3, within ±5 + + def test_outside_tolerance(self): + assert not _location_match("line 42", "line 55") # diff=13, outside ±5 + + def test_range_overlap(self): + assert _location_match("line 40-45", "line 43") + + def test_none_inputs(self): + assert not _location_match(None, "line 42") + assert not _location_match("line 42", None) + assert not _location_match(None, None) + + +# --------------------------------------------------------------------------- +# Fuzzy matching +# --------------------------------------------------------------------------- + +class TestFuzzyMatch: + def test_identical(self): + assert _fuzzy_match("SQL injection", "SQL injection") == 1.0 + + def test_similar(self): + score = _fuzzy_match("SQL injection via user input", "SQL injection through user input") + assert score >= FUZZY_THRESHOLD + + def test_dissimilar(self): + score = _fuzzy_match("SQL injection", "Dead code removal") + assert score < FUZZY_THRESHOLD + + def test_case_insensitive(self): + assert _fuzzy_match("SQL Injection", "sql injection") == 1.0 + + +FUZZY_THRESHOLD = 0.6 + + +# --------------------------------------------------------------------------- +# Matching algorithm +# --------------------------------------------------------------------------- + +class TestMatchFindings: + def test_perfect_tp(self): + """All GT findings matched exactly.""" + gt = [_make_gt_finding("GT-001")] + qf = [_make_q_finding("F-001")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings(gt, qf, []) + assert tp == 1 + assert fp == 0 + assert fn == 0 + assert results[0].matched + assert results[0].severity_match + + def test_false_negative(self): + """GT finding with no matching Quorum finding.""" + gt = [_make_gt_finding("GT-001")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings(gt, [], []) + assert tp == 0 + assert fn == 1 + assert not results[0].matched + + def test_false_positive(self): + """Quorum finding with no matching GT finding.""" + qf = [_make_q_finding("F-001", category="correctness", critic="correctness")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings([], qf, []) + assert tp == 0 + assert fp == 1 + assert fn == 0 + + def test_trapped_false_positive(self): + """Quorum finding that matches a false-positive trap.""" + qf = [_make_q_finding("F-001", location="line 30", + description="Use of eval() on hardcoded constant input")] + traps = [FPTrap(description="Use of eval() on hardcoded constant input", location="line 30")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings([], qf, traps) + assert trapped == 1 + assert fp == 0 + + def test_critic_mismatch_no_match(self): + """Different critics should not match.""" + gt = [_make_gt_finding("GT-001", critic="security", category="security")] + qf = [_make_q_finding("F-001", critic="correctness", category="correctness")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings(gt, qf, []) + assert tp == 0 + assert fn == 1 + assert fp == 1 + + def test_severity_mismatch_still_matches(self): + """Detection matches even if severity differs.""" + gt = [_make_gt_finding("GT-001", severity="HIGH")] + qf = [_make_q_finding("F-001", severity="CRITICAL")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings(gt, qf, []) + assert tp == 1 + assert not results[0].severity_match + assert sev_d == [1] # |3-4| = 1 + + def test_location_fuzzy_fallback(self): + """Match by description fuzzy when no location overlap.""" + gt = [_make_gt_finding("GT-001", location=None)] + gt[0].description = "SQL injection via unsanitized user input" + qf = [_make_q_finding("F-001", location=None, + description="SQL injection through unsanitized user input")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings(gt, qf, []) + assert tp == 1 + + def test_greedy_one_to_one(self): + """Each GT and Quorum finding matched at most once.""" + gt = [_make_gt_finding("GT-001"), _make_gt_finding("GT-002")] + qf = [_make_q_finding("F-001")] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings(gt, qf, []) + assert tp == 1 + assert fn == 1 + + def test_empty_inputs(self): + """No findings at all.""" + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings([], [], []) + assert tp == 0 + assert fp == 0 + assert fn == 0 + + +# --------------------------------------------------------------------------- +# Severity distance +# --------------------------------------------------------------------------- + +class TestSeverityDistance: + def test_exact_match(self): + gt = [_make_gt_finding("GT-001", severity="CRITICAL")] + qf = [_make_q_finding("F-001", severity="CRITICAL")] + _, _, _, _, _, sev_m, sev_d = match_findings(gt, qf, []) + assert sev_m == 1 + assert sev_d == [0] + + def test_one_tier_off(self): + gt = [_make_gt_finding("GT-001", severity="HIGH")] + qf = [_make_q_finding("F-001", severity="CRITICAL")] + _, _, _, _, _, sev_m, sev_d = match_findings(gt, qf, []) + assert sev_m == 0 + assert sev_d == [1] + + def test_two_tiers_off(self): + gt = [_make_gt_finding("GT-001", severity="LOW")] + qf = [_make_q_finding("F-001", severity="HIGH")] + _, _, _, _, _, sev_m, sev_d = match_findings(gt, qf, []) + assert sev_d == [2] + + +# --------------------------------------------------------------------------- +# Clean (PASS) artifact handling +# --------------------------------------------------------------------------- + +class TestCleanArtifacts: + def test_clean_no_findings(self): + """PASS artifact with no Quorum findings → perfect.""" + s = ArtifactScore( + artifact="clean.py", + expected_verdict="PASS", + actual_verdict="PASS", + verdict_correct=True, + tp=0, fp=0, fn=0, + metadata={"complexity": "low", "domain": "python-code", "source": "synthetic"}, + ) + metrics = compute_metrics([s]) + assert metrics["aggregate"]["fp_rate_clean"] == 0.0 + + def test_clean_with_fp(self): + """PASS artifact where Quorum incorrectly flags something.""" + s = ArtifactScore( + artifact="clean.py", + expected_verdict="PASS", + actual_verdict="PASS_WITH_NOTES", + verdict_correct=False, + tp=0, fp=2, fn=0, + metadata={"complexity": "low", "domain": "python-code", "source": "synthetic"}, + ) + metrics = compute_metrics([s]) + assert metrics["aggregate"]["fp_rate_clean"] == 1.0 + + +# --------------------------------------------------------------------------- +# SHA-256 integrity +# --------------------------------------------------------------------------- + +class TestSHA256: + def test_known_hash(self, tmp_path): + f = tmp_path / "test.txt" + content = "hello world\n" + f.write_text(content) + expected = hashlib.sha256(content.encode()).hexdigest() + assert compute_sha256(f) == expected + + def test_empty_file(self, tmp_path): + f = tmp_path / "empty.txt" + f.write_text("") + expected = hashlib.sha256(b"").hexdigest() + assert compute_sha256(f) == expected + + +# --------------------------------------------------------------------------- +# Annotation parsing +# --------------------------------------------------------------------------- + +class TestAnnotationParsing: + def test_round_trip(self, tmp_golden): + content = "print('hello')\n" + sha = _make_artifact(tmp_golden, "artifacts/python/test.py", content) + ann_data = { + "schema_version": "1.0", + "artifact": "artifacts/python/test.py", + "artifact_sha256": sha, + "expected_verdict": "PASS", + "findings": [], + "false_positive_traps": [], + "metadata": { + "source": "synthetic", + "domain": "python-code", + "complexity": "low", + "rubric": "python-code", + "depth": "standard", + "author": "test", + "created": "2026-03-12", + }, + } + _make_annotation(tmp_golden, "test", ann_data) + ann = parse_annotation(tmp_golden / "annotations" / "test.annotations.yaml") + assert ann.artifact == "artifacts/python/test.py" + assert ann.expected_verdict == "PASS" + assert len(ann.findings) == 0 + + def test_with_findings(self, tmp_golden): + sha = _make_artifact(tmp_golden, "artifacts/python/vuln.py", "x = 1\n") + ann_data = { + "schema_version": "1.0", + "artifact": "artifacts/python/vuln.py", + "artifact_sha256": sha, + "expected_verdict": "REVISE", + "findings": [ + { + "id": "GT-001", + "description": "SQL injection", + "location": "line 42", + "severity": "CRITICAL", + "category": "security", + "critic": "security", + } + ], + "metadata": { + "source": "synthetic", "domain": "python-code", "complexity": "medium", + "rubric": "python-code", "depth": "standard", "author": "test", "created": "2026-03-12", + }, + } + _make_annotation(tmp_golden, "vuln", ann_data) + ann = parse_annotation(tmp_golden / "annotations" / "vuln.annotations.yaml") + assert len(ann.findings) == 1 + assert ann.findings[0].id == "GT-001" + assert ann.findings[0].severity == "CRITICAL" + + +# --------------------------------------------------------------------------- +# Validation mode +# --------------------------------------------------------------------------- + +class TestValidation: + def test_valid_annotation(self, tmp_golden): + content = "x = 1\n" + sha = _make_artifact(tmp_golden, "artifacts/python/test.py", content) + _make_annotation(tmp_golden, "test", { + "schema_version": "1.0", + "artifact": "artifacts/python/test.py", + "artifact_sha256": sha, + "expected_verdict": "PASS", + "findings": [], + "metadata": { + "source": "synthetic", "domain": "python-code", "complexity": "low", + "rubric": "python-code", "depth": "standard", "author": "test", "created": "2026-03-12", + }, + }) + errors = validate_annotations(tmp_golden / "annotations", tmp_golden) + assert errors == [] + + def test_sha_mismatch(self, tmp_golden): + _make_artifact(tmp_golden, "artifacts/python/test.py", "x = 1\n") + _make_annotation(tmp_golden, "test", { + "schema_version": "1.0", + "artifact": "artifacts/python/test.py", + "artifact_sha256": "deadbeef" * 8, + "expected_verdict": "PASS", + "findings": [], + "metadata": { + "source": "synthetic", "domain": "python-code", "complexity": "low", + "rubric": "python-code", "depth": "standard", "author": "test", "created": "2026-03-12", + }, + }) + errors = validate_annotations(tmp_golden / "annotations", tmp_golden) + assert any("SHA-256 mismatch" in e for e in errors) + + def test_missing_artifact(self, tmp_golden): + _make_annotation(tmp_golden, "test", { + "schema_version": "1.0", + "artifact": "artifacts/python/nonexistent.py", + "artifact_sha256": "abc123", + "expected_verdict": "PASS", + "findings": [], + "metadata": { + "source": "synthetic", "domain": "python-code", "complexity": "low", + "rubric": "python-code", "depth": "standard", "author": "test", "created": "2026-03-12", + }, + }) + errors = validate_annotations(tmp_golden / "annotations", tmp_golden) + assert any("not found" in e for e in errors) + + def test_pass_with_findings_error(self, tmp_golden): + sha = _make_artifact(tmp_golden, "artifacts/python/test.py", "x = 1\n") + _make_annotation(tmp_golden, "test", { + "schema_version": "1.0", + "artifact": "artifacts/python/test.py", + "artifact_sha256": sha, + "expected_verdict": "PASS", + "findings": [{"id": "GT-001", "description": "bug", "severity": "HIGH", + "category": "security", "critic": "security"}], + "metadata": { + "source": "synthetic", "domain": "python-code", "complexity": "low", + "rubric": "python-code", "depth": "standard", "author": "test", "created": "2026-03-12", + }, + }) + errors = validate_annotations(tmp_golden / "annotations", tmp_golden) + assert any("PASS verdict but has" in e for e in errors) + + def test_no_annotations(self, tmp_golden): + errors = validate_annotations(tmp_golden / "annotations", tmp_golden) + assert any("No annotation" in e for e in errors) + + +# --------------------------------------------------------------------------- +# Edge cases +# --------------------------------------------------------------------------- + +class TestEdgeCases: + def test_all_false_positives(self): + """Quorum only produces findings that don't match anything.""" + qf = [ + _make_q_finding("F-001", critic="correctness", category="correctness"), + _make_q_finding("F-002", critic="completeness", category="completeness"), + ] + results, tp, fp, fn, trapped, sev_m, sev_d = match_findings([], qf, []) + assert tp == 0 + assert fp == 2 + assert fn == 0 + + def test_metrics_with_zero_scores(self): + """Empty scores list should not crash.""" + metrics = compute_metrics([]) + assert metrics["aggregate"]["precision"] == 0.0 + assert metrics["aggregate"]["recall"] == 0.0 + + def test_safe_div_zero(self): + assert _safe_div(1, 0) == 0.0 + assert _safe_div(0, 0) == 0.0 + assert _safe_div(3, 4) == 0.75