feat(eval): expand async and supply-chain fixture packs

haasonsaas · haasonsaas · commit 89742b0f1949 · 2026-03-14T14:09:23.000-07:00
Add new supply-chain cases for typosquats, unpinned dependencies, version downgrades, and go replace directives, plus new async cases for blocking runtime calls, nested event loops, and cancel ordering with checked-in pack tests.
diff --git a/eval/fixtures/deep_review_suite/review_depth_async.json b/eval/fixtures/deep_review_suite/review_depth_async.json
@@ -5,7 +5,9 @@
   "description": "Async-correctness benchmark pack for deeper live review runs.",
   "languages": [
     "rust",
-    "typescript"
+    "typescript",
+    "python",
+    "go"
   ],
   "categories": [
     "bug",
@@ -142,7 +144,8 @@
           ],
           "tags_any": [
             "async"
-          ]
+          ],
+          "rule_id": "bug.async.foreach-no-await"
         }
       ],
       "negative_findings": [
@@ -155,6 +158,108 @@
       "max_total": 8,
       "description": "Notification fan-out should keep awaiting all spawned async work.",
       "source": "deep-review-suite"
+    },
+    {
+      "name": "rust-blocking-in-async-runtime",
+      "category": "bug",
+      "language": "rust",
+      "difficulty": "Hard",
+      "diff_content": "diff --git a/src/cache.rs b/src/cache.rs\nindex 1111111..2222222 100644\n--- a/src/cache.rs\n+++ b/src/cache.rs\n@@ -1,5 +1,7 @@\n pub async fn warm_cache(path: &str) -> anyhow::Result<String> {\n-    tokio::fs::read_to_string(path).await.map_err(Into::into)\n+    tokio::spawn(async move {\n+        std::fs::read_to_string(path).unwrap()\n+    }).await.unwrap();\n+    Ok(String::new())\n }\n",
+      "expected_findings": [
+        {
+          "description": "Blocking filesystem I/O is performed inside async runtime work instead of using Tokio's async APIs or spawn_blocking.",
+          "severity": "Warning",
+          "category": "Bug",
+          "file_pattern": "src/cache.rs",
+          "line_hint": 3,
+          "contains_any": [
+            "blocking call in async context",
+            "std::fs::read_to_string",
+            "blocks the async runtime",
+            "use spawn_blocking",
+            "use tokio::fs"
+          ],
+          "rule_id": "bug.async.blocking-runtime-call"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Async Rust code should avoid blocking std::fs work on runtime threads.",
+      "source": "deep-review-suite"
+    },
+    {
+      "name": "python-asyncio-run-in-running-loop",
+      "category": "bug",
+      "language": "python",
+      "difficulty": "Medium",
+      "diff_content": "diff --git a/app/tasks.py b/app/tasks.py\nindex 1111111..2222222 100644\n--- a/app/tasks.py\n+++ b/app/tasks.py\n@@ -1,5 +1,5 @@\n async def refresh_reports():\n-    await sync_reports()\n+    asyncio.run(sync_reports())\n     return {\"ok\": True}\n",
+      "expected_findings": [
+        {
+          "description": "asyncio.run is called from inside an async function where an event loop is already running.",
+          "severity": "Warning",
+          "category": "Bug",
+          "file_pattern": "app/tasks.py",
+          "line_hint": 2,
+          "contains_any": [
+            "asyncio.run",
+            "already running event loop",
+            "nested event loop",
+            "runtimeerror",
+            "should await the coroutine directly"
+          ],
+          "rule_id": "bug.async.nested-event-loop"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Python async functions should not start a fresh event loop with asyncio.run.",
+      "source": "deep-review-suite"
+    },
+    {
+      "name": "go-context-cancel-defer-order",
+      "category": "bug",
+      "language": "go",
+      "difficulty": "Medium",
+      "diff_content": "diff --git a/service/worker.go b/service/worker.go\nindex 1111111..2222222 100644\n--- a/service/worker.go\n+++ b/service/worker.go\n@@ -1,7 +1,8 @@\n func StartWorker(parent context.Context) error {\n     ctx, cancel := context.WithTimeout(parent, time.Second)\n     go runWorker(ctx)\n-    defer cancel()\n+    cancel()\n+    defer cancel()\n     return nil\n }\n",
+      "expected_findings": [
+        {
+          "description": "Context is canceled immediately after spawning the goroutine, so the worker may observe a canceled context before it starts useful work.",
+          "severity": "Warning",
+          "category": "Bug",
+          "file_pattern": "service/worker.go",
+          "line_hint": 4,
+          "contains_any": [
+            "context canceled too early",
+            "goroutine may start with a canceled context",
+            "premature cancel",
+            "cancel is invoked immediately",
+            "defer cancel"
+          ],
+          "rule_id": "bug.async.context-cancel-order"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Go concurrency code should not cancel shared contexts before spawned work has a chance to run.",
+      "source": "deep-review-suite"
     }
   ]
 }
diff --git a/eval/fixtures/deep_review_suite/review_depth_supply_chain.json b/eval/fixtures/deep_review_suite/review_depth_supply_chain.json
@@ -4,6 +4,10 @@
   "version": "1.0.0",
   "description": "Supply-chain and dependency trust benchmark pack for deeper live review runs.",
   "languages": [
+    "typescript",
+    "python",
+    "rust",
+    "go",
     "yaml",
     "docker"
   ],
@@ -163,6 +167,145 @@
       "max_total": 8,
       "description": "CI workflows should not combine pull_request_target with attacker-controlled shell input.",
       "source": "deep-review-suite"
+    },
+    {
+      "name": "npm-typosquat-package",
+      "category": "security",
+      "language": "typescript",
+      "difficulty": "Medium",
+      "diff_content": "diff --git a/package.json b/package.json\nindex 1111111..2222222 100644\n--- a/package.json\n+++ b/package.json\n@@ -5,5 +5,6 @@\n   \"dependencies\": {\n     \"express\": \"^4.19.2\",\n-    \"lodash\": \"^4.17.21\"\n+    \"lodash\": \"^4.17.21\",\n+    \"lodahs\": \"^4.17.21\"\n   }\n }\n",
+      "expected_findings": [
+        {
+          "description": "New dependency name looks like a typosquat of lodash.",
+          "severity": "Warning",
+          "category": "Security",
+          "file_pattern": "package.json",
+          "line_hint": 7,
+          "contains_any": [
+            "typosquat",
+            "typosquatting",
+            "suspicious package name",
+            "lodahs",
+            "looks like lodash"
+          ],
+          "tags_any": [
+            "typosquatting",
+            "supply-chain"
+          ],
+          "rule_id": "sec.supply-chain.new-dependency"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Review newly added dependencies for typo-squatting and other supply-chain red flags.",
+      "source": "deep-review-suite"
+    },
+    {
+      "name": "python-unpinned-dependency",
+      "category": "security",
+      "language": "python",
+      "difficulty": "Easy",
+      "diff_content": "diff --git a/requirements.txt b/requirements.txt\nindex 1111111..2222222 100644\n--- a/requirements.txt\n+++ b/requirements.txt\n@@ -1,2 +1,3 @@\n flask==3.0.3\n sqlalchemy==2.0.36\n+requests\n",
+      "expected_findings": [
+        {
+          "description": "Dependency is added without a pinned version.",
+          "severity": "Info",
+          "category": "Security",
+          "file_pattern": "requirements.txt",
+          "line_hint": 3,
+          "contains_any": [
+            "unpinned dependency",
+            "pin to a specific version",
+            "requests without version",
+            "loosely pinned dependency"
+          ],
+          "rule_id": "sec.supply-chain.unpinned-version"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Requirements files should pin newly introduced packages to exact or tightly controlled versions.",
+      "source": "deep-review-suite"
+    },
+    {
+      "name": "rust-yanked-crate-version",
+      "category": "security",
+      "language": "rust",
+      "difficulty": "Hard",
+      "diff_content": "diff --git a/Cargo.toml b/Cargo.toml\nindex 1111111..2222222 100644\n--- a/Cargo.toml\n+++ b/Cargo.toml\n@@ -8,5 +8,5 @@ edition = \"2021\"\n [dependencies]\n anyhow = \"1\"\n-time = \"0.3.36\"\n+time = \"=0.1.43\" # downgrade to an old yanked release from a previous incident postmortem\n tokio = { version = \"1\", features = [\"full\"] }\n",
+      "expected_findings": [
+        {
+          "description": "Dependency is downgraded to an old yanked or known-bad crate version.",
+          "severity": "Warning",
+          "category": "Security",
+          "file_pattern": "Cargo.toml",
+          "line_hint": 4,
+          "contains_any": [
+            "old yanked version",
+            "downgrade to an older version",
+            "known bad version",
+            "version downgrade",
+            "time = \"=0.1.43\""
+          ],
+          "rule_id": "sec.supply-chain.version-downgrade"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Dependency downgrades to suspiciously old or yanked releases should be treated as supply-chain risk.",
+      "source": "deep-review-suite"
+    },
+    {
+      "name": "go-replace-directive-remote",
+      "category": "security",
+      "language": "go",
+      "difficulty": "Medium",
+      "diff_content": "diff --git a/go.mod b/go.mod\nindex 1111111..2222222 100644\n--- a/go.mod\n+++ b/go.mod\n@@ -3,3 +3,4 @@ module example.com/payments\n go 1.22\n require github.com/gorilla/mux v1.8.1\n+replace github.com/gorilla/mux => corp.example.com/forks/mux v1.8.1\n",
+      "expected_findings": [
+        {
+          "description": "go.mod replace directive redirects module resolution to a non-standard remote source.",
+          "severity": "Warning",
+          "category": "Security",
+          "file_pattern": "go.mod",
+          "line_hint": 4,
+          "contains_any": [
+            "replace directive",
+            "redirects module resolution",
+            "non-standard remote module path",
+            "suspicious go.mod replace",
+            "override dependency source"
+          ],
+          "rule_id": "sec.supply-chain.override-directive"
+        }
+      ],
+      "negative_findings": [
+        {
+          "description": "Avoid style-only comments.",
+          "contains": "style"
+        }
+      ],
+      "min_total": 1,
+      "max_total": 8,
+      "description": "Module replacement directives should be scrutinized because they bypass the normal upstream source of a dependency.",
+      "source": "deep-review-suite"
     }
   ]
 }
diff --git a/src/commands/eval/fixtures.rs b/src/commands/eval/fixtures.rs
