feat(compile)!: trigger filter IR with data-driven Python evaluator (#345)

Author: jamesadevine

.github/workflows/release.yml

@@ -56,13 +56,20 @@ jobs:
           cd target/release
           cp ado-aw ado-aw-linux-x64
 
+      - name: Package scripts bundle
+        run: |
+          set -euo pipefail
+          cd scripts
+          zip -r ../scripts.zip .
+
       - name: Upload release assets
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           TAG="${{ needs.release-please.outputs.tag_name || github.event.inputs.tag_name }}"
           gh release upload "$TAG" \
             target/release/ado-aw-linux-x64 \
+            scripts.zip \
             --clobber
 
   build-windows:
@@ -152,11 +159,13 @@ jobs:
           TAG="${{ needs.release-please.outputs.tag_name || github.event.inputs.tag_name }}"
           gh release download "$TAG" \
             --pattern "ado-aw-*" \
+            --pattern "scripts.zip" \
             --repo "${{ github.repository }}"
           test -f ado-aw-linux-x64 || { echo "Missing ado-aw-linux-x64"; exit 1; }
           test -f ado-aw-windows-x64.exe || { echo "Missing ado-aw-windows-x64.exe"; exit 1; }
           test -f ado-aw-darwin-arm64 || { echo "Missing ado-aw-darwin-arm64"; exit 1; }
-          sha256sum ado-aw-linux-x64 ado-aw-windows-x64.exe ado-aw-darwin-arm64 > checksums.txt
+          test -f scripts.zip || { echo "Missing scripts.zip"; exit 1; }
+          sha256sum ado-aw-linux-x64 ado-aw-windows-x64.exe ado-aw-darwin-arm64 scripts.zip > checksums.txt
 
       - name: Upload checksums
         env:

.gitignore

@@ -1,2 +1,4 @@
 target
 examples/sample-agent.yml
+*.pyc
+__pycache__/

AGENTS.md

@@ -53,10 +53,13 @@ Every compiled pipeline runs as three sequential jobs:
 │   │   ├── standalone.rs # Standalone pipeline compiler
 │   │   ├── onees.rs      # 1ES Pipeline Template compiler
 │   │   ├── gitattributes.rs # .gitattributes management for compiled pipelines
+│   │   ├── filter_ir.rs  # Filter expression IR: Fact/Predicate types, lowering, validation, codegen
+│   │   ├── pr_filters.rs # PR trigger filter generation (native ADO + gate steps)
 │   │   ├── extensions/   # CompilerExtension trait and infrastructure extensions
 │   │   │   ├── mod.rs    # Trait, Extension enum, collect_extensions(), re-exports
 │   │   │   ├── github.rs # Always-on GitHub MCP extension
 │   │   │   ├── safe_outputs.rs # Always-on SafeOutputs MCP extension
+│   │   │   ├── trigger_filters.rs # Trigger filter extension (gate evaluator delivery)
 │   │   │   └── tests.rs  # Extension integration tests
 │   │   └── types.rs      # Front matter grammar and types
 │   ├── init.rs           # Repository initialization for AI-first authoring
@@ -116,6 +119,9 @@ Every compiled pipeline runs as three sequential jobs:
 │           └── execute.rs   # Stage 3 runtime (validate/copy)
 ├── ado-aw-derive/        # Proc-macro crate: #[derive(SanitizeConfig)], #[derive(SanitizeContent)]
 ├── examples/             # Example agent definitions
+├── scripts/              # Supporting scripts shipped as release artifacts
+│   ├── gate-eval.py      # Python gate evaluator (data-driven filter evaluation)
+│   └── gate-spec.schema.json # JSON Schema for gate spec (generated from Rust types)
 ├── tests/                # Integration tests and fixtures
 ├── docs/                 # Per-concept reference documentation (see index below)
 ├── Cargo.toml            # Rust dependencies
@@ -174,6 +180,9 @@ index to jump to the right page.
 - [`docs/extending.md`](docs/extending.md) — adding new CLI commands, compile
   targets, front-matter fields, template markers, safe-output tools,
   first-class tools, and runtimes; the `CompilerExtension` trait.
+- [`docs/filter-ir.md`](docs/filter-ir.md) — filter expression IR
+  specification: `Fact`/`Predicate` types, three-pass compilation (lower →
+  validate → codegen), gate step generation, adding new filter types.
 - [`docs/local-development.md`](docs/local-development.md) — local development
   setup notes.
 

Cargo.toml

@@ -13,7 +13,7 @@ dirs = "6"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_yaml = "0.9.34"
 serde_json = "1.0.149"
-schemars = "1.2"
+schemars = { version = "1.2", features = ["derive"] }
 rmcp = { version = "0.8.0", features = [
     "server",
     "transport-io",

docs/extending.md

@@ -33,15 +33,56 @@ Runtimes and first-party tools declare their compilation requirements via the `C
 ```rust
 pub trait CompilerExtension: Send {
     fn name(&self) -> &str;                                    // Display name
+    fn phase(&self) -> ExtensionPhase;                         // Runtime (0) < Tool (1)
     fn required_hosts(&self) -> Vec<String>;                   // AWF network allowlist
     fn required_bash_commands(&self) -> Vec<String>;           // Agent bash allow-list
     fn prompt_supplement(&self) -> Option<String>;              // Agent prompt markdown
-    fn prepare_steps(&self) -> Vec<String>;                    // Pipeline steps (install, etc.)
-    fn mcpg_servers(&self, ctx) -> Result<Vec<(String, McpgServerConfig)>>; // MCPG entries
+    fn prepare_steps(&self) -> Vec<String>;                    // Execution job steps (install, etc.)
+    fn setup_steps(&self, ctx: &CompileContext) -> Vec<String>; // Setup job steps (gates, pre-checks)
+    fn mcpg_servers(&self, ctx: &CompileContext) -> Result<Vec<(String, McpgServerConfig)>>; // MCPG entries
+    fn allowed_copilot_tools(&self) -> Vec<String>;            // --allow-tool values
+    fn validate(&self, ctx: &CompileContext) -> Result<Vec<String>>; // Compile-time warnings/errors
+    fn required_pipeline_vars(&self) -> Vec<PipelineEnvMapping>; // Container env var mappings
     fn required_awf_mounts(&self) -> Vec<AwfMount>;            // AWF Docker volume mounts
     fn awf_path_prepends(&self) -> Vec<String>;                // Directories to add to chroot PATH
-    fn validate(&self, ctx) -> Result<Vec<String>>;            // Compile-time warnings
 }
 ```
 
+**`prepare_steps()` vs `setup_steps()`**: `prepare_steps()` injects into the
+Execution job (before the agent runs). `setup_steps()` injects into the Setup
+job (before the Execution job starts). Use `setup_steps()` for pre-activation
+gates or checks that must complete before the agent is launched.
+
+**Phase ordering**: Extensions are sorted by phase — runtimes
+(`ExtensionPhase::Runtime`) execute before tools (`ExtensionPhase::Tool`).
+This guarantees runtime install steps run before tool steps that may depend
+on them.
+
 To add a new runtime or tool: (1) create a directory under `src/tools/` or `src/runtimes/`, (2) implement `CompilerExtension` in `extension.rs`, (3) add a variant to the `Extension` enum and a collection check in `collect_extensions()` in `src/compile/extensions/mod.rs`.
+
+### Filter IR (`src/compile/filter_ir.rs`)
+
+Trigger filter expressions (PR filters, pipeline filters) are compiled to bash
+gate steps via a three-pass IR pipeline:
+
+1. **Lower** — `PrFilters` / `PipelineFilters` → `Vec<FilterCheck>` (typed
+   predicates over typed facts)
+2. **Validate** — detect conflicts at compile time (impossible combinations,
+   redundant checks)
+3. **Codegen** — dependency-ordered fact acquisition + predicate evaluation →
+   bash gate step
+
+To add a new filter type:
+
+1. **Add a `Fact` variant** (if the filter needs a new data source) — implement
+   `dependencies()`, `kind()`, `ado_exports()`, and
+   `failure_policy()` on the new variant
+2. **Add a `Predicate` variant** (if the filter needs a new test shape) —
+   implement the codegen match arm in `emit_predicate_check()`
+3. **Extend lowering** — add the filter field to `PrFilters` or
+   `PipelineFilters` in `types.rs`, then add the lowering logic in
+   `lower_pr_filters()` or `lower_pipeline_filters()` in `filter_ir.rs`
+4. **Add validation rules** — check for conflicts with other filters in
+   `validate_pr_filters()` or `validate_pipeline_filters()`
+5. **Write tests** — lowering test, validation test, and codegen test in
+   `filter_ir.rs`