Refactor compile extensions into module-based layout with per-extension impl files

src/compile/extensions/azure_devops.rs

@@ -0,0 +1,193 @@
+// ─── Azure DevOps MCP ────────────────────────────────────────────────
+
+use super::{
+    CompileContext, CompilerExtension, ExtensionPhase, McpgServerConfig, PipelineEnvMapping,
+};
+use crate::allowed_hosts::mcp_required_hosts;
+use crate::compile::common::{
+    ADO_MCP_ENTRYPOINT, ADO_MCP_IMAGE, ADO_MCP_PACKAGE, ADO_MCP_SERVER_NAME,
+};
+use crate::compile::types::AzureDevOpsToolConfig;
+use anyhow::Result;
+use std::collections::HashMap;
+
+/// Azure DevOps first-party tool extension.
+///
+/// Injects: network hosts (ADO domains), MCPG server entry (containerized
+/// ADO MCP), and compile-time validation (org inference, duplicate MCP).
+pub struct AzureDevOpsExtension {
+    config: AzureDevOpsToolConfig,
+    auth_mode: AdoAuthMode,
+}
+
+/// Authentication mode for the ADO MCP server.
+///
+/// Pipelines use bearer tokens (JWT from ARM service connections).
+/// Local development uses PATs (Personal Access Tokens).
+#[derive(Debug, Clone, Copy, Default)]
+pub enum AdoAuthMode {
+    /// `-a envvar` + `ADO_MCP_AUTH_TOKEN` — bearer JWT from ARM (pipeline default)
+    #[default]
+    Bearer,
+    /// `-a pat` + `AZURE_DEVOPS_EXT_PAT` — Personal Access Token (local dev)
+    Pat,
+}
+
+impl AzureDevOpsExtension {
+    pub fn new(config: AzureDevOpsToolConfig) -> Self {
+        Self {
+            config,
+            auth_mode: AdoAuthMode::default(),
+        }
+    }
+
+    /// Set the authentication mode (e.g., `AdoAuthMode::Pat` for local runs).
+    pub fn with_auth_mode(mut self, mode: AdoAuthMode) -> Self {
+        self.auth_mode = mode;
+        self
+    }
+}
+
+impl CompilerExtension for AzureDevOpsExtension {
+    fn name(&self) -> &str {
+        "Azure DevOps MCP"
+    }
+
+    fn phase(&self) -> ExtensionPhase {
+        ExtensionPhase::Tool
+    }
+
+    fn required_hosts(&self) -> Vec<String> {
+        let mut hosts: Vec<String> = mcp_required_hosts("ado")
+            .iter()
+            .map(|h| (*h).to_string())
+            .collect();
+        // The ADO MCP runs in a container via `npx -y @azure-devops/mcp`.
+        // npx needs npm registry access to resolve and install the package.
+        hosts.push("node".to_string());
+        hosts
+    }
+
+    fn allowed_copilot_tools(&self) -> Vec<String> {
+        vec![ADO_MCP_SERVER_NAME.to_string()]
+    }
+
+    fn mcpg_servers(&self, ctx: &CompileContext) -> Result<Vec<(String, McpgServerConfig)>> {
+        // Build entrypoint args: npx -y @azure-devops/mcp <org> [-d toolset1 toolset2 ...]
+        let mut entrypoint_args = vec!["-y".to_string(), ADO_MCP_PACKAGE.to_string()];
+
+        // Org: use explicit override, then inferred from git remote, then fail
+        let org = self
+            .config
+            .org()
+            .map(|s| s.to_string())
+            .or_else(|| ctx.ado_org().map(|s| s.to_string()))
+            .ok_or_else(|| {
+                anyhow::anyhow!(
+                    "Agent '{}' has tools.azure-devops enabled but no ADO organization could be \
+                     determined. Either set tools.azure-devops.org explicitly, or compile from \
+                     within a git repository with an Azure DevOps remote URL.",
+                    ctx.agent_name
+                )
+            })?;
+        if !org.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') {
+            anyhow::bail!(
+                "Invalid ADO org name '{}': must contain only alphanumerics and hyphens",
+                org
+            );
+        }
+        entrypoint_args.push(org);
+
+        // Toolsets: passed as -d flag followed by space-separated toolset names
+        if !self.config.toolsets().is_empty() {
+            entrypoint_args.push("-d".to_string());
+            for toolset in self.config.toolsets() {
+                if !toolset
+                    .chars()
+                    .all(|c| c.is_ascii_alphanumeric() || c == '-')
+                {
+                    anyhow::bail!(
+                        "Invalid ADO toolset name '{}': must contain only alphanumerics and hyphens",
+                        toolset
+                    );
+                }
+                entrypoint_args.push(toolset.clone());
+            }
+        }
+
+        // Tool allow-list for MCPG filtering
+        let tools = if self.config.allowed().is_empty() {
+            None
+        } else {
+            Some(self.config.allowed().to_vec())
+        };
+
+        // ADO MCP authentication: the @azure-devops/mcp npm package accepts
+        // auth type via CLI arg (-a) and token via env var.
+        //   Bearer: `-a envvar` reads ADO_MCP_AUTH_TOKEN (pipeline JWT from ARM)
+        //   Pat:    `-a pat`    reads PERSONAL_ACCESS_TOKEN (base64-encoded PAT)
+        let (auth_flag, token_var) = match self.auth_mode {
+            AdoAuthMode::Bearer => ("envvar", "ADO_MCP_AUTH_TOKEN"),
+            AdoAuthMode::Pat => ("pat", "PERSONAL_ACCESS_TOKEN"),
+        };
+        entrypoint_args.extend(["-a".to_string(), auth_flag.to_string()]);
+
+        let env = Some(HashMap::from([(
+            token_var.to_string(),
+            String::new(), // Passthrough from MCPG process env
+        )]));
+
+        // --network host: AWF's DOCKER-USER iptables rules block outbound from
+        // containers on Docker's default bridge. Host networking bypasses FORWARD
+        // chain rules so the ADO MCP can reach dev.azure.com.
+        // This matches gh-aw's approach for its built-in agentic-workflows MCP.
+        let args = Some(vec!["--network".to_string(), "host".to_string()]);
+
+        Ok(vec![(
+            ADO_MCP_SERVER_NAME.to_string(),
+            McpgServerConfig {
+                server_type: "stdio".to_string(),
+                container: Some(ADO_MCP_IMAGE.to_string()),
+                entrypoint: Some(ADO_MCP_ENTRYPOINT.to_string()),
+                entrypoint_args: Some(entrypoint_args),
+                mounts: None,
+                args,
+                url: None,
+                headers: None,
+                env,
+                tools,
+            },
+        )])
+    }
+
+    fn validate(&self, ctx: &CompileContext) -> Result<Vec<String>> {
+        let mut warnings = Vec::new();
+
+        // Warn if user also has a manual mcp-servers entry for azure-devops
+        if ctx
+            .front_matter
+            .mcp_servers
+            .contains_key(ADO_MCP_SERVER_NAME)
+        {
+            warnings.push(format!(
+                "Agent '{}' has both tools.azure-devops and mcp-servers.azure-devops configured. \
+                 The tools.azure-devops auto-configuration takes precedence. \
+                 Remove the mcp-servers entry to silence this warning.",
+                ctx.agent_name
+            ));
+        }
+
+        Ok(warnings)
+    }
+    fn required_pipeline_vars(&self) -> Vec<PipelineEnvMapping> {
+        match self.auth_mode {
+            AdoAuthMode::Bearer => vec![PipelineEnvMapping {
+                container_var: "ADO_MCP_AUTH_TOKEN".to_string(),
+                pipeline_var: "SC_READ_TOKEN".to_string(),
+            }],
+            // PAT mode: no pipeline var mapping needed — the PAT is passed
+            // directly via AZURE_DEVOPS_EXT_PAT in the MCPG env file.
+            AdoAuthMode::Pat => vec![],
+        }
+    }
+}

src/compile/extensions/cache_memory.rs

@@ -0,0 +1,90 @@
+use super::{CompilerExtension, ExtensionPhase};
+use crate::compile::types::CacheMemoryToolConfig;
+
+/// Cache memory tool extension.
+///
+/// Injects: prepare steps (download/restore previous memory), and a
+/// prompt supplement informing the agent about its memory directory.
+pub struct CacheMemoryExtension {
+    /// Config options (e.g., `allowed-extensions`) are consumed at Stage 3
+    /// execution time, not at compile time. Retained here for potential
+    /// future compile-time validation.
+    #[allow(dead_code)]
+    config: CacheMemoryToolConfig,
+}
+
+impl CacheMemoryExtension {
+    pub fn new(config: CacheMemoryToolConfig) -> Self {
+        Self { config }
+    }
+}
+
+impl CompilerExtension for CacheMemoryExtension {
+    fn name(&self) -> &str {
+        "Cache Memory"
+    }
+
+    fn phase(&self) -> ExtensionPhase {
+        ExtensionPhase::Tool
+    }
+
+    fn prepare_steps(&self) -> Vec<String> {
+        vec![generate_memory_download()]
+    }
+
+    fn prompt_supplement(&self) -> Option<String> {
+        Some(
+            "\n\
+---\n\
+\n\
+## Agent Memory\n\
+\n\
+You have persistent memory across runs. Your memory directory is located at `/tmp/awf-tools/staging/agent_memory/`.\n\
+\n\
+- **Read** previous memory files from this directory to recall context from prior runs.\n\
+- **Write** new files or update existing ones in this directory to persist knowledge for future runs.\n\
+- Use this memory to track patterns, accumulate findings, remember decisions, and improve over time.\n\
+- The memory directory is yours to organize as you see fit (files, subdirectories, any structure).\n\
+- Memory files are sanitized between runs for security; avoid including pipeline commands or secrets.\n"
+                .to_string(),
+        )
+    }
+}
+
+/// Generate the steps to download agent memory from the previous successful run
+/// and restore it to the staging directory.
+fn generate_memory_download() -> String {
+    r#"- task: DownloadPipelineArtifact@2
+  displayName: "Download previous agent memory"
+  condition: eq(${{ parameters.clearMemory }}, false)
+  continueOnError: true
+  inputs:
+    source: "specific"
+    project: "$(System.TeamProject)"
+    pipeline: "$(System.DefinitionId)"
+    runVersion: "latestFromBranch"
+    branchName: "$(Build.SourceBranch)"
+    artifact: "safe_outputs"
+    targetPath: "$(Agent.TempDirectory)/previous_memory"
+    allowPartiallySucceededBuilds: true
+
+- bash: |
+    mkdir -p /tmp/awf-tools/staging/agent_memory
+    if [ -d "$(Agent.TempDirectory)/previous_memory/agent_memory" ]; then
+      cp -a "$(Agent.TempDirectory)/previous_memory/agent_memory/." /tmp/awf-tools/staging/agent_memory/ 2>/dev/null || true
+      echo "Previous agent memory restored to /tmp/awf-tools/staging/agent_memory"
+      ls -laR /tmp/awf-tools/staging/agent_memory
+    else
+      echo "No previous agent memory found - empty memory directory created"
+    fi
+  displayName: "Restore previous agent memory"
+  condition: eq(${{ parameters.clearMemory }}, false)
+  continueOnError: true
+
+- bash: |
+    mkdir -p /tmp/awf-tools/staging/agent_memory
+    echo "Memory cleared by pipeline parameter - starting fresh"
+  displayName: "Initialize empty agent memory (clearMemory=true)"
+  condition: eq(${{ parameters.clearMemory }}, true)"#
+        .to_string()
+}

src/compile/extensions/github.rs

@@ -0,0 +1,24 @@
+use super::{CompilerExtension, ExtensionPhase};
+
+// ─── GitHub (always-on, internal) ────────────────────────────────────
+
+/// GitHub MCP extension.
+///
+/// Always-on internal extension that grants the agent access to the
+/// Copilot CLI built-in GitHub MCP server via `--allow-tool github`.
+/// The GitHub MCP uses `GITHUB_TOKEN` from the pipeline environment.
+pub struct GitHubExtension;
+
+impl CompilerExtension for GitHubExtension {
+    fn name(&self) -> &str {
+        "GitHub"
+    }
+
+    fn phase(&self) -> ExtensionPhase {
+        ExtensionPhase::Tool
+    }
+
+    fn allowed_copilot_tools(&self) -> Vec<String> {
+        vec!["github".to_string()]
+    }
+}

src/compile/extensions/lean.rs

@@ -0,0 +1,79 @@
+// ─── Lean 4 ──────────────────────────────────────────────────────────
+
+use super::{CompileContext, CompilerExtension, ExtensionPhase};
+use crate::runtimes::lean::{self, LEAN_BASH_COMMANDS, LeanRuntimeConfig};
+use anyhow::Result;
+
+/// Lean 4 runtime extension.
+///
+/// Injects: network hosts (elan, lean-lang), bash commands (lean, lake,
+/// elan), install steps (elan + toolchain), and a prompt supplement.
+pub struct LeanExtension {
+    config: LeanRuntimeConfig,
+}
+
+impl LeanExtension {
+    pub fn new(config: LeanRuntimeConfig) -> Self {
+        Self { config }
+    }
+}
+
+impl CompilerExtension for LeanExtension {
+    fn name(&self) -> &str {
+        "Lean 4"
+    }
+
+    fn phase(&self) -> ExtensionPhase {
+        ExtensionPhase::Runtime
+    }
+
+    fn required_hosts(&self) -> Vec<String> {
+        vec!["lean".to_string()]
+    }
+
+    fn required_bash_commands(&self) -> Vec<String> {
+        LEAN_BASH_COMMANDS
+            .iter()
+            .map(|c| (*c).to_string())
+            .collect()
+    }
+
+    fn prompt_supplement(&self) -> Option<String> {
+        Some(
+            "\n\
+---\n\
+\n\
+## Lean 4 Formal Verification\n\
+\n\
+Lean 4 is installed and available. Use `lean` to typecheck `.lean` files, \
+`lake build` to build Lake projects, and `lake env printPaths` to inspect \
+the toolchain. Lean files use the `.lean` extension.\n"
+                .to_string(),
+        )
+    }
+
+    fn prepare_steps(&self) -> Vec<String> {
+        vec![lean::generate_lean_install(&self.config)]
+    }
+
+    fn validate(&self, ctx: &CompileContext) -> Result<Vec<String>> {
+        let mut warnings = Vec::new();
+
+        let is_bash_disabled = ctx
+            .front_matter
+            .tools
+            .as_ref()
+            .and_then(|t| t.bash.as_ref())
+            .is_some_and(|cmds| cmds.is_empty());
+
+        if is_bash_disabled {
+            warnings.push(format!(
+                "Agent '{}' has runtimes.lean enabled but tools.bash is empty. \
+                 Lean requires bash access (lean, lake, elan commands).",
+                ctx.agent_name
+            ));
+        }
+
+        Ok(warnings)
+    }
+}