Inital commit

Co-authored-by: Copilot <copilot@github.com>

Author: leestott

.devcontainer/devcontainer.json

@@ -0,0 +1,32 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Model Router AutoEval",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:3-3.10-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/azure-cli:1": {},
+		"ghcr.io/azure/azure-dev/azd:0": {}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-toolsai.jupyter",
+				"ms-python.python"
+			]
+		}
+	}
+	
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.env.example

@@ -0,0 +1,18 @@
+# Azure Model Router endpoint
+AZURE_MODEL_ROUTER_ENDPOINT=https://your-resource.services.ai.azure.com/models
+AZURE_MODEL_ROUTER_KEY=your-model-router-api-key
+AZURE_MODEL_ROUTER_DEPLOYMENT=model-router
+
+# Azure OpenAI endpoint (for baseline model)
+AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com
+AZURE_OPENAI_KEY=your-azure-openai-api-key
+AZURE_BASELINE_DEPLOYMENT=gpt-5
+
+# Judge model (defaults to same endpoint as baseline; override to use a different model)
+AZURE_JUDGE_ENDPOINT=https://your-resource.openai.azure.com
+AZURE_JUDGE_KEY=your-azure-openai-api-key
+AZURE_JUDGE_DEPLOYMENT=gpt-5
+
+# Microsoft Foundry (optional — for cloud evaluation via Foundry SDK)
+AZURE_AI_PROJECT_ENDPOINT=https://your-project.services.ai.azure.com
+AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-5

.github/workflows/tests.yml

@@ -0,0 +1,61 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test (Python ${{ matrix.python-version }} on ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.9", "3.11", "3.13"]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,foundry,db]"
+
+      - name: Lint with ruff
+        run: ruff check src/ tests/ scripts/
+
+      - name: Run unit tests
+        run: pytest tests/ -v -m "not integration and not live"
+
+  smoke-demo:
+    name: Smoke test the demo (no API keys)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+      - name: Install
+        run: pip install -e .
+      - name: Generate sample report
+        run: python scripts/generate_sample_report.py --output-dir results/demo
+      - name: Verify outputs exist
+        run: |
+          test -f results/demo/dashboard.html
+          test -f results/demo/report.md
+          test -f results/demo/results.json