chore(merge): merge microsoft/main, drop doc/api.rst (migrated to auto-generated api docs)

Author: athal7

component-governance.yml .azuredevops/component-governance.ymlcomponent-governance.yml renamed to .azuredevops/component-governance.yml

@@ -0,0 +1,21 @@
+# Runs end-to-end scenario tests using pyrit_scan CLI
+
+trigger: none  # Disable automatic CI triggers
+
+schedules:
+- cron: "0 7 * * *"  # 7 AM UTC = 11 PM PST (UTC-8) / Midnight PDT (UTC-7)
+  displayName: Nightly E2E Tests at 11 PM PST
+  branches:
+    include:
+    - main
+  always: true  # Run even if there are no code changes
+
+jobs:
+- template: test-job-template.yml
+  parameters:
+    jobName: EndToEndTests
+    jobDisplayName: "Run end-to-end scenario tests"
+    testAzureSubscription: 'integration-test-service-connection'
+    newDir: e2e_test_directory
+    testsFolder: end_to_end
+    makeTarget: end-to-end-test

.azuredevops/end-to-end-tests.yml

@@ -0,0 +1,19 @@
+
+# Builds the pyrit environment and runs integration tests
+
+trigger:
+  branches:
+    include:
+      - main
+
+# There are additional PR triggers for this that are configurable in ADO.
+
+jobs:
+- template: test-job-template.yml
+  parameters:
+    jobName: IntegrationTests
+    jobDisplayName: "Builds the pyrit environment and runs integration tests"
+    testAzureSubscription: 'integration-test-service-connection'
+    newDir: integration_test_directory
+    testsFolder: integration
+    makeTarget: integration-test

.azuredevops/integration-tests.yml

@@ -0,0 +1,21 @@
+# Builds the pyrit environment and runs partner integration tests. Partner integration tests are to ensure that we
+# are not breaking any contract between PyRIT and its partners.
+trigger: none  # Disable automatic CI triggers
+
+schedules:
+- cron: "0 6 * * *"  # 6 AM UTC = 10 PM PST (UTC-8) / 11PM PDT (UTC-6)
+  displayName: Nightly Partner Integration Tests at 10 PM PST
+  branches:
+    include:
+    - main
+  always: true  # Run even if there are no code changes
+
+jobs:
+- template: test-job-template.yml
+  parameters:
+    jobName: PartnerIntegrationTests
+    jobDisplayName: "Builds the pyrit environment and runs partner integration tests"
+    testAzureSubscription: 'integration-test-service-connection'
+    newDir: partner_integration_test_directory
+    testsFolder: partner_integration
+    makeTarget: partner-integration-test

.azuredevops/partner-integration-tests.yml

@@ -1,17 +1,21 @@
-
-# Builds the pyrit environment and runs integration tests
-
-trigger:
-  branches:
-    include:
-      - main
-
-# There are additional PR triggers for this that are configurable in ADO.
+parameters:
+  - name: jobName
+    type: string
+  - name: jobDisplayName
+    type: string
+  - name: testAzureSubscription
+    type: string
+  - name: newDir
+    type: string
+  - name: testsFolder
+    type: string
+  - name: makeTarget
+    type: string
 
 jobs:
-- job: IntegrationTests
-  displayName: "Builds the pyrit environment and runs integration tests"
-  timeoutInMinutes: 360   # Allows the job to run up to 6 hours
+- job: ${{ parameters.jobName }}
+  displayName: ${{ parameters.jobDisplayName }}
+  timeoutInMinutes: 360
   pool:
     vmImage: ubuntu-latest
   steps:
@@ -79,21 +83,21 @@ jobs:
     name: install_PyRIT
   - bash: df -all -h
     name: disk_space_check
-# This step ensures that integration tests are run outside of the PyRIT repository to test that .env files are accessed correctly.
+# This step ensures that tests are run outside of the PyRIT repository to test that .env files are accessed correctly.
   - bash: |
       PyRIT_DIR=$(pwd)
-      NEW_DIR="integration_test_directory"
+      NEW_DIR="${{ parameters.newDir }}"
       cd ..
       mkdir -p $NEW_DIR/tests
       cp -r $PyRIT_DIR/doc $NEW_DIR
       cp -r $PyRIT_DIR/assets $NEW_DIR
-      cp -r $PyRIT_DIR/tests/integration $NEW_DIR/tests
+      cp -r $PyRIT_DIR/tests/${{ parameters.testsFolder }} $NEW_DIR/tests
       cd $NEW_DIR
-    displayName: "Create and switch to new integration test directory"
+    displayName: "Create and switch to new test directory"
   - task: AzureCLI@2
     displayName: "Authenticate with service principal, cache Cognitive Services access token, and run tests"
     inputs:
-      azureSubscription: 'integration-test-service-connection'
+      azureSubscription: ${{ parameters.testAzureSubscription }}
       scriptType: 'bash'
       scriptLocation: 'inlineScript'
       inlineScript: |
@@ -109,8 +113,8 @@ jobs:
         az account get-access-token --scope https://database.windows.net/.default --output none
         echo "Azure SQL Database access token cached successfully."
 
-        # Run integration tests
-        make integration-test
+        # Run tests
+        make ${{ parameters.makeTarget }}
   - bash: |
       rm -f ~/.pyrit/.env ~/.pyrit/.env.local
     name: clean_up_env_files

integration-tests.yml .azuredevops/test-job-template.ymlintegration-tests.yml renamed to .azuredevops/test-job-template.yml

@@ -1,7 +1,9 @@
-FROM --platform=linux/amd64 mcr.microsoft.com/devcontainers/python:3.11
+# syntax=docker/dockerfile:1
+FROM mcr.microsoft.com/devcontainers/python:3.11-bookworm
 
 # Makes installation faster
 ENV UV_COMPILE_BYTECODE=1
+ENV DEBIAN_FRONTEND=noninteractive
 
 SHELL ["/bin/bash", "-c"]
 
@@ -11,66 +13,63 @@ USER root
 RUN rm -f /etc/apt/sources.list.d/yarn.list 2>/dev/null || true
 
 # Install required system packages + ODBC prerequisites
-RUN apt-get update && apt-get install -y \
-    sudo \
-    unixodbc \
-    unixodbc-dev \
-    libgl1 \
-    git \
-    curl \
-    xdg-utils \
-    build-essential \
- && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    apt-get update \
+ && apt-get install -y --no-install-recommends \
+      sudo \
+      unixodbc \
+      unixodbc-dev \
+      libgl1 \
+      git \
+      curl \
+      xdg-utils \
+      build-essential
 
-# Install the Azure CLI, Microsoft ODBC Driver 18 & SQL tools
+# Install Microsoft ODBC Driver 18 & SQL tools
 # Note: Debian Trixie's sqv rejects SHA1 signatures, so we use gpg directly to import the Microsoft key
-RUN apt-get update && apt-get install -y \
-      apt-transport-https \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    apt-get update \
+ && apt-get install -y --no-install-recommends \
       ca-certificates \
       gnupg \
-      lsb-release \
  && curl -sL https://packages.microsoft.com/keys/microsoft.asc \
       | gpg --dearmor \
       > /usr/share/keyrings/microsoft-archive-keyring.gpg \
- && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" \
+ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/debian/12/prod bookworm main" \
       > /etc/apt/sources.list.d/microsoft.list \
  && apt-get update \
- && ACCEPT_EULA=Y apt-get install -y \
+ && ACCEPT_EULA=Y apt-get install -y --no-install-recommends \
       msodbcsql18 \
-      mssql-tools \
-      unixodbc-dev \
- && apt-get install -y azure-cli \
- && echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> /etc/profile.d/sqltools.sh \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
+      mssql-tools18 \
+ && echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> /etc/profile.d/sqltools.sh
 
 # audio back-ends needed by Azure Speech SDK
-RUN apt-get update \
- && DEBIAN_FRONTEND=noninteractive \
- apt-get install -y --no-install-recommends \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    apt-get update \
+ && apt-get install -y --no-install-recommends \
       libasound2 \
-      libpulse0 \
- && rm -rf /var/lib/apt/lists/*
+      libpulse0
 
 # Install uv system-wide and create pyrit-dev venv
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh \
- && mv /root/.local/bin/uv /usr/local/bin/uv \
- && rm -rf /opt/venv \
- && uv venv /opt/venv --python 3.11 --prompt pyrit-dev \
- && chown -R vscode:vscode /opt/venv \
- && ls -la /opt/venv/bin/activate
+RUN curl -LsSf https://astral.sh/uv/0.10.8/install.sh | sh \
+ && mv /root/.local/bin/uv /bin/uv \
+ && mv /root/.local/bin/uvx /bin/uvx
+RUN uv venv /opt/venv --python 3.11 --prompt pyrit-dev \
+ && chown -R vscode:vscode /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
+# Install Node.js 24.x LTS for frontend development
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
+    curl -fsSL https://deb.nodesource.com/setup_24.x | bash - \
+ && apt-get install -y --no-install-recommends nodejs
+
 # vscode user already exists in the base image, just ensure sudo access
 RUN echo "vscode ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 
-# Install Node.js 20.x and npm for frontend development
-RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \
- && apt-get install -y nodejs \
- && npm install -g npm@latest \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
 # Pre-create common user caches and fix permissions
 RUN mkdir -p /home/vscode/.cache/pre-commit \
  && mkdir -p /home/vscode/.vscode-server \
@@ -79,7 +78,7 @@ RUN mkdir -p /home/vscode/.cache/pre-commit \
  && mkdir -p /home/vscode/.cache/venv \
  && mkdir -p /home/vscode/.cache/pylance \
  && chown -R vscode:vscode /home/vscode/.cache /home/vscode/.vscode-server \
- && chmod -R 777 /home/vscode/.cache/pip /home/vscode/.cache/pylance /home/vscode/.cache/venv /home/vscode/.cache/uv\
+ && chmod -R 755 /home/vscode/.cache/pip /home/vscode/.cache/pylance /home/vscode/.cache/venv /home/vscode/.cache/uv \
  && chmod -R 755 /home/vscode/.vscode-server
 
 USER vscode
@@ -95,6 +94,6 @@ RUN git config --global core.preloadindex true \
  && git config --global status.showUntrackedFiles all \
  && git config --global core.fsmonitor true
 
- # Set cache directories so they can be mounted
+# Set cache directories so they can be mounted
 ENV PIP_CACHE_DIR="/home/vscode/.cache/pip"
 ENV UV_CACHE_DIR="/home/vscode/.cache/uv"

.devcontainer/Dockerfile

@@ -84,10 +84,20 @@
         "dbaeumer.vscode-eslint",
         "esbenp.prettier-vscode",
         "ms-playwright.playwright",
-        "orta.vscode-jest"
+        "orta.vscode-jest",
+        "github.copilot",
+        "github.copilot-chat"
       ]
     }
   },
+  "features": {
+    "ghcr.io/devcontainers/features/azure-cli:1": {
+      "version": "latest"
+    },
+    "ghcr.io/devcontainers/features/copilot-cli:1": {
+      "version": "latest"
+    }
+  },
   "postCreateCommand": "/bin/bash -i .devcontainer/devcontainer_setup.sh",
   "forwardPorts": [3000, 4213, 5000, 8000, 8888]
 }

.devcontainer/devcontainer.json

@@ -8,7 +8,7 @@ if [ ! -d "$MYPY_CACHE" ]; then
     echo "Creating mypy cache directory..."
     sudo mkdir -p $MYPY_CACHE
     sudo chown vscode:vscode $MYPY_CACHE
-    sudo chmod 777 $MYPY_CACHE
+    sudo chmod 755 $MYPY_CACHE
 else
     # Check ownership
     OWNER=$(stat -c '%U:%G' $MYPY_CACHE)
@@ -21,9 +21,9 @@ else
     # Check permissions
     PERMS=$(stat -c '%a' $MYPY_CACHE)
 
-    if [ "$PERMS" != "777" ]; then
+    if [ "$PERMS" != "755" ]; then
         echo "Fixing mypy cache directory permissions..."
-        sudo chmod -R 777 $MYPY_CACHE
+        sudo chmod -R 755 $MYPY_CACHE
     fi
 fi
 
@@ -70,6 +70,7 @@ if [ -f "package.json" ]; then
     npm install
 
     # Install Playwright browsers and system dependencies for E2E testing
+    # This may fail if apt repos have signature issues - don't block setup
     echo "📦 Installing Playwright browsers..."
 
     # Remove third-party repos with SHA1 signature issues (rejected since 2026-02-01)
@@ -78,7 +79,11 @@ if [ -f "package.json" ]; then
                /etc/apt/sources.list.d/nodesource.list \
                /etc/apt/sources.list.d/microsoft.list 2>/dev/null || true
 
-    npx playwright install --with-deps chromium
+    if npx playwright install --with-deps chromium; then
+        echo "✅ Playwright browsers installed."
+    else
+        echo "⚠️  Playwright installation failed (apt signature issues). Run 'npx playwright install chromium' manually if needed for E2E tests."
+    fi
 
     echo "✅ Frontend dependencies installed."
 fi

.devcontainer/devcontainer_setup.sh

@@ -1,6 +1,5 @@
 services:
   devcontainer:
-    platform: linux/amd64
     build:
       context: ..
       dockerfile: .devcontainer/Dockerfile