diff --git a/scenarios/evaluate/AI_RedTeaming/.env.sample b/scenarios/evaluate/AI_RedTeaming/.env.sample new file mode 100644 index 00000000..8dbcf570 --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/.env.sample @@ -0,0 +1,28 @@ +# Azure AI Red Teaming Environment Variables +# Copy this file to .env and populate with your deployment outputs + +# Azure AI Project endpoint (from AZURE_AI_PROJECT output) +AZURE_AI_PROJECT="https://your-foundry-name.services.ai.azure.com/api/projects/your-project-name" + +# Azure OpenAI Configuration (from AZURE_OPENAI_CONFIG output) +AZURE_OPENAI_CONFIG='{"endpoint": "https://your-foundry-name.cognitiveservices.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21", "api_key": "your-api-key", "deployment": "gpt-4o", "api_version": "2024-10-21"}' + +# Storage Account (from AZURE_STORAGE_ACCOUNT output) +AZURE_STORAGE_ACCOUNT="your-storage-account-name" + +# Additional deployment outputs (for reference) +AZURE_SUBSCRIPTION_ID="12345678-1234-1234-1234-123456789012" +AZURE_RESOURCE_GROUP="rg-your-resource-group" +AI_PROJECT_NAME="your-project-name" +AI_FOUNDRY_ENDPOINT="https://your-foundry-name.services.ai.azure.com" +AZURE_OPENAI_ENDPOINT="https://your-foundry-name.cognitiveservices.azure.com" +AZURE_OPENAI_API_KEY="your-api-key" +AZURE_STORAGE_ACCOUNT_NAME="your-storage-account-name" + +# Note: The main variables needed for RedTeam are: +# - azure_ai_project (for the project endpoint) +# - azure_openai_config (for the model configuration) +# +# These values are automatically generated by the deployment script. +# Run: .\infra\deploy.ps1 +# Then copy the output values to this file. diff --git a/scenarios/evaluate/AI_RedTeaming/AI_RedTeaming.ipynb b/scenarios/evaluate/AI_RedTeaming/AI_RedTeaming.ipynb index 85bbd7c7..102a0780 100644 --- a/scenarios/evaluate/AI_RedTeaming/AI_RedTeaming.ipynb +++ b/scenarios/evaluate/AI_RedTeaming/AI_RedTeaming.ipynb @@ -15,11 +15,21 @@ "## Before you begin\n", "\n", "### Prerequisite\n", - "First, if you have an Azure subscription, create an [Azure AI hub](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/ai-resources) then [create an Azure AI project](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/ai-resources). AI projects and Hubs can be served within a private network and are compatible with private endpoints. You **do not** need to provide your own LLM deployment as the AI Red Teaming Agent hosts adversarial models for both simulation and evaluation of harmful content and connects to it via your Azure AI project.\n", + "First, deploy the Azure AI Red Teaming infrastructure using the deployment script in the `infra/` folder:\n", "\n", - "**Note**: In order to upload your results to Azure AI Foundry, you must have the `Storage Blob Data Contributor` role\n", + "```bash\n", + "# Navigate to the AI_RedTeaming directory\n", + "cd scenarios/evaluate/AI_RedTeaming\n", + "\n", + "# Run the deployment script\n", + ".\\infra\\deploy.ps1\n", + "```\n", "\n", - "**Important**: First, ensure that you've installed the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) and then make sure to authenticate to Azure using `az login` in your terminal before running this notebook.\n", + "This will create an Azure AI Foundry hub and project with all necessary resources including GPT-4o model deployment. The deployment script will output environment variables that you need to copy to a `.env` file.\n", + "\n", + "**Note**: In order to upload your results to Azure AI Foundry, you must have the `Storage Blob Data Contributor` role (automatically assigned by the deployment script).\n", + "\n", + "**Important**: Ensure that you've installed the [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) and authenticated using `az login` before running the deployment.\n", "\n", "### Installation\n", "From a terminal window, navigate to your working directory which contains this sample notebook, and execute the following.\n", @@ -30,7 +40,7 @@ "Then, activate the virtual environment created:\n", "\n", "```bash\n", - "# %source .venv/bin/activate # If using Mac/Linux OS\n", + "# source .venv/bin/activate # If using Mac/Linux OS\n", ".venv/Scripts/activate # If using Windows OS\n", "```\n", "\n", @@ -38,9 +48,11 @@ "\n", "```bash\n", "pip install uv\n", - "uv pip install azure-ai-evaluation[redteam] azure-identity openai\n", + "uv pip install azure-ai-evaluation[redteam] azure-identity openai python-dotenv\n", "```\n", "\n", + "### Environment Setup\n", + "After deployment, create a `.env` file in this directory and copy the environment variables from the deployment output. A sample file is provided at `.env.sample`.\n", "\n", "Now open VSCode with the following command, and ensure your virtual environment is used as kernel to run the remainder of this notebook.\n", "```bash\n", @@ -64,6 +76,11 @@ "from typing import Optional, Dict, Any\n", "import os\n", "\n", + "# Load environment variables from .env file\n", + "from dotenv import load_dotenv\n", + "\n", + "load_dotenv()\n", + "\n", "# Azure imports\n", "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", "from azure.ai.evaluation.red_team import RedTeam, RiskCategory, AttackStrategy\n", @@ -93,17 +110,23 @@ "For reference, here's an example of what your populated environment variables should look like:\n", "\n", "```\n", - "# Azure OpenAI\n", - "AZURE_OPENAI_API_KEY=\"your-api-key-here\"\n", - "AZURE_OPENAI_ENDPOINT=\"https://endpoint-name.openai.azure.com/openai/deployments/deployment-name/chat/completions\"\n", - "AZURE_OPENAI_DEPLOYMENT_NAME=\"gpt-4\"\n", - "AZURE_OPENAI_API_VERSION=\"2024-12-01-preview\"\n", + "# Azure AI Project (from deployment output)\n", + "AZURE_AI_PROJECT=\"https://your-foundry-name.services.ai.azure.com/api/projects/your-project-name\"\n", + "\n", + "# Azure OpenAI Configuration (from deployment output)\n", + "AZURE_OPENAI_CONFIG='{\"endpoint\": \"https://your-foundry-name.cognitiveservices.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2024-10-21\", \"api_key\": \"your-api-key\", \"deployment\": \"gpt-4o\", \"api_version\": \"2024-10-21\"}'\n", + "\n", + "# Storage Account (from deployment output)\n", + "AZURE_STORAGE_ACCOUNT=\"your-storage-account-name\"\n", "\n", - "# Azure AI Project\n", + "# Additional deployment outputs\n", "AZURE_SUBSCRIPTION_ID=\"12345678-1234-1234-1234-123456789012\"\n", - "AZURE_RESOURCE_GROUP_NAME=\"your-resource-group\"\n", - "AZURE_PROJECT_NAME=\"your-project-name\"\n", - "```" + "AZURE_RESOURCE_GROUP=\"rg-your-resource-group\"\n", + "AI_PROJECT_NAME=\"your-project-name\"\n", + "AI_FOUNDRY_ENDPOINT=\"https://your-foundry-name.services.ai.azure.com\"\n", + "```\n", + "\n", + "**Note**: These values are automatically generated by the deployment script and should be set in your `.env` file." ] }, { @@ -112,20 +135,37 @@ "metadata": {}, "outputs": [], "source": [ - "# Azure AI Project information\n", - "azure_ai_project = {\n", - " \"subscription_id\": os.environ.get(\"AZURE_SUBSCRIPTION_ID\"),\n", - " \"resource_group_name\": os.environ.get(\"AZURE_RESOURCE_GROUP_NAME\"),\n", - " \"project_name\": os.environ.get(\"AZURE_PROJECT_NAME\"),\n", - "}\n", - "\n", - "# Azure OpenAI deployment information\n", - "azure_openai_deployment = os.environ.get(\"AZURE_OPENAI_DEPLOYMENT\") # e.g., \"gpt-4\"\n", - "azure_openai_endpoint = os.environ.get(\n", - " \"AZURE_OPENAI_ENDPOINT\"\n", - ") # e.g., \"https://endpoint-name.openai.azure.com/openai/deployments/deployment-name/chat/completions\"\n", - "azure_openai_api_key = os.environ.get(\"AZURE_OPENAI_API_KEY\") # e.g., \"your-api-key\"\n", - "azure_openai_api_version = os.environ.get(\"AZURE_OPENAI_API_VERSION\") # Use the latest API version" + "import json\n", + "\n", + "# Azure AI Project endpoint (from deployment output)\n", + "azure_ai_project = os.environ.get(\"AZURE_AI_PROJECT\")\n", + "\n", + "# Parse Azure OpenAI configuration from deployment output\n", + "azure_openai_config_str = os.environ.get(\"AZURE_OPENAI_CONFIG\")\n", + "if azure_openai_config_str:\n", + " azure_openai_config = json.loads(azure_openai_config_str)\n", + " azure_openai_endpoint = azure_openai_config[\"endpoint\"]\n", + " azure_openai_deployment = azure_openai_config[\"deployment\"]\n", + " azure_openai_api_key = azure_openai_config[\"api_key\"]\n", + " azure_openai_api_version = azure_openai_config[\"api_version\"]\n", + "else:\n", + " # Fallback to individual environment variables if needed\n", + " azure_openai_endpoint = os.environ.get(\"AZURE_OPENAI_ENDPOINT\")\n", + " azure_openai_deployment = \"gpt-4o\" # Default deployment name\n", + " azure_openai_api_key = os.environ.get(\"AZURE_OPENAI_API_KEY\")\n", + " azure_openai_api_version = \"2024-10-21\"\n", + "\n", + " # Create config dict for RedTeam\n", + " azure_openai_config = {\n", + " \"azure_endpoint\": azure_openai_endpoint,\n", + " \"azure_deployment\": azure_openai_deployment,\n", + " \"api_key\": azure_openai_api_key,\n", + " \"api_version\": azure_openai_api_version,\n", + " }\n", + "\n", + "print(f\"Azure AI Project: {azure_ai_project}\")\n", + "print(f\"Azure OpenAI Deployment: {azure_openai_deployment}\")\n", + "print(f\"Azure OpenAI Endpoint: {azure_openai_endpoint}\")" ] }, { @@ -203,7 +243,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Create the `RedTeam` instance with minimal configurations\n", + "# Create the `RedTeam` instance with the deployed Azure AI Foundry project\n", "red_team = RedTeam(\n", " azure_ai_project=azure_ai_project,\n", " credential=credential,\n", @@ -257,12 +297,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Define a model configuration to test\n", - "azure_oai_model_config = {\n", - " \"azure_endpoint\": azure_openai_endpoint,\n", - " \"azure_deployment\": azure_openai_deployment,\n", - " \"api_key\": azure_openai_api_key,\n", - "}" + "# Use the model configuration from deployment output\n", + "# This configuration is automatically loaded from the azure_openai_config environment variable\n", + "print(f\"Using Azure OpenAI configuration: {azure_openai_config}\")" ] }, { @@ -278,9 +315,9 @@ "metadata": {}, "outputs": [], "source": [ - "# Run the red team scan called \"Intermediary-Model-Target-Scan\"\n", + "# Run the red team scan called \"Intermediary-Model-Target-Scan\" using the deployed model\n", "result = await red_team.scan(\n", - " target=azure_oai_model_config, scan_name=\"Intermediary-Model-Target-Scan\", attack_strategies=[AttackStrategy.Flip]\n", + " target=azure_openai_config, scan_name=\"Intermediary-Model-Target-Scan\", attack_strategies=[AttackStrategy.Flip]\n", ")" ] }, @@ -346,7 +383,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Create the RedTeam instance with all of the risk categories with 5 attack objectives generated for each category\n", + "# Create the RedTeam instance with all of the risk categories using the deployed Azure AI Foundry project\n", "model_red_team = RedTeam(\n", " azure_ai_project=azure_ai_project,\n", " credential=credential,\n", @@ -420,9 +457,10 @@ "metadata": {}, "outputs": [], "source": [ - "path_to_prompts = \".\\data\\prompts.json\"\n", + "path_to_prompts = \"./data/prompts.json\"\n", "\n", "# Create the RedTeam specifying the custom attack seed prompts to use as objectives\n", + "# Using the deployed Azure AI Foundry project\n", "custom_red_team = RedTeam(\n", " azure_ai_project=azure_ai_project,\n", " credential=credential,\n", diff --git a/scenarios/evaluate/AI_RedTeaming/azure.yaml b/scenarios/evaluate/AI_RedTeaming/azure.yaml new file mode 100644 index 00000000..491af904 --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/azure.yaml @@ -0,0 +1,45 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: ai-red-teaming +metadata: + template: ai-red-teaming@0.0.1-beta +summary: Azure AI Red Teaming Infrastructure +description: | + Deploy the required Azure infrastructure for AI Red Teaming scenarios including Azure AI Foundry, + OpenAI services, storage, and proper RBAC configuration for authentication. +author: Azure AI Evaluation Team +repositoryPath: . + +workflows: + up: + steps: + - azd: provision + - azd: deploy + +infra: + provider: bicep + path: infra + +pipeline: + provider: azdo + path: .azdo + +services: {} + +hooks: + postprovision: + shell: pwsh + run: | + # Output the environment variables for easy copy-paste + Write-Host "=== Azure AI Red Teaming Environment Variables ===" -ForegroundColor Green + Write-Host "Copy these environment variables to your .env file or notebook:" -ForegroundColor Yellow + Write-Host "" + Write-Host "AZURE_SUBSCRIPTION_ID=$env:AZURE_SUBSCRIPTION_ID" -ForegroundColor Cyan + Write-Host "AZURE_RESOURCE_GROUP=$env:AZURE_RESOURCE_GROUP" -ForegroundColor Cyan + Write-Host "AZURE_AI_PROJECT_NAME=$env:AZURE_AI_PROJECT_NAME" -ForegroundColor Cyan + Write-Host "AZURE_AI_PROJECT_CONNECTION_STRING=$env:AZURE_AI_PROJECT_CONNECTION_STRING" -ForegroundColor Cyan + Write-Host "AZURE_OPENAI_ENDPOINT=$env:AZURE_OPENAI_ENDPOINT" -ForegroundColor Cyan + Write-Host "AZURE_STORAGE_ACCOUNT_NAME=$env:AZURE_STORAGE_ACCOUNT_NAME" -ForegroundColor Cyan + Write-Host "" + Write-Host "Authentication is configured to use DefaultAzureCredential." -ForegroundColor Green + Write-Host "Make sure you're logged in with 'az login' or 'azd auth login'." -ForegroundColor Yellow diff --git a/scenarios/evaluate/AI_RedTeaming/infra/.env.sample b/scenarios/evaluate/AI_RedTeaming/infra/.env.sample new file mode 100644 index 00000000..8511c273 --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/.env.sample @@ -0,0 +1,36 @@ +# Azure AI Red Teaming Environment Variables +# Copy this file to .env and update with your actual values after deployment + +# Azure Subscription and Resource Group +AZURE_SUBSCRIPTION_ID=your-subscription-id +AZURE_TENANT_ID=your-tenant-id +AZURE_RESOURCE_GROUP=your-resource-group-name +AZURE_LOCATION=eastus + +# Azure AI Foundry +AZURE_AI_HUB_NAME=your-ai-hub-name +AZURE_AI_PROJECT_NAME=your-ai-project-name +AZURE_AI_PROJECT_CONNECTION_STRING=azureml://subscriptions/your-sub-id/resourcegroups/your-rg/providers/Microsoft.MachineLearningServices/workspaces/your-project-name + +# Azure OpenAI (with latest models) +AZURE_OPENAI_ENDPOINT=https://your-openai-account.openai.azure.com/ +AZURE_OPENAI_API_KEY=your-openai-api-key +# Available models: gpt-4o, gpt-4o-mini, o1-preview + +# Azure Storage +AZURE_STORAGE_ACCOUNT_NAME=your-storage-account-name +AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=... + +# Azure AI Search (optional) +AZURE_SEARCH_SERVICE_NAME=your-search-service-name +AZURE_SEARCH_ENDPOINT=https://your-search-service.search.windows.net/ +AZURE_SEARCH_API_KEY=your-search-api-key + +# Key Vault (optional) +AZURE_KEY_VAULT_NAME=your-key-vault-name + +# Authentication - DefaultAzureCredential will be used +# Make sure you're logged in with one of: +# - az login (Azure CLI) +# - azd auth login (Azure Developer CLI) +# - Connect-AzAccount (Azure PowerShell) \ No newline at end of file diff --git a/scenarios/evaluate/AI_RedTeaming/infra/DEPLOYMENT_GUIDE.md b/scenarios/evaluate/AI_RedTeaming/infra/DEPLOYMENT_GUIDE.md new file mode 100644 index 00000000..be622e4c --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/DEPLOYMENT_GUIDE.md @@ -0,0 +1,433 @@ +# Azure AI Red Teaming Infrastructure - Complete Deployment Guide + +## ๐ŸŽฏ Overview + +This Azure Developer CLI (azd) template provides a complete infrastructure solution for AI Red Teaming scenarios using the Azure AI Evaluation SDK. It deploys a simplified but complete Azure AI Foundry setup with proper authentication and RBAC permissions. + +## ๐Ÿ—๏ธ What Gets Deployed + +### Core Azure Resources +- **Azure AI Foundry Account** - AI services account with project management capabilities +- **Azure AI Foundry Project** - Dedicated workspace for red teaming activities +- **OpenAI Model Deployment** - Single GPT-4o model deployment for cost efficiency +- **Azure Storage Account** - Secure storage for artifacts, logs, and evaluation data +- **Azure AI Search** - Enhanced search capabilities for data retrieval +- **Azure Key Vault** - Secure secret and credential management +- **Log Analytics & Application Insights** - Comprehensive monitoring and logging + +### Authentication & Security +- **Managed Identity Configuration** - Seamless service-to-service authentication +- **Role-Based Access Control (RBAC)** - Least-privilege access to all resources +- **DefaultAzureCredential Support** - Multiple authentication methods supported +- **Optional Permission Assignment** - User permissions (disabled by default for deployment reliability) + +## ๐Ÿš€ Quick Start (3 minutes) + +### Option 1: Automated Deployment Script (Recommended) +```powershell +# Navigate to the AI_RedTeaming directory (not infra subdirectory) +cd scenarios/evaluate/AI_RedTeaming + +# Run the deployment script +.\infra\deploy.ps1 +``` + +### Option 2: Manual Azure Developer CLI +```bash +# Navigate to the AI_RedTeaming directory (not infra subdirectory) +cd scenarios/evaluate/AI_RedTeaming + +# Login to Azure +azd auth login + +# Deploy infrastructure +azd up +``` + +**Important**: Run azd commands from the `AI_RedTeaming` directory, not the `infra` subdirectory, since the `azure.yaml` file is located there. + +### Option 3: GitHub Actions (CI/CD) +1. Fork this repository +2. Set up GitHub repository secrets (see CI/CD section below) +3. Go to Actions โ†’ "Deploy AI Red Teaming Infrastructure" โ†’ Run workflow + +## ๐Ÿ“‹ Prerequisites + +### Required Tools +- [Azure Developer CLI (azd)](https://learn.microsoft.com/azure/developer/azure-developer-cli/install-azd) - Version 1.5.0 or later +- [Azure CLI](https://docs.microsoft.com/cli/azure/install-azure-cli) - Version 2.50.0 or later (optional, for advanced scenarios) +- PowerShell 7+ (for deployment script) + +### Azure Requirements +- Azure subscription with **Contributor** permissions +- Ability to create resources in your chosen region +- Sufficient quota for OpenAI services (GPT-4o model) + +### Required Permissions for Red Team Operations +The deployment will assign the following roles to enable red team operations: +- **Cognitive Services Contributor** - Required for `Microsoft.CognitiveServices/accounts/AIServices/evaluations/write` permission +- **Cognitive Services OpenAI User** - For accessing OpenAI models +- **Storage Blob Data Contributor** - For managing evaluation data +- **Key Vault Secrets User** - For accessing stored credentials + +These permissions are automatically assigned when `createRoleAssignments` is enabled during deployment. + +**Important**: If you encounter a `ClientAuthenticationError` with message about lacking `Microsoft.CognitiveServices/accounts/AIServices/evaluations/write` permission, re-run the deployment with role assignments enabled: + +```bash +# Re-deploy with role assignments +.\infra\deploy.ps1 -PrincipalId $(az ad signed-in-user show --query "id" -o tsv) +``` + +### Verifying Your Permissions +Before deployment, verify you have the required permissions: + +#### Method 1: Using Azure Portal +1. Go to [Azure Portal](https://portal.azure.com) +2. Navigate to **Subscriptions** โ†’ Select your subscription +3. Click **Access control (IAM)** โ†’ **Role assignments** +4. Search for your email to verify you have **Contributor** role + +#### Method 2: Using Azure CLI +```bash +# Check current subscription and account +az account show + +# List your role assignments +az role assignment list --assignee $(az account show --query user.name -o tsv) --scope "/subscriptions/$(az account show --query id -o tsv)" --query "[].{Role:roleDefinitionName, Scope:scope}" -o table + +# Switch subscription if needed +az account set --subscription +``` + +#### Method 3: Test Deployment (Recommended) +The simplest way is to try the deployment - it will fail quickly if you lack permissions: +```bash +# Navigate to AI_RedTeaming directory +cd scenarios/evaluate/AI_RedTeaming + +# Try deployment - will fail fast if permissions are insufficient +azd up +``` + +### Python Environment (for testing) +```bash +pip install azure-ai-evaluation azure-ai-projects azure-storage-blob openai azure-identity +``` + +## โš™๏ธ Configuration Options + +### Environment Variables +Set these before deployment to customize the infrastructure: + +```bash +# Required +export AZURE_ENV_NAME="" # Insert your environment name +export AZURE_LOCATION="" # Insert your Azure region + +# Optional (role assignments disabled by default for deployment reliability) +export AZURE_PRINCIPAL_ID="" # Insert your principal ID to enable user permission assignments +``` + +### Supported Azure Regions +Primary regions with GPT-4o model availability: +- `eastus2` (default) - Good performance and model availability +- `eastus` - Best overall model availability +- `northcentralus` - Central US option +- `westus2` - West coast option +- `westus3` - Latest west coast region +- `westeurope` - European option +- `uksouth` - UK option +- `australiaeast` - Asia-Pacific option +- `japaneast` - Asia option + +### Model Deployment +The template deploys a single model by default for cost efficiency: +- **GPT-4o** (gpt-4o) - 1 TPM capacity - Latest multimodal model + +To customize capacity or add models, edit `main.bicepparam`: +```bicep +# Current default configuration (cost-optimized) +param modelDeployments = [ + { + name: 'gpt-4o-mini' + model: { format: 'OpenAI', name: 'gpt-4o-mini', version: '2024-07-18' } + sku: { name: 'GlobalStandard', capacity: 5 } + } +] +``` + +## ๐Ÿ” Authentication Setup + +### For Local Development +After deployment, authenticate using any of these methods: + +```bash +# Option 1: Azure CLI (recommended) +az login + +# Option 2: Azure Developer CLI +azd auth login + +# Option 3: Azure PowerShell +Connect-AzAccount +``` + +### For Python Code +The infrastructure is configured to work with `DefaultAzureCredential`: + +```python +from azure.identity import DefaultAzureCredential +from azure.ai.projects import AIProjectClient +import os + +# This will automatically use your logged-in credentials +credential = DefaultAzureCredential() +project_client = AIProjectClient.from_connection_string( + conn_str=os.environ["AZURE_AI_PROJECT_CONNECTION_STRING"], + credential=credential +) +``` + +## ๐Ÿงช Testing Your Deployment + +### Automated Testing +Run the included test script to verify everything works: + +```bash +# Navigate to the infra directory +cd infra + +# Set environment variables from deployment output +# (These will be displayed after successful deployment) +export AZURE_AI_PROJECT_CONNECTION_STRING="" +export AZURE_OPENAI_ENDPOINT="" +export AZURE_STORAGE_ACCOUNT_NAME="" + +# Run tests +python test_deployment.py +``` + +### Manual Verification +Check these components individually: + +1. **Azure AI Project**: Visit [AI Studio](https://ai.azure.com) and verify your project appears +2. **OpenAI Models**: Test in Azure OpenAI Studio playground +3. **Storage**: Check containers in Azure Storage Explorer +4. **Permissions**: Run `az role assignment list --scope /subscriptions/...` to verify RBAC + +## ๐Ÿ”ง Troubleshooting + +### Common Issues + +#### Authentication Errors +``` +DefaultAzureCredential failed to retrieve a token +``` +**Solution**: +```bash +# Clear any cached credentials and re-login +az account clear +az login +azd auth login +``` + +#### Resource Naming Conflicts +``` +The storage account name is already taken +``` +**Solution**: Change environment name to generate new unique names: +```bash +azd up --set environmentName="my-unique-name-$(date +%s)" +``` + +#### Quota Exceeded +``` +Deployment failed: Insufficient quota for OpenAI +``` +**Solution**: +1. Check quota in Azure Portal โ†’ Cognitive Services โ†’ Quotas +2. Request quota increase or try different region +3. Reduce model capacity in `main.bicepparam` + +#### Permission Denied +``` +The client does not have authorization to perform action +``` +**Solution**: +1. Verify you have **Contributor** role on subscription +2. Check subscription with: `az account show` +3. Switch subscription with: `az account set --subscription ` +4. List available subscriptions: `az account list --output table` + +#### Role Assignment Issues +``` +Role assignments failed during deployment +``` +**Solution**: +The template disables role assignments by default for deployment reliability. This is normal and doesn't affect functionality since Azure AI services use managed identities for authentication. + +### Getting Help +1. Check deployment logs: `azd show --environment ` +2. Review Azure Portal โ†’ Resource Groups โ†’ Activity Log +3. Validate Bicep templates: `az deployment group validate` + +## ๐Ÿข CI/CD with GitHub Actions + +The repository includes GitHub Actions workflow for automated deployment. + +### Setup Steps +1. **Create Service Principal**: + ```bash + az ad sp create-for-rbac --name "ai-red-team-deploy" \ + --role contributor \ + --scopes /subscriptions/ \ + --sdk-auth + ``` + +2. **Set Repository Secrets**: + - `AZURE_CREDENTIALS`: Full JSON output from step 1 + - `AZURE_SUBSCRIPTION_ID`: Your subscription ID + - `AZURE_TENANT_ID`: Your tenant ID + +### Workflow Usage +The GitHub Actions workflow is located in `.github/workflows/` and can be triggered manually or on push to main branch. + +## ๐Ÿ“ File Structure + +``` +AI_RedTeaming/ +โ”œโ”€โ”€ azure.yaml # Azure Developer CLI configuration +โ”œโ”€โ”€ AI_RedTeaming.ipynb # Main red teaming notebook +โ”œโ”€โ”€ README.md # Quick start guide +โ”œโ”€โ”€ data/ # Sample data for testing +โ”œโ”€โ”€ .azure/ # azd environment data (created during deployment) +โ””โ”€โ”€ infra/ # Infrastructure as Code + โ”œโ”€โ”€ main.bicep # Main deployment template + โ”œโ”€โ”€ ai-red-team.bicep # Resource definitions + โ”œโ”€โ”€ main.bicepparam # Default parameters + โ”œโ”€โ”€ deploy.ps1 # Automated deployment script + โ”œโ”€โ”€ test_deployment.py # Verification script + โ”œโ”€โ”€ .env.sample # Environment variables template + โ”œโ”€โ”€ README.md # Basic usage instructions + โ”œโ”€โ”€ DEPLOYMENT_GUIDE.md # This comprehensive guide + โ””โ”€โ”€ .github/ + โ””โ”€โ”€ workflows/ + โ””โ”€โ”€ deploy.yml # GitHub Actions workflow +``` + +## ๐Ÿงน Cleanup + +### Remove All Resources +```bash +# Navigate to the AI_RedTeaming directory +cd scenarios/evaluate/AI_RedTeaming + +# Destroy everything +azd down --environment + +# Force removal and purge Key Vault +azd down --environment --force --purge +``` + +### Partial Cleanup +```bash +# Remove specific resource group +az group delete --name --yes --no-wait + +# Purge soft-deleted Key Vault +az keyvault purge --name +``` + +## ๐Ÿ’ก Advanced Customization + +### Adding Custom Resources +Edit `ai-red-team.bicep` to add additional resources: + +```bicep +// Example: Add Azure Cosmos DB +resource cosmosDb 'Microsoft.DocumentDB/databaseAccounts@2023-04-15' = { + name: 'cosmos-${resourceToken}' + location: location + properties: { + databaseAccountOfferType: 'Standard' + locations: [ { locationName: location, failoverPriority: 0 } ] + } +} +``` + +### Custom Model Deployments +Modify the hardcoded model in `ai-red-team.bicep` or add additional deployments: + +```bicep +// Current single model deployment +resource modelDeployment 'Microsoft.CognitiveServices/accounts/deployments@2024-10-01'= { + parent: aiFoundry + name: 'gpt-4o' + sku : { + capacity: 1 // Increase capacity if needed + name: 'GlobalStandard' + } + properties: { + model:{ + name: 'gpt-4o' + format: 'OpenAI' + } + } +} +``` + +### Environment-Specific Configurations +Create multiple `.bicepparam` files for different environments: + +- `dev.bicepparam` - Development settings +- `prod.bicepparam` - Production settings +- `test.bicepparam` - Testing configurations + +### Enabling Role Assignments +To enable automatic role assignments (requires higher permissions): + +1. Edit `main.bicepparam`: +```bicep +param principalId = 'your-user-object-id' +``` + +2. Update the `createRoleAssignments` parameter in `main.bicep`: +```bicep +createRoleAssignments: true +``` + +## ๐ŸŽ‰ Success! What's Next? + +After successful deployment: + +1. **Copy environment variables** from deployment output (automatically displayed) +2. **Run test script** to verify everything works: `python infra/test_deployment.py` +3. **Start your AI Red Teaming** with the `AI_RedTeaming.ipynb` notebook +4. **Explore Azure AI Studio** to manage your project and models at [ai.azure.com](https://ai.azure.com) +5. **Set up monitoring** using Application Insights dashboards + +### Key Resources Created +Your deployment creates these main resources: +- AI Foundry Account: `aifoundry-{token}` +- AI Project: `aiproject-{token}` +- Storage Account: `st{token}` +- Key Vault: `kv-{token}` +- Search Service: `search-{token}` +- Model Deployment: `gpt-4o` (ready to use) + +### Authentication Notes +- The infrastructure uses **managed identities** for service-to-service authentication +- Your user account may need explicit role assignments for full access +- Use `DefaultAzureCredential` in your Python code for seamless authentication + +## ๐Ÿ“š Additional Resources + +- [Azure AI Evaluation SDK Documentation](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/evaluation/azure-ai-evaluation) +- [Azure AI Studio Documentation](https://learn.microsoft.com/azure/ai-studio/) +- [Azure Developer CLI Documentation](https://learn.microsoft.com/azure/developer/azure-developer-cli/) +- [Azure OpenAI Service Documentation](https://learn.microsoft.com/azure/cognitive-services/openai/) + +--- + +**Need Help?** Open an issue in the repository or consult the troubleshooting section above. \ No newline at end of file diff --git a/scenarios/evaluate/AI_RedTeaming/infra/README.md b/scenarios/evaluate/AI_RedTeaming/infra/README.md new file mode 100644 index 00000000..5b0ae502 --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/README.md @@ -0,0 +1,202 @@ +# Azure Developer CLI Template for AI Red Teaming + +This template provides the necessary Azure infrastructure for running AI Red Teaming scenarios using the Azure AI Evaluation SDK. It deploys a simplified Azure AI Foundry setup with proper authentication and permissions configured for red team operations. + +## Prerequisites + +- [Azure Developer CLI (azd)](https://learn.microsoft.com/azure/developer/azure-developer-cli/install-azd) +- [Azure CLI](https://docs.microsoft.com/cli/azure/install-azure-cli) +- PowerShell 7+ (for the deployment script) +- An Azure subscription with Contributor permissions + +## Quick Start + +1. **Clone and navigate to this directory** + ```bash + cd scenarios/evaluate/AI_RedTeaming/ + ``` + +2. **Run the deployment script (recommended)** + ```bash + .\deploy.ps1 + ``` + + The script will: + - Check prerequisites and authenticate with Azure + - Automatically detect your user principal ID for permissions + - Deploy all required Azure resources + - Configure RBAC permissions for red team operations + - Output environment variables for your `.env` file + +3. **Alternative: Manual deployment with azd** + ```bash + # Login to Azure + azd auth login + + # Initialize and deploy + azd init + azd up + ``` + +4. **Set up your environment variables** + - Copy the output from the deployment + - Create a `.env` file in the parent directory (`AI_RedTeaming/.env`) + - The variables follow this format: + ``` + azure_ai_project="https://your-foundry.services.ai.azure.com/api/projects/your-project" + azure_openai_config='{"endpoint": "...", "api_key": "...", "deployment": "gpt-4o", "api_version": "2024-10-21"}' + azure_storage_account="your-storage-account-name" + ``` + +## What Gets Deployed + +This template creates the following Azure resources: + +- **Azure AI Foundry Account** - Simplified AI services account with built-in OpenAI capabilities +- **Azure AI Foundry Project** - Project workspace for red teaming operations +- **GPT-4o Model Deployment** - Single cost-optimized model deployment +- **Azure Storage Account** - For storing evaluation artifacts and data +- **Azure AI Search** - For enhanced search capabilities (basic tier) +- **Azure Key Vault** - For secure secret management +- **Log Analytics & Application Insights** - For monitoring and logging + +## Authentication & Permissions + +The template automatically configures: + +- **Role-Based Access Control (RBAC)** for all services +- **Managed Identity** permissions for Azure AI services +- **User permissions** for your account (automatically detected during deployment) +- **DefaultAzureCredential** support for seamless authentication + +Required roles assigned for red team operations: +- `Cognitive Services Contributor` - **Required for red team evaluations** (provides `Microsoft.CognitiveServices/accounts/AIServices/evaluations/write` permission) +- `Cognitive Services OpenAI User` - For OpenAI model access +- `Storage Blob Data Contributor` - For storage access and evaluation data +- `Search Index Data Contributor` - For search access +- `Key Vault Secrets User` - For Key Vault access + +**Note:** The deployment script automatically detects your user principal ID and assigns the necessary permissions for red team operations. + +## Configuration Options + +You can customize the deployment: + +**Using the deployment script (recommended):** +```bash +# Deploy to a specific location +.\deploy.ps1 -Location "westus2" + +# Use custom environment name +.\deploy.ps1 -EnvironmentName "my-red-team-env" + +# Specify user principal ID manually +.\deploy.ps1 -PrincipalId "your-user-object-id" +``` + +**Using azd directly:** +```bash +# Deploy to a specific location +azd up --set location=westus2 + +# Assign permissions to specific user +azd up --set principalId=your-user-object-id +``` + +**Supported Azure regions with GPT-4o and RedTeam availability:** +- `eastus2` (default) +- `swedencentral` + +## Environment Variables + +After deployment, use these environment variables in your Python code: + +```python +import os +import json +from dotenv import load_dotenv +from azure.ai.evaluation.red_team import RedTeam +from azure.identity import DefaultAzureCredential + +# Load environment variables from .env file +load_dotenv() + +# Initialize credential +credential = DefaultAzureCredential() + +# Get configuration from deployment outputs +azure_ai_project = os.environ.get("azure_ai_project") +azure_openai_config = json.loads(os.environ.get("azure_openai_config")) + +# Initialize RedTeam for evaluation +red_team = RedTeam( + azure_ai_project=azure_ai_project, + credential=credential +) +``` + +## Troubleshooting + +### Red Team Permission Errors + +If you encounter the error `Microsoft.CognitiveServices/accounts/AIServices/evaluations/write`: + +1. **Redeploy with proper permissions:** + ```bash + .\deploy.ps1 + ``` + The script automatically assigns the required `Cognitive Services Contributor` role. + +2. **Manual permission assignment:** + ```bash + # Get your user object ID + az ad signed-in-user show --query "id" -o tsv + + # Deploy with explicit principal ID + .\deploy.ps1 -PrincipalId + ``` + +3. **Verify permissions in Azure Portal:** + - Navigate to your AI Foundry resource + - Check **Access control (IAM)** โ†’ **Role assignments** + - Ensure you have `Cognitive Services Contributor` role + +### Authentication Issues + +If you encounter authentication errors: + +1. Ensure you're logged in: `az login` or `azd auth login` +2. Check your permissions in the Azure portal +3. Verify the environment variables are set correctly +4. Try `az account show` to confirm your active subscription + +### Resource Deployment Failures + +Common issues: +- **Quota limits**: Check OpenAI and other service quotas in your region +- **Naming conflicts**: Resource names must be globally unique +- **Permissions**: Ensure you have Contributor access to the subscription + +### Testing Your Deployment + +Run the test script to verify everything is working: +```bash +python test_deployment.py +``` + +## Clean Up + +To remove all deployed resources: + +```bash +azd down +``` + +This will delete the resource group and all contained resources. + +## Support + +For issues with: +- **Azure Developer CLI**: Visit [azd documentation](https://learn.microsoft.com/azure/developer/azure-developer-cli/) +- **Azure AI Evaluation SDK**: Check the [SDK documentation](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/evaluation/azure-ai-evaluation) +- **Azure AI Foundry**: See [Azure AI Studio documentation](https://learn.microsoft.com/azure/ai-studio/) diff --git a/scenarios/evaluate/AI_RedTeaming/infra/ai-red-team.bicep b/scenarios/evaluate/AI_RedTeaming/infra/ai-red-team.bicep new file mode 100644 index 00000000..aa3fb2bc --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/ai-red-team.bicep @@ -0,0 +1,336 @@ +@description('Azure region of the deployment') +param location string + +@description('Tags to add to the resources') +param tags object + +@description('Resource token to use for unique resource names') +param resourceToken string + +@description('Id of the user or app to assign application roles') +param principalId string + +@description('Whether to create role assignments for the principal') +param createRoleAssignments bool = false + +// Variables for naming +var aiFoundryName = 'aifoundry-${resourceToken}' +var aiProjectName = 'aiproject-${resourceToken}' +var keyVaultName = 'kv-${resourceToken}' +var storageAccountName = 'stor${resourceToken}' +var searchServiceName = 'search-${resourceToken}' +var logAnalyticsName = 'log-${resourceToken}' +var appInsightsName = 'appi-${resourceToken}' + +// Create Key Vault +resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' = { + name: keyVaultName + location: location + tags: tags + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: subscription().tenantId + enableRbacAuthorization: true + enableSoftDelete: true + softDeleteRetentionInDays: 7 + enablePurgeProtection: true // Changed from false to true - required by Azure policy + publicNetworkAccess: 'Enabled' + networkAcls: { + defaultAction: 'Allow' + bypass: 'AzureServices' + } + } +} + +// Create Storage Account +resource storageAccount 'Microsoft.Storage/storageAccounts@2023-05-01' = { + name: storageAccountName + location: location + tags: tags + sku: { + name: 'Standard_LRS' + } + kind: 'StorageV2' + properties: { + accessTier: 'Hot' + allowBlobPublicAccess: false + allowSharedKeyAccess: true + defaultToOAuthAuthentication: false + minimumTlsVersion: 'TLS1_2' + networkAcls: { + defaultAction: 'Allow' + } + publicNetworkAccess: 'Enabled' + supportsHttpsTrafficOnly: true + } +} + +// Create Log Analytics workspace +resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = { + name: logAnalyticsName + location: location + tags: tags + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + features: { + searchVersion: 1 + legacy: 0 + enableLogAccessUsingOnlyResourcePermissions: true + } + } +} + +// Create Application Insights +resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { + name: appInsightsName + location: location + tags: tags + kind: 'web' + properties: { + Application_Type: 'web' + WorkspaceResourceId: logAnalytics.id + IngestionMode: 'LogAnalytics' + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } +} + +// Remove the separate OpenAI account since models are deployed directly on AI Foundry +// The OpenAI functionality is now provided through the AI Foundry account + +// Deploy OpenAI models +// Note: Models are now deployed as part of the AI Foundry account resource above + +// Create Azure AI Search service +resource searchService 'Microsoft.Search/searchServices@2024-06-01-preview' = { + name: searchServiceName + location: location + tags: tags + sku: { + name: 'basic' + } + properties: { + replicaCount: 1 + partitionCount: 1 + hostingMode: 'default' + publicNetworkAccess: 'enabled' + networkRuleSet: { + ipRules: [] + } + encryptionWithCmk: { + enforcement: 'Unspecified' + } + disableLocalAuth: false + authOptions: { + apiKeyOnly: {} + } + } +} + +/* + An AI Foundry resources is a variant of a CognitiveServices/account resource type +*/ +resource aiFoundry 'Microsoft.CognitiveServices/accounts@2025-04-01-preview' = { + name: aiFoundryName + location: location + tags: tags + identity: { + type: 'SystemAssigned' } + sku: { + name: 'S0' + } + kind: 'AIServices' + properties: { + // required to work in AI Foundry + allowProjectManagement: true + + // Defines developer API endpoint subdomain + customSubDomainName: aiFoundryName + + // Enable local auth to allow key listing for outputs + disableLocalAuth: false + + // Add required publicNetworkAccess property + publicNetworkAccess: 'Enabled' + + // Add required networkAcls configuration + networkAcls: { + defaultAction: 'Allow' + ipRules: [] + virtualNetworkRules: [] + } + } +} + +/* + Developer APIs are exposed via a project, which groups in- and outputs that relate to one use case, including files. + Its advisable to create one project right away, so development teams can directly get started. + Projects may be granted individual RBAC permissions and identities on top of what account provides. +*/ +resource aiProject 'Microsoft.CognitiveServices/accounts/projects@2025-04-01-preview' = { + name: aiProjectName + parent: aiFoundry + location: location + identity: { + type: 'SystemAssigned' + } + properties: {} +} + +/* + Optionally deploy a model to use in playground, agents and other tools. +*/ +resource modelDeployment 'Microsoft.CognitiveServices/accounts/deployments@2024-10-01'= { + parent: aiFoundry + name: 'gpt-4o' + sku : { + capacity: 1 + name: 'GlobalStandard' + } + properties: { + model:{ + name: 'gpt-4o' + format: 'OpenAI' + } + } +} + +// RBAC Role Definitions +var roleDefinitionIds = { + Owner: '8e3af657-a8ff-443c-a75c-2fe8c4bcb635' + Contributor: 'b24988ac-6180-42a0-ab88-20f7382dd24c' + Reader: 'acdd72a7-3385-48ef-bd42-f606fba81ae7' + 'Storage Blob Data Contributor': 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' + 'Storage Blob Data Reader': '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' + 'Cognitive Services OpenAI User': '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' + 'Cognitive Services User': 'a97b65f3-24c7-4388-baec-2e87135dc908' + 'Cognitive Services Contributor': '25fbc0a9-bd7c-42a3-aa1a-3b75d497ee68' + 'Search Index Data Contributor': '8ebe5a00-799e-43f5-93ac-243d3dce84a7' + 'Search Service Contributor': '7ca78c08-252a-4471-8644-bb5ff32d4ba0' + 'Key Vault Secrets User': '4633458b-17de-408a-b874-0445c86b69e6' + 'AzureML Data Scientist': 'f6c7c914-8db3-469d-8ca1-694a8f32e121' +} + +// Assign roles to the user/service principal if provided +resource storageRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments && !empty(principalId)) { + scope: storageAccount + name: guid(storageAccount.id, principalId, roleDefinitionIds['Storage Blob Data Contributor']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Storage Blob Data Contributor']) + principalId: principalId + principalType: 'User' + } +} + +resource aiFoundryRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments && !empty(principalId)) { + scope: aiFoundry + name: guid(aiFoundry.id, principalId, roleDefinitionIds['Cognitive Services OpenAI User']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Cognitive Services OpenAI User']) + principalId: principalId + principalType: 'User' + } +} + +resource searchRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments && !empty(principalId)) { + scope: searchService + name: guid(searchService.id, principalId, roleDefinitionIds['Search Index Data Contributor']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Search Index Data Contributor']) + principalId: principalId + principalType: 'User' + } +} + +resource keyVaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments && !empty(principalId)) { + scope: keyVault + name: guid(keyVault.id, principalId, roleDefinitionIds['Key Vault Secrets User']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Key Vault Secrets User']) + principalId: principalId + principalType: 'User' + } +} + +resource aiProjectRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments && !empty(principalId)) { + scope: aiFoundry // Assign role to AI Foundry account instead of project + name: guid(aiFoundry.id, principalId, roleDefinitionIds['Cognitive Services User']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Cognitive Services User']) + principalId: principalId + principalType: 'User' + } +} + +// Add Cognitive Services Contributor role for red team operations +resource aiFoundryContributorRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments && !empty(principalId)) { + scope: aiFoundry + name: guid(aiFoundry.id, principalId, roleDefinitionIds['Cognitive Services Contributor']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Cognitive Services Contributor']) + principalId: principalId + principalType: 'User' + } +} + +// Assign AI Foundry managed identity permissions (conditional) +resource aiFoundryStorageRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments) { + scope: storageAccount + name: guid(storageAccount.id, aiFoundry.id, roleDefinitionIds['Storage Blob Data Contributor']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Storage Blob Data Contributor']) + principalId: aiFoundry.identity.principalId + principalType: 'ServicePrincipal' + } +} + +resource aiFoundryKeyVaultRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments) { + scope: keyVault + name: guid(keyVault.id, aiFoundry.id, roleDefinitionIds['Key Vault Secrets User']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Key Vault Secrets User']) + principalId: aiFoundry.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Assign AI Project managed identity permissions (conditional) +resource aiProjectStorageRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments) { + scope: storageAccount + name: guid(storageAccount.id, aiProject.id, roleDefinitionIds['Storage Blob Data Contributor']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Storage Blob Data Contributor']) + principalId: aiProject.identity.principalId + principalType: 'ServicePrincipal' + } +} + +resource aiProjectFoundryRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (createRoleAssignments) { + scope: aiFoundry + name: guid(aiFoundry.id, aiProject.id, roleDefinitionIds['Cognitive Services OpenAI User']) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', roleDefinitionIds['Cognitive Services OpenAI User']) + principalId: aiProject.identity.principalId + principalType: 'ServicePrincipal' + } +} + +// Outputs +output aiFoundryName string = aiFoundry.name +output aiProjectName string = aiProject.name +output aiProjectEndpoint string = 'https://${aiFoundry.name}.services.ai.azure.com/api/projects/${aiProject.name}' +output aiFoundryEndpoint string = 'https://${aiFoundry.name}.services.ai.azure.com' +output openAiEndpoint string = aiFoundry.properties.endpoint +output openAiApiKey string = aiFoundry.listKeys().key1 +output storageAccountName string = storageAccount.name +output storageConnectionString string = 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};AccountKey=${storageAccount.listKeys().keys[0].value};EndpointSuffix=${environment().suffixes.storage}' +output keyVaultName string = keyVault.name +output searchServiceName string = searchService.name +output searchServiceEndpoint string = 'https://${searchService.name}.search.windows.net/' +output searchServiceApiKey string = searchService.listAdminKeys().primaryKey \ No newline at end of file diff --git a/scenarios/evaluate/AI_RedTeaming/infra/deploy.ps1 b/scenarios/evaluate/AI_RedTeaming/infra/deploy.ps1 new file mode 100644 index 00000000..9fb40755 --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/deploy.ps1 @@ -0,0 +1,418 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + Deploy Azure AI Red Teaming infrastructure using Azure Developer CLI + +.DESCRIPTION + This script simplifies the deployment of Azure AI Red Teaming infrastructure. + It checks prerequisites, authenticates with Azure, and deploys the resources. + +.PARAMETER EnvironmentName + Name of the environment (default: ai-red-team-dev) + +.PARAMETER Location + Azure region for deployment (default: eastus) + +.PARAMETER PrincipalId + Object ID of user to assign permissions to (optional) + +.PARAMETER SkipPrereqs + Skip prerequisite checks + +.EXAMPLE + .\deploy.ps1 + Deploy with default settings + +.EXAMPLE + .\deploy.ps1 -EnvironmentName "my-red-team" -Location "westus2" + Deploy with custom environment name and location + +.EXAMPLE + .\deploy.ps1 -PrincipalId "12345678-1234-1234-1234-123456789012" + Deploy and assign permissions to specific user +#> + +param( + [string]$EnvironmentName = "ai-red-team-dev", + [string]$Location = "eastus2", + [string]$PrincipalId = "", + [switch]$SkipPrereqs = $false +) + +# Color functions for better output +function Write-Success { param($Message) Write-Host $Message -ForegroundColor Green } +function Write-Warning { param($Message) Write-Host $Message -ForegroundColor Yellow } +function Write-Error { param($Message) Write-Host $Message -ForegroundColor Red } +function Write-Info { param($Message) Write-Host $Message -ForegroundColor Cyan } + +Write-Host "๐Ÿš€ Azure AI Red Teaming Infrastructure Deployment" -ForegroundColor Magenta +Write-Host "=================================================" -ForegroundColor Magenta + +# Check prerequisites +if (-not $SkipPrereqs) { + Write-Info "Checking prerequisites..." + + # Check Azure Developer CLI + try { + $azdVersion = azd version 2>$null + Write-Success "โœ“ Azure Developer CLI found: $azdVersion" + } + catch { + Write-Error "โœ— Azure Developer CLI not found. Please install from: https://learn.microsoft.com/azure/developer/azure-developer-cli/install-azd" + exit 1 + } + + # Check Azure CLI + try { + $azVersion = az version --query '"azure-cli"' -o tsv 2>$null + Write-Success "โœ“ Azure CLI found: $azVersion" + } + catch { + Write-Error "โœ— Azure CLI not found. Please install from: https://docs.microsoft.com/cli/azure/install-azure-cli" + exit 1 + } +} + +# Check if user is logged in +Write-Info "Checking Azure authentication..." +try { + $currentUser = az account show --query "user.name" -o tsv 2>$null + if ($currentUser) { + Write-Success "โœ“ Logged in as: $currentUser" + } else { + Write-Warning "Not logged in to Azure. Initiating login..." + azd auth login + if ($LASTEXITCODE -ne 0) { + Write-Error "Login failed. Please run 'azd auth login' manually." + exit 1 + } + } +} +catch { + Write-Warning "Could not verify Azure login status. Attempting to login..." + azd auth login +} + +# Check and select Azure subscription +Write-Info "Checking Azure subscription..." +try { + # Try different approaches to get subscription info + $currentSub = $null + + # First try with azd + try { + $azdSub = azd env get-value AZURE_SUBSCRIPTION_ID 2>$null + if ($azdSub) { + Write-Info "Found subscription from azd environment: $azdSub" + $currentSub = @{ name = "From azd environment"; id = $azdSub } + } + } + catch { } + + # If azd didn't work, try az CLI + if (-not $currentSub) { + try { + $azSub = az account show --query "{name:name, id:id}" -o json 2>$null + if ($azSub) { + $currentSub = $azSub | ConvertFrom-Json + } + } + catch { } + } + + if ($currentSub) { + Write-Host "Current subscription: $($currentSub.name) ($($currentSub.id))" -ForegroundColor White + Write-Success "โœ“ Using subscription: $($currentSub.name)" + + # Check if user has sufficient permissions (basic check) + Write-Info "๐Ÿ’ก Make sure you have Contributor permissions on this subscription for deployment to succeed" + } else { + Write-Warning "Could not determine current subscription automatically." + Write-Info "Available options:" + Write-Host "1. Set subscription with azd: azd env set AZURE_SUBSCRIPTION_ID " -ForegroundColor White + Write-Host "2. Set subscription with az CLI: az account set --subscription " -ForegroundColor White + Write-Host "3. List subscriptions: az account list --output table" -ForegroundColor White + Write-Host "" + + $manualSub = Read-Host "Enter your Azure subscription ID (or press Enter to continue and let azd handle it)" + if ($manualSub) { + Write-Info "Setting subscription: $manualSub" + azd env set AZURE_SUBSCRIPTION_ID $manualSub + if ($LASTEXITCODE -eq 0) { + Write-Success "โœ“ Subscription set successfully" + } + } else { + Write-Info "Continuing without explicit subscription selection - azd will prompt if needed" + } + } +} +catch { + Write-Warning "Could not verify subscription. Continuing anyway - azd will handle subscription selection." +} + +# Principal ID is optional - role assignments are disabled by default +# Check if PrincipalId was passed as parameter or exists in azd environment +if ([string]::IsNullOrEmpty($PrincipalId)) { + try { + $existingPrincipalId = azd env get-value AZURE_PRINCIPAL_ID 2>$null + if ($existingPrincipalId) { + $PrincipalId = $existingPrincipalId + Write-Info "Found existing Principal ID in environment: $PrincipalId" + } else { + Write-Info "Principal ID not specified - role assignments will be skipped (recommended for most deployments)" + } + } catch { + Write-Info "Principal ID not specified - role assignments will be skipped (recommended for most deployments)" + } +} else { + Write-Info "Principal ID provided as parameter: $PrincipalId" +} + +# Display deployment configuration +Write-Host "" +Write-Info "Deployment Configuration:" +Write-Host " Environment Name: $EnvironmentName" -ForegroundColor White +Write-Host " Location: $Location" -ForegroundColor White +Write-Host " Principal ID: $(if ($PrincipalId) { $PrincipalId } else { 'Not specified (role assignments disabled)' })" -ForegroundColor White +Write-Host "" + +# Confirm deployment +$confirmation = Read-Host "Do you want to proceed with the deployment? (y/N)" +if ($confirmation -ne 'y' -and $confirmation -ne 'Y') { + Write-Warning "Deployment cancelled." + exit 0 +} + +# Set environment variables for azd +$env:AZURE_ENV_NAME = $EnvironmentName +$env:AZURE_LOCATION = $Location + +# Ensure azd environment has the current parameter values +Write-Info "Updating azd environment with current parameters..." +azd env set AZURE_LOCATION $Location +if ($LASTEXITCODE -eq 0) { + Write-Success "โœ“ Location set to: $Location" +} else { + Write-Warning "Could not set location in azd environment, continuing anyway..." +} + +# Get current user's principal ID if not provided for red team operations +if (-not $PrincipalId) { + Write-Info "Getting current user's principal ID for role assignments..." + try { + $PrincipalId = az ad signed-in-user show --query "id" -o tsv 2>$null + if ($PrincipalId) { + Write-Success "โœ“ Found current user principal ID: $PrincipalId" + } else { + Write-Warning "Could not get current user principal ID. Role assignments will be disabled." + } + } + catch { + Write-Warning "Could not get current user principal ID. Role assignments will be disabled." + } +} + +# Set principal ID and enable role assignments for red team operations +if ($PrincipalId) { + $env:AZURE_PRINCIPAL_ID = $PrincipalId + azd env set AZURE_PRINCIPAL_ID $PrincipalId + if ($LASTEXITCODE -eq 0) { + Write-Success "โœ“ Principal ID set: $PrincipalId - role assignments will be enabled" + Write-Info " This enables required permissions for red team evaluation operations" + } else { + Write-Warning "Could not set Principal ID in azd environment, continuing anyway..." + } +} else { + Write-Warning "No Principal ID available - role assignments will be disabled" + Write-Warning "You may need to manually assign permissions after deployment" +} + +# Ensure we're using the correct subscription +try { + $currentSubId = azd env get-value AZURE_SUBSCRIPTION_ID 2>$null + if (-not $currentSubId) { + $currentSubId = az account show --query "id" -o tsv 2>$null + } + if ($currentSubId) { + Write-Info "Using subscription: $currentSubId" + } else { + Write-Info "Subscription will be selected during deployment" + } +} +catch { + Write-Info "Subscription will be selected during deployment" +} + +# Initialize azd environment if needed +Write-Info "Initializing Azure Developer CLI environment..." +if (-not (Test-Path ".azure")) { + azd init --environment $EnvironmentName --no-prompt + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to initialize azd environment." + exit 1 + } +} + +# Deploy infrastructure +Write-Info "Starting infrastructure deployment..." +Write-Host "This may take 10-15 minutes to complete..." -ForegroundColor Yellow +Write-Host "" +Write-Info "๐Ÿ’ก If you encounter permission errors:" +Write-Host " 1. Make sure you're using a subscription where you have Contributor permissions" -ForegroundColor White +Write-Host " 2. You can switch subscriptions with: az account set --subscription " -ForegroundColor White +Write-Host " 3. List available subscriptions with: az account list --output table" -ForegroundColor White +Write-Host "" + +azd up --environment $EnvironmentName --no-prompt +if ($LASTEXITCODE -ne 0) { + Write-Error "โŒ Deployment failed!" + Write-Host "" Write-Warning "๐Ÿ”ง Troubleshooting steps:" + Write-Host "1. Check that you're using the correct Azure subscription" -ForegroundColor White + Write-Host "2. Verify you have Contributor permissions on the subscription" -ForegroundColor White + Write-Host "3. Try using a different environment name if resources already exist" -ForegroundColor White + Write-Host "4. Run 'azd env list' to see existing environments" -ForegroundColor White + Write-Host "5. List subscriptions with: az account list --output table" -ForegroundColor White + Write-Host "6. Switch subscription with: az account set --subscription " -ForegroundColor White + exit 1 +} + +Write-Success "๐ŸŽ‰ Deployment completed successfully!" +Write-Host "" + +# Create .env file with deployment outputs +Write-Info "Creating .env file with deployment outputs..." +$envFilePath = Join-Path (Get-Location) ".env" +try { + $envVars = azd env get-values + if ($LASTEXITCODE -eq 0 -and $envVars) { + # Parse environment variables into a hashtable + $envHash = @{} + $envVars | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)$') { + $varName = $Matches[1] + $varValue = $Matches[2].Trim('"') + $envHash[$varName] = $varValue + } } + # Create the correct Azure AI Foundry project endpoint format + # Format: https://{foundry-name}.services.ai.azure.com/api/projects/{project-name} + if ($envHash.ContainsKey('AI_FOUNDRY_NAME') -and + $envHash.ContainsKey('AI_PROJECT_NAME')) { + + $projectEndpoint = "https://$($envHash['AI_FOUNDRY_NAME']).services.ai.azure.com/api/projects/$($envHash['AI_PROJECT_NAME'])" + $envHash['AZURE_AI_PROJECT_ENDPOINT'] = $projectEndpoint + Write-Info "โœ“ Generated Azure AI Foundry project endpoint" + } + + # Ensure all required variables are present for the test script + if ($envHash.ContainsKey('AI_FOUNDRY_ENDPOINT')) { + # Keep the foundry endpoint for the test script + Write-Info "โœ“ AI Foundry endpoint available" + } + + # Ensure we have the resource group name + if (-not $envHash.ContainsKey('AZURE_RESOURCE_GROUP') -and $envHash.ContainsKey('AZURE_RESOURCE_GROUP_NAME')) { + $envHash['AZURE_RESOURCE_GROUP'] = $envHash['AZURE_RESOURCE_GROUP_NAME'] + } + + # Write corrected environment variables to .env file + $envContent = @() + $envHash.GetEnumerator() | Sort-Object Name | ForEach-Object { + $envContent += "$($_.Key)=`"$($_.Value)`"" + } + $envContent | Out-File -FilePath $envFilePath -Encoding UTF8 + Write-Success "โœ“ Created .env file: $envFilePath" + # Display key environment variables + Write-Host "" + Write-Info "Key environment variables:" + Write-Host " AZURE_SUBSCRIPTION_ID=`"$($envHash['AZURE_SUBSCRIPTION_ID'])`"" -ForegroundColor White + Write-Host " AZURE_RESOURCE_GROUP=`"$($envHash['AZURE_RESOURCE_GROUP'])`"" -ForegroundColor White + Write-Host " AI_PROJECT_NAME=`"$($envHash['AI_PROJECT_NAME'])`"" -ForegroundColor White + Write-Host " AI_FOUNDRY_ENDPOINT=`"$($envHash['AI_FOUNDRY_ENDPOINT'])`"" -ForegroundColor White + Write-Host " AZURE_OPENAI_ENDPOINT=`"$($envHash['AZURE_OPENAI_ENDPOINT'])`"" -ForegroundColor White + Write-Host " AZURE_STORAGE_ACCOUNT_NAME=`"$($envHash['AZURE_STORAGE_ACCOUNT_NAME'])`"" -ForegroundColor White + } else { + Write-Warning "Could not retrieve environment variables from azd" + } +} +catch { + Write-Warning "Failed to create .env file: $($_.Exception.Message)" +} + +# Run deployment test +Write-Host "" +Write-Info "Running deployment verification test..." +$testScript = Join-Path $PSScriptRoot "test_deployment.py" +if (Test-Path $testScript) { + try { + # Check if python is available + $pythonCmd = $null + foreach ($cmd in @("python", "python3", "py")) { + try { + $version = & $cmd --version 2>$null + if ($LASTEXITCODE -eq 0) { + $pythonCmd = $cmd + Write-Info "Using Python: $version" + break + } + } + catch { } + } + + if ($pythonCmd) { + Write-Host "" Write-Info "๐Ÿงช Testing deployment..." + # Set environment variables for the test using corrected values + $envVars = azd env get-values + if ($envVars) { + # Parse and set environment variables with corrections + $envHash = @{} + $envVars | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)$') { + $varName = $Matches[1] + $varValue = $Matches[2].Trim('"') + $envHash[$varName] = $varValue + } + } # Apply the Azure AI Foundry project endpoint + if ($envHash.ContainsKey('AI_FOUNDRY_NAME') -and + $envHash.ContainsKey('AI_PROJECT_NAME')) { + + $projectEndpoint = "https://$($envHash['AI_FOUNDRY_NAME']).services.ai.azure.com/api/projects/$($envHash['AI_PROJECT_NAME'])" + $envHash['AZURE_AI_PROJECT_ENDPOINT'] = $projectEndpoint + } + + # Ensure we have the resource group name + if (-not $envHash.ContainsKey('AZURE_RESOURCE_GROUP') -and $envHash.ContainsKey('AZURE_RESOURCE_GROUP_NAME')) { + $envHash['AZURE_RESOURCE_GROUP'] = $envHash['AZURE_RESOURCE_GROUP_NAME'] + } + + # Set all environment variables + $envHash.GetEnumerator() | ForEach-Object { + Set-Item -Path "env:$($_.Key)" -Value $_.Value + } + } + + & $pythonCmd $testScript + if ($LASTEXITCODE -eq 0) { + Write-Success "โœ“ Deployment test passed!" + } else { + Write-Warning "โš  Deployment test had issues. Check the output above." + } + } else { + Write-Warning "Python not found. Please install Python to run deployment tests." + Write-Info "You can manually run: python infra/test_deployment.py" + } + } + catch { + Write-Warning "Error running deployment test: $($_.Exception.Message)" + } +} else { + Write-Warning "Test script not found at: $testScript" +} + +Write-Host "" +Write-Success "Next steps:" +Write-Host "1. โœ“ Environment variables are saved in .env file" -ForegroundColor White +Write-Host "2. โœ“ Deployment has been tested" -ForegroundColor White +Write-Host "3. Run your AI Red Teaming notebooks or scripts" -ForegroundColor White +Write-Host "" +Write-Info "To clean up resources later, run: azd down --environment $EnvironmentName" \ No newline at end of file diff --git a/scenarios/evaluate/AI_RedTeaming/infra/main.bicep b/scenarios/evaluate/AI_RedTeaming/infra/main.bicep new file mode 100644 index 00000000..33c088f9 --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/main.bicep @@ -0,0 +1,67 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the the environment which is used to generate a short unique hash used in all resources.') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +@description('Id of the user or app to assign application roles') +param principalId string = '' + +// Tags that should be applied to all resources. +var tags = { + 'azd-env-name': environmentName + 'ai-toolkit': 'red-teaming' +} + +// Generate a unique token to be used in naming resources. +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +// Name of the resource group for the AI Red Teaming resources +var resourceGroupName = 'rg-${environmentName}-${resourceToken}' + +resource rg 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location + tags: tags +} + +module aiRedTeam 'ai-red-team.bicep' = { + name: 'ai-red-team' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + principalId: principalId + createRoleAssignments: false + } +} + +// Outputs - Standard Azure environment variables +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output AZURE_SUBSCRIPTION_ID string = subscription().subscriptionId +output AZURE_RESOURCE_GROUP string = rg.name + +// Primary outputs for .env file generation +output AZURE_AI_PROJECT string = aiRedTeam.outputs.aiProjectEndpoint +output AZURE_OPENAI_CONFIG string = '{"endpoint": "${aiRedTeam.outputs.openAiEndpoint}", "api_key": "${aiRedTeam.outputs.openAiApiKey}", "deployment": "gpt-4o", "api_version": "2024-10-21"}' +output AZURE_STORAGE_ACCOUNT string = aiRedTeam.outputs.storageAccountName + +// Additional resource outputs for debugging and advanced use cases +output AI_FOUNDRY_NAME string = aiRedTeam.outputs.aiFoundryName +output AI_PROJECT_NAME string = aiRedTeam.outputs.aiProjectName +output AI_FOUNDRY_ENDPOINT string = aiRedTeam.outputs.aiFoundryEndpoint +output AZURE_OPENAI_ENDPOINT string = aiRedTeam.outputs.openAiEndpoint +output AZURE_OPENAI_API_KEY string = aiRedTeam.outputs.openAiApiKey +output AZURE_STORAGE_ACCOUNT_NAME string = aiRedTeam.outputs.storageAccountName +output AZURE_STORAGE_CONNECTION_STRING string = aiRedTeam.outputs.storageConnectionString +output SEARCH_SERVICE_NAME string = aiRedTeam.outputs.searchServiceName +output SEARCH_SERVICE_ENDPOINT string = aiRedTeam.outputs.searchServiceEndpoint +output SEARCH_SERVICE_API_KEY string = aiRedTeam.outputs.searchServiceApiKey +output KEY_VAULT_NAME string = aiRedTeam.outputs.keyVaultName diff --git a/scenarios/evaluate/AI_RedTeaming/infra/main.bicepparam b/scenarios/evaluate/AI_RedTeaming/infra/main.bicepparam new file mode 100644 index 00000000..a7d1910f --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/main.bicepparam @@ -0,0 +1,9 @@ +using 'main.bicep' + +// Environment configuration +param environmentName = readEnvironmentVariable('AZURE_ENV_NAME', 'ai-red-team-dev') +param location = readEnvironmentVariable('AZURE_LOCATION', 'eastus2') + +// User permissions (optional - set via azd up --set principalId=) +// Leave empty for now to skip role assignments due to permission issues +param principalId = readEnvironmentVariable('AZURE_PRINCIPAL_ID', '') \ No newline at end of file diff --git a/scenarios/evaluate/AI_RedTeaming/infra/test_deployment.py b/scenarios/evaluate/AI_RedTeaming/infra/test_deployment.py new file mode 100644 index 00000000..eea7678e --- /dev/null +++ b/scenarios/evaluate/AI_RedTeaming/infra/test_deployment.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +""" +Test script to verify Azure AI Red Teaming infrastructure deployment. + +This script tests the deployed infrastructure by: +1. Connecting to Azure AI Project using DefaultAzureCredential +2. Verifying OpenAI connection and model availability +3. Testing storage account access +4. Validating authentication setup + +Run this script after deployment to verify everything is working correctly. + +Prerequisites: +- azure-ai-evaluation package installed +- azure-ai-projects package installed +- Environment variables set (from deployment output) +- Azure authentication configured (az login or azd auth login) + +Usage: + python test_deployment.py +""" + +import os +import sys + + +# Color output helper +class Colors: + GREEN = "\033[92m" + YELLOW = "\033[93m" + RED = "\033[91m" + BLUE = "\033[94m" + BOLD = "\033[1m" + END = "\033[0m" + + +def print_success(message: str) -> None: + print(f"{Colors.GREEN}โœ“ {message}{Colors.END}") + + +def print_warning(message: str) -> None: + print(f"{Colors.YELLOW}โš  {message}{Colors.END}") + + +def print_error(message: str) -> None: + print(f"{Colors.RED}โœ— {message}{Colors.END}") + + +def print_info(message: str) -> None: + print(f"{Colors.BLUE}i {message}{Colors.END}") + + +def check_environment_variables() -> bool: + """Check if required environment variables are set.""" + required_vars = [ + "AZURE_SUBSCRIPTION_ID", + "AZURE_RESOURCE_GROUP", + "AI_PROJECT_NAME", + "AI_FOUNDRY_ENDPOINT", + "AZURE_OPENAI_ENDPOINT", + "AZURE_STORAGE_ACCOUNT_NAME", + "AZURE_OPENAI_API_KEY", + ] + + missing_vars = [] + for var in required_vars: + if not os.getenv(var): + missing_vars.append(var) + + if missing_vars: + print_error("Missing required environment variables:") + for var in missing_vars: + print(f" - {var}") + print_info("Please set these variables from your deployment output.") + return False + + print_success("All required environment variables are set") + return True + + +def test_azure_ai_project() -> bool: + """Test Azure AI Project connection.""" + try: + from azure.ai.projects import AIProjectClient + from azure.identity import DefaultAzureCredential + + print_info("Testing Azure AI Project connection...") + credential = DefaultAzureCredential() # Get required parameters from environment variables + subscription_id = os.environ["AZURE_SUBSCRIPTION_ID"] + resource_group_name = os.environ["AZURE_RESOURCE_GROUP"] + project_name = os.environ["AI_PROJECT_NAME"] + endpoint = os.environ["AI_FOUNDRY_ENDPOINT"] + + # Create project client to test connection + AIProjectClient( + endpoint=endpoint, + credential=credential, + subscription_id=subscription_id, + resource_group_name=resource_group_name, + project_name=project_name, + ) + + # If we reach here, connection was successful + print_success(f"Connected to Azure AI Project: {project_name}") + print_info(f"Resource Group: {resource_group_name}") + print_info(f"Subscription: {subscription_id}") + return True + + except ImportError: + print_error("azure-ai-projects package not installed. Run: pip install azure-ai-projects") + return False + except KeyError as e: + print_error(f"Missing environment variable: {e!s}") + return False + except Exception as e: + print_error(f"Failed to connect to Azure AI Project: {e!s}") + return False + + +def test_openai_connection() -> bool: + """Test OpenAI connection and model availability.""" + try: + from openai import AzureOpenAI + + print_info("Testing Azure OpenAI connection...") + + client = AzureOpenAI( + azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], + api_key=os.environ["AZURE_OPENAI_API_KEY"], + api_version="2025-01-01-preview", + ) + # Test with a simple completion + response = client.chat.completions.create( + model="gpt-4o", # Use the deployed GPT-4o model + messages=[{"role": "user", "content": "Say hello!"}], + max_tokens=10, + ) + + print_success("OpenAI connection successful") + print_info(f"Test response: {response.choices[0].message.content}") + return True + + except ImportError: + print_error("openai package not installed. Run: pip install openai") + return False + except Exception as e: + print_error(f"Failed to connect to Azure OpenAI: {e!s}") + print_info("Check that gpt-4o model is deployed and you have proper permissions") + return False + + +def test_storage_access() -> bool: + """Test Azure Storage access.""" + try: + from azure.storage.blob import BlobServiceClient + from azure.identity import DefaultAzureCredential + + print_info("Testing Azure Storage access...") + + credential = DefaultAzureCredential() + storage_account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"] + + blob_service_client = BlobServiceClient( + account_url=f"https://{storage_account_name}.blob.core.windows.net", credential=credential + ) + + # Try to list containers (this tests authentication) + containers = list(blob_service_client.list_containers()) + print_success(f"Storage access successful. Found {len(containers)} containers") + return True + + except ImportError: + print_error("azure-storage-blob package not installed. Run: pip install azure-storage-blob") + return False + except Exception as e: + print_error(f"Failed to access Azure Storage: {e!s}") + return False + + +def test_ai_evaluation() -> bool: + """Test Azure AI Evaluation package.""" + try: + import importlib.util + + spec = importlib.util.find_spec("azure.ai.evaluation") + if spec is None: + print_error("azure-ai-evaluation package not installed. Run: pip install azure-ai-evaluation") + return False + + print_success("Azure AI Evaluation package is available") + return True + except ImportError: + print_error("azure-ai-evaluation package not installed. Run: pip install azure-ai-evaluation") + return False + except Exception as e: + print_warning(f"Azure AI Evaluation import warning: {e!s}") + return True + + +def test_red_team_permissions() -> bool: + """Test if user has required permissions for red team operations.""" + try: + from azure.identity import DefaultAzureCredential + from azure.mgmt.cognitiveservices import CognitiveServicesManagementClient + from azure.core.exceptions import ClientAuthenticationError, HttpResponseError + + print_info("Testing red team permissions...") + credential = DefaultAzureCredential() + + subscription_id = os.environ["AZURE_SUBSCRIPTION_ID"] + resource_group = os.environ["AZURE_RESOURCE_GROUP"] + + # Try to create a management client to test permissions + try: + mgmt_client = CognitiveServicesManagementClient(credential, subscription_id) + + # List accounts to test basic read permissions + accounts = list(mgmt_client.accounts.list_by_resource_group(resource_group)) + if accounts: + print_success("โœ“ Basic Cognitive Services permissions verified") + print_info("Red team operations should work with deployed permissions") + return True + print_warning("No Cognitive Services accounts found in resource group") + return False + + except ClientAuthenticationError as e: + print_error(f"Authentication failed: {e!s}") + print_warning("You may need 'Cognitive Services Contributor' role for red team operations") + return False + except HttpResponseError as e: + if "Authorization" in str(e) or "Permission" in str(e): + print_error(f"Permission error: {e!s}") + print_warning("You may need 'Cognitive Services Contributor' role for red team operations") + return False + print_warning(f"HTTP error (may be temporary): {e!s}") + return True + except Exception as e: + print_warning(f"Could not verify permissions (continuing anyway): {e!s}") + return True + + except ImportError: + print_warning("azure-mgmt-cognitiveservices not available for permission testing") + print_info("Install with: pip install azure-mgmt-cognitiveservices") + return True # Don't fail the test for missing optional dependency + except Exception as e: + print_warning(f"Permission test failed (continuing anyway): {e!s}") + return True + + +def main() -> None: + """Run all deployment tests.""" + print(f"{Colors.BOLD}๐Ÿงช Azure AI Red Teaming Deployment Test{Colors.END}") + print("=" * 50) + + tests = [ + ("Environment Variables", check_environment_variables), + ("Azure AI Evaluation Package", test_ai_evaluation), + ("Azure AI Project", test_azure_ai_project), + ("Azure OpenAI", test_openai_connection), + ("Azure Storage", test_storage_access), + ("Red Team Permissions", test_red_team_permissions), + ] + + results = [] + for test_name, test_func in tests: + print(f"\n{Colors.BOLD}Testing {test_name}...{Colors.END}") + try: + result = test_func() + results.append((test_name, result)) + except Exception as e: + print_error(f"Unexpected error in {test_name}: {e!s}") + results.append((test_name, False)) + + # Summary + print(f"\n{Colors.BOLD}Test Summary:{Colors.END}") + print("=" * 30) + + passed = sum(1 for _, result in results if result) + total = len(results) + + for test_name, result in results: + status = "PASS" if result else "FAIL" + color = Colors.GREEN if result else Colors.RED + print(f"{color}{status:<6}{Colors.END} {test_name}") + + print(f"\nResults: {passed}/{total} tests passed") + + if passed == total: + print_success("๐ŸŽ‰ All tests passed! Your deployment is ready for AI Red Teaming.") + else: + print_warning("โš ๏ธ Some tests failed. Check the error messages above and verify your deployment.") + sys.exit(1) + + +if __name__ == "__main__": + main()