diff --git a/azure.yaml b/azure.yaml index 7ae23bbf5..a857e8b3a 100644 --- a/azure.yaml +++ b/azure.yaml @@ -14,6 +14,8 @@ hooks: run: | Write-Host "Web app URL: " Write-Host "$env:WEB_APP_URL" -ForegroundColor Cyan + Write-Host "✅ If you want to use the Sample Data, run the following command in the Bash terminal to process it:" + ./infra/scripts/process_sample_data.sh shell: pwsh continueOnError: false interactive: true @@ -21,6 +23,8 @@ hooks: run: | echo "Web app URL: " echo $WEB_APP_URL + echo "✅ If you want to use the Sample Data, run the following command in the Bash terminal to process it:" + ./infra/scripts/process_sample_data.sh shell: sh continueOnError: false interactive: true diff --git a/infra/deploy_post_deployment_scripts.bicep b/infra/deploy_post_deployment_scripts.bicep deleted file mode 100644 index 96086a3c8..000000000 --- a/infra/deploy_post_deployment_scripts.bicep +++ /dev/null @@ -1,93 +0,0 @@ -@description('Solution Name') -param solutionName string -@description('Specifies the location for resources.') -param solutionLocation string -param baseUrl string -param managedIdentityObjectId string -param managedIdentityClientId string -param storageAccountName string -param containerName string -param containerAppName string = '${ solutionName }containerapp' -param environmentName string = '${ solutionName }containerappenv' -param imageName string = 'python:3.11-alpine' -param setupCopyKbFiles string = '${baseUrl}infra/scripts/copy_kb_files.sh' -param setupCreateIndexScriptsUrl string = '${baseUrl}infra/scripts/run_create_index_scripts.sh' -param createSqlUserAndRoleScriptsUrl string = '${baseUrl}infra/scripts/add_user_scripts/create-sql-user-and-role.ps1' -param keyVaultName string -param sqlServerName string -param sqlDbName string -param sqlUsers array = [ -] -param logAnalyticsWorkspaceResourceName string -var resourceGroupName = resourceGroup().name - -resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2020-10-01' existing = { - name: logAnalyticsWorkspaceResourceName - scope: resourceGroup() -} - -resource containerAppEnv 'Microsoft.App/managedEnvironments@2022-03-01' = { - name: environmentName - location: solutionLocation - properties: { - zoneRedundant: false - appLogsConfiguration: { - destination: 'log-analytics' - logAnalyticsConfiguration: { - customerId: logAnalytics.properties.customerId - sharedKey: logAnalytics.listKeys().primarySharedKey - } - } - } -} - -resource containerApp 'Microsoft.App/containerApps@2022-03-01' = { - name: containerAppName - location: solutionLocation - identity: { - type: 'UserAssigned' - userAssignedIdentities: { - '${managedIdentityObjectId}': {} - } - } - properties: { - managedEnvironmentId: containerAppEnv.id - configuration: { - ingress: null - activeRevisionsMode: 'Single' - } - template: { - scale:{ - minReplicas: 1 - maxReplicas: 1 - } - containers: [ - { - name: containerAppName - image: imageName - resources: { - cpu: 2 - memory: '4.0Gi' - } - command: [ - '/bin/sh', '-c', 'mkdir -p /scripts && apk add --no-cache curl bash jq py3-pip gcc musl-dev libffi-dev openssl-dev python3-dev && pip install --upgrade azure-cli && apk add --no-cache --virtual .build-deps build-base unixodbc-dev && curl -s -o msodbcsql18_18.4.1.1-1_amd64.apk https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/msodbcsql18_18.4.1.1-1_amd64.apk && curl -s -o mssql-tools18_18.4.1.1-1_amd64.apk https://download.microsoft.com/download/7/6/d/76de322a-d860-4894-9945-f0cc5d6a45f8/mssql-tools18_18.4.1.1-1_amd64.apk && apk add --allow-untrusted msodbcsql18_18.4.1.1-1_amd64.apk && apk add --allow-untrusted mssql-tools18_18.4.1.1-1_amd64.apk && curl -s -o /scripts/copy_kb_files.sh ${setupCopyKbFiles} && chmod +x /scripts/copy_kb_files.sh && sh -x /scripts/copy_kb_files.sh ${storageAccountName} ${containerName} ${baseUrl} ${managedIdentityClientId} && curl -s -o /scripts/run_create_index_scripts.sh ${setupCreateIndexScriptsUrl} && chmod +x /scripts/run_create_index_scripts.sh && sh -x /scripts/run_create_index_scripts.sh ${baseUrl} ${keyVaultName} ${managedIdentityClientId} && apk add --no-cache ca-certificates less ncurses-terminfo-base krb5-libs libgcc libintl libssl3 libstdc++ tzdata userspace-rcu zlib icu-libs curl && apk -X https://dl-cdn.alpinelinux.org/alpine/edge/main add --no-cache lttng-ust openssh-client && curl -L https://github.com/PowerShell/PowerShell/releases/download/v7.5.0/powershell-7.5.0-linux-musl-x64.tar.gz -o /tmp/powershell.tar.gz && mkdir -p /opt/microsoft/powershell/7 && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 && chmod +x /opt/microsoft/powershell/7/pwsh && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && curl -s -o /scripts/create-sql-user-and-role.ps1 ${createSqlUserAndRoleScriptsUrl} && chmod +x /scripts/create-sql-user-and-role.ps1 && pwsh -File /scripts/create-sql-user-and-role.ps1 -SqlServerName ${sqlServerName} -SqlDatabaseName ${sqlDbName} -ClientId ${sqlUsers[0].principalId} -DisplayName ${sqlUsers[0].principalName} -ManagedIdentityClientId ${managedIdentityClientId} -DatabaseRole ${sqlUsers[0].databaseRoles[0]} && pwsh -File /scripts/create-sql-user-and-role.ps1 -SqlServerName ${sqlServerName} -SqlDatabaseName ${sqlDbName} -ClientId ${sqlUsers[0].principalId} -DisplayName ${sqlUsers[0].principalName} -ManagedIdentityClientId ${managedIdentityClientId} -DatabaseRole ${sqlUsers[0].databaseRoles[1]} && az login --identity --client-id ${managedIdentityClientId} && az containerapp update --name ${containerAppName} --resource-group ${resourceGroupName} --min-replicas 0 --cpu 0.25 --memory 0.5Gi && az containerapp revision deactivate -g ${resourceGroupName} --revision $(az containerapp revision list -n ${containerAppName} -g ${resourceGroupName} --query "[0].name" -o tsv) && echo "Container app setup completed successfully."' - ] - env: [ - { - name: 'STORAGE_ACCOUNT_NAME' - value: storageAccountName - } - { - name: 'CONTAINER_NAME' - value: containerName - } - { - name:'APPSETTING_WEBSITE_SITE_NAME' - value:'DUMMY' - } - ] - } - ] - } - } -} diff --git a/infra/main.bicep b/infra/main.bicep index c969a2e5d..20ac0ec18 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -157,29 +157,34 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' existing = { } //========== Deployment script to upload sample data ========== // -module uploadFiles 'deploy_post_deployment_scripts.bicep' = { - name : 'deploy_post_deployment_scripts' - params:{ - solutionName: solutionPrefix - solutionLocation: secondaryLocation - baseUrl: baseUrl - storageAccountName: storageAccount.outputs.storageName - containerName: storageAccount.outputs.storageContainer - managedIdentityObjectId:managedIdentityModule.outputs.managedIdentityOutput.id - managedIdentityClientId:managedIdentityModule.outputs.managedIdentityOutput.clientId - keyVaultName:aifoundry.outputs.keyvaultName - logAnalyticsWorkspaceResourceName: aifoundry.outputs.logAnalyticsWorkspaceResourceName - sqlServerName: sqlDBModule.outputs.sqlServerName - sqlDbName: sqlDBModule.outputs.sqlDbName - sqlUsers: [ - { - principalId: managedIdentityModule.outputs.managedIdentityBackendAppOutput.clientId // Replace with actual Principal ID - principalName: managedIdentityModule.outputs.managedIdentityBackendAppOutput.name // Replace with actual user email or name - databaseRoles: ['db_datareader', 'db_datawriter'] - } - ] - } -} +// module uploadFiles 'deploy_post_deployment_scripts.bicep' = { +// name : 'deploy_post_deployment_scripts' +// params:{ +// solutionName: solutionPrefix +// solutionLocation: secondaryLocation +// baseUrl: baseUrl +// storageAccountName: storageAccount.outputs.storageName +// containerName: storageAccount.outputs.storageContainer +// managedIdentityObjectId:managedIdentityModule.outputs.managedIdentityOutput.id +// managedIdentityClientId:managedIdentityModule.outputs.managedIdentityOutput.clientId +// keyVaultName:aifoundry.outputs.keyvaultName +// logAnalyticsWorkspaceResourceName: aifoundry.outputs.logAnalyticsWorkspaceResourceName +// sqlServerName: sqlDBModule.outputs.sqlServerName +// sqlDbName: sqlDBModule.outputs.sqlDbName +// sqlUsers: [ +// { +// principalId: managedIdentityModule.outputs.managedIdentityChartsOutput.clientId // Replace with actual Principal ID +// principalName: managedIdentityModule.outputs.managedIdentityChartsOutput.name // Replace with actual user email or name +// databaseRoles: ['db_datareader', 'db_datawriter'] +// } +// { +// principalId: managedIdentityModule.outputs.managedIdentityRagOutput.clientId // Replace with actual Principal ID +// principalName: managedIdentityModule.outputs.managedIdentityRagOutput.name // Replace with actual user email or name +// databaseRoles: ['db_datareader'] +// } +// ] +// } +// } module hostingplan 'deploy_app_service_plan.bicep' = { name: 'deploy_app_service_plan' diff --git a/infra/main.json b/infra/main.json index 8843ca450..03904bd8a 100644 --- a/infra/main.json +++ b/infra/main.json @@ -1314,7 +1314,7 @@ "resources": [ { "copy": { - "name": "database::list", + "name": "list", "count": "[length(variables('containers'))]" }, "type": "Microsoft.DocumentDB/databaseAccounts/sqlDatabases/containers", diff --git a/infra/scripts/add_user_scripts/create-sql-user-and-role.ps1 b/infra/scripts/add_user_scripts/create-sql-user-and-role.ps1 index 2e3686b29..38c9c1e87 100644 --- a/infra/scripts/add_user_scripts/create-sql-user-and-role.ps1 +++ b/infra/scripts/add_user_scripts/create-sql-user-and-role.ps1 @@ -7,7 +7,7 @@ .DESCRIPTION During an application deployment, the managed identity (and potentially the developer identity) must be added to the SQL database as a user and assigned to one or more roles. This script - accomplishes this task using the owner-managed identity for authentication. + accomplishes this task using Azure AD authentication. .PARAMETER SqlServerName The name of the Azure SQL Server resource. @@ -21,43 +21,36 @@ .PARAMETER DisplayName The Object (Principal) display name of the identity to be added. -.PARAMETER ManagedIdentityClientId - The Client ID of the managed identity that will authenticate to the SQL database. +.PARAMETER UseManagedIdentity + Switch to indicate whether to use a Managed Identity for authentication (useful for automation). + If not provided, it will use your currently logged-in Azure AD account. .PARAMETER DatabaseRole The database role that should be assigned to the user (e.g., db_datareader, db_datawriter, db_owner). #> -Param( +param ( [string] $SqlServerName, [string] $SqlDatabaseName, [string] $ClientId, [string] $DisplayName, - [string] $ManagedIdentityClientId, + [switch] $UseManagedIdentity, [string] $DatabaseRole ) function Resolve-Module($moduleName) { - # If module is imported; say that and do nothing - if (Get-Module | Where-Object { $_.Name -eq $moduleName }) { - Write-Debug "Module $moduleName is already imported" - } elseif (Get-Module -ListAvailable | Where-Object { $_.Name -eq $moduleName }) { - Import-Module $moduleName - } elseif (Find-Module -Name $moduleName | Where-Object { $_.Name -eq $moduleName }) { - Install-Module $moduleName -Force -Scope CurrentUser - Import-Module $moduleName - } else { - Write-Error "Module $moduleName not found" - [Environment]::exit(1) + if (-not (Get-Module -ListAvailable -Name $moduleName)) { + Install-Module -Name $moduleName -Scope CurrentUser -Force -AllowClobber } + Import-Module -Name $moduleName -Force } -### -### MAIN SCRIPT -### +### Load Required Modules +Resolve-Module -moduleName Az.Accounts Resolve-Module -moduleName Az.Resources Resolve-Module -moduleName SqlServer +### Generate SQL Script $sql = @" DECLARE @username nvarchar(max) = N'$($DisplayName)'; DECLARE @clientId uniqueidentifier = '$($ClientId)'; @@ -70,8 +63,21 @@ END EXEC sp_addrolemember '$($DatabaseRole)', @username; "@ -Write-Output "`nSQL:`n$($sql)`n`n" +Write-Output "`nSQL to be executed:`n$($sql)`n" + +### Authenticate and Get Access Token +if ($UseManagedIdentity) { + Write-Host "[INFO] Logging in using Managed Identity..." + Connect-AzAccount -Identity +} else { + Write-Host "[INFO] Logging in using current user identity..." + Connect-AzAccount +} -Connect-AzAccount -Identity -AccountId $ManagedIdentityClientId $token = (Get-AzAccessToken -ResourceUrl https://database.windows.net/).Token -Invoke-SqlCmd -ServerInstance "$SqlServerName" -Database $SqlDatabaseName -AccessToken $token -Query $sql -ErrorAction 'Stop' \ No newline at end of file + +### Execute the SQL Command +Write-Host "[INFO] Executing SQL against $SqlDatabaseName..." +Invoke-Sqlcmd -ServerInstance "$SqlServerName.database.windows.net" -Database $SqlDatabaseName -AccessToken $token -Query $sql -ErrorAction Stop + +Write-Host "[SUCCESS] User and role assignment completed." diff --git a/infra/scripts/copy_kb_files.sh b/infra/scripts/copy_kb_files.sh index 7835597a6..ffe7bad24 100644 --- a/infra/scripts/copy_kb_files.sh +++ b/infra/scripts/copy_kb_files.sh @@ -3,33 +3,71 @@ # Variables storageAccount="$1" fileSystem="$2" -baseUrl="$3" +# baseUrl="$3" managedIdentityClientId="$4" +keyVaultName="$5" # ✅ NEW ARG REQUIRED zipFileName1="call_transcripts.zip" extractedFolder1="call_transcripts" -zipUrl1=${baseUrl}"infra/data/call_transcripts.zip" +zipUrl1="infra/data/call_transcripts.zip" zipFileName2="audio_data.zip" extractedFolder2="audiodata" -zipUrl2=${baseUrl}"infra/data/audio_data.zip" +zipUrl2="infra/data/audio_data.zip" -# Create folders if they do not exist -mkdir -p "/mnt/azscripts/azscriptinput/$extractedFolder1" -mkdir -p "/mnt/azscripts/azscriptinput/$extractedFolder2" +unzip infra/data/"$zipFileName1" -d infra/data/"$extractedFolder1" +unzip infra/data/"$zipFileName2" -d infra/data/"$extractedFolder2" -# Download the zip file -curl --output /mnt/azscripts/azscriptinput/"$zipFileName1" "$zipUrl1" -curl --output /mnt/azscripts/azscriptinput/"$zipFileName2" "$zipUrl2" +echo "Script Started" -# Extract the zip file -unzip /mnt/azscripts/azscriptinput/"$zipFileName1" -d /mnt/azscripts/azscriptinput/"$extractedFolder1" -unzip /mnt/azscripts/azscriptinput/"$zipFileName2" -d /mnt/azscripts/azscriptinput/"$extractedFolder2" +# Authenticate with Azure +if az account show &> /dev/null; then + echo "Already authenticated with Azure." +else + if [ -n "$managedIdentityClientId" ]; then + echo "Authenticating with Managed Identity..." + az login --identity --client-id ${managedIdentityClientId} + else + echo "Authenticating with Azure CLI..." + az login + fi + echo "Not authenticated with Azure. Attempting to authenticate..." +fi -echo "Script Started" +echo "Getting signed in user id" +signed_user_id=$(az ad signed-in-user show --query id -o tsv) + +echo "Getting storage account resource id" +storage_account_resource_id=$(az storage account show --name $storageAccount --query id --output tsv) + +# ✅ Assign Storage Blob Data Contributor role (if not already assigned) +echo "Checking if user has the Storage Blob Data Contributor role" +storage_role_assignment=$(az role assignment list --assignee $signed_user_id --role "Storage Blob Data Contributor" --scope $storage_account_resource_id --query "[].roleDefinitionId" -o tsv) + +if [ -z "$storage_role_assignment" ]; then + echo "Assigning Storage Blob Data Contributor role..." + az role assignment create --assignee $signed_user_id --role "Storage Blob Data Contributor" --scope $storage_account_resource_id --output none + echo "Role assignment for Blob Storage completed." +else + echo "User already has Storage Blob Data Contributor role." +fi + +# ✅ Assign Key Vault Secrets User role (NEW BLOCK) +echo "Getting Key Vault resource ID" +key_vault_resource_id=$(az keyvault show --name $keyVaultName --query id --output tsv) + +echo "Checking if user has Key Vault Secrets User role" +kv_role_assignment=$(az role assignment list --assignee $signed_user_id --role "Key Vault Secrets User" --scope $key_vault_resource_id --query "[].roleDefinitionId" -o tsv) + +if [ -z "$kv_role_assignment" ]; then + echo "Assigning Key Vault Secrets User role..." + az role assignment create --assignee $signed_user_id --role "Key Vault Secrets User" --scope $key_vault_resource_id --output none + echo "Role assignment for Key Vault completed." +else + echo "User already has Key Vault Secrets User role." +fi -# Authenticate with Azure using managed identity -az login --identity --client-id ${managedIdentityClientId} -# Using az storage blob upload-batch to upload files with managed identity authentication, as the az storage fs directory upload command is not working with managed identity authentication. -az storage blob upload-batch --account-name "$storageAccount" --destination data/"$extractedFolder1" --source /mnt/azscripts/azscriptinput/"$extractedFolder1" --auth-mode login --pattern '*' --overwrite -az storage blob upload-batch --account-name "$storageAccount" --destination data/"$extractedFolder2" --source /mnt/azscripts/azscriptinput/"$extractedFolder2" --auth-mode login --pattern '*' --overwrite \ No newline at end of file +# Upload files to Azure Storage +echo "Uploading files to Azure Storage" +az storage blob upload-batch --account-name "$storageAccount" --destination "$fileSystem"/"$extractedFolder1" --source infra/data/"$extractedFolder1" --auth-mode login --pattern '*' --overwrite --output none +az storage blob upload-batch --account-name "$storageAccount" --destination "$fileSystem"/"$extractedFolder2" --source infra/data/"$extractedFolder2" --auth-mode login --pattern '*' --overwrite --output none \ No newline at end of file diff --git a/infra/scripts/index_scripts/01_create_search_index.py b/infra/scripts/index_scripts/01_create_search_index.py index e40e09570..2b7e17d84 100644 --- a/infra/scripts/index_scripts/01_create_search_index.py +++ b/infra/scripts/index_scripts/01_create_search_index.py @@ -1,8 +1,9 @@ from azure.keyvault.secrets import SecretClient from azure.identity import DefaultAzureCredential +import sys -key_vault_name = 'kv_to-be-replaced' -managed_identity_client_id = 'mici_to-be-replaced' +key_vault_name=sys.argv[1] +managed_identity_client_id = sys.argv[2] index_name = "call_transcripts_index" def get_secrets_from_kv(kv_name, secret_name): diff --git a/infra/scripts/index_scripts/02_create_cu_template_audio.py b/infra/scripts/index_scripts/02_create_cu_template_audio.py index a320ba936..c11986ebf 100644 --- a/infra/scripts/index_scripts/02_create_cu_template_audio.py +++ b/infra/scripts/index_scripts/02_create_cu_template_audio.py @@ -7,8 +7,8 @@ from pathlib import Path from azure.identity import DefaultAzureCredential, get_bearer_token_provider -key_vault_name = 'kv_to-be-replaced' -managed_identity_client_id = 'mici_to-be-replaced' +key_vault_name=sys.argv[1] +managed_identity_client_id = sys.argv[2] def get_secrets_from_kv(kv_name, secret_name): diff --git a/infra/scripts/index_scripts/02_create_cu_template_text.py b/infra/scripts/index_scripts/02_create_cu_template_text.py index 9efc02ca2..835e09538 100644 --- a/infra/scripts/index_scripts/02_create_cu_template_text.py +++ b/infra/scripts/index_scripts/02_create_cu_template_text.py @@ -7,8 +7,8 @@ from pathlib import Path from azure.identity import DefaultAzureCredential, get_bearer_token_provider -key_vault_name = 'kv_to-be-replaced' -managed_identity_client_id = 'mici_to-be-replaced' +key_vault_name=sys.argv[1] +managed_identity_client_id = sys.argv[2] def get_secrets_from_kv(kv_name, secret_name): diff --git a/infra/scripts/index_scripts/03_cu_process_data_text.py b/infra/scripts/index_scripts/03_cu_process_data_text.py index 2855bb6c4..e28ce64dd 100644 --- a/infra/scripts/index_scripts/03_cu_process_data_text.py +++ b/infra/scripts/index_scripts/03_cu_process_data_text.py @@ -13,9 +13,10 @@ import base64 import pyodbc import struct +import sys -key_vault_name = 'kv_to-be-replaced' -managed_identity_client_id = 'mici_to-be-replaced' +key_vault_name=sys.argv[1] +managed_identity_client_id = sys.argv[2] file_system_client_name = "data" directory = 'call_transcripts' diff --git a/infra/scripts/process_sample_data.sh b/infra/scripts/process_sample_data.sh new file mode 100644 index 000000000..6dfa9a629 --- /dev/null +++ b/infra/scripts/process_sample_data.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -e # Exit on first error +set -o pipefail +set -u # Treat unset variables as error + +# === Configuration Parameters === +STORAGE_ACCOUNT_NAME="$1" +CONTAINER_NAME="$2" +# BASE_URL="$3" +MANAGED_IDENTITY_CLIENT_ID="$3" +KEY_VAULT_NAME="$4" +SQL_SERVER_NAME="$5" +SQL_DB_NAME="$6" +RG_NAME="$7" + +# === Functions === +log() { + echo -e "\033[1;32m[INFO]\033[0m $1" +} + +error() { + echo -e "\033[1;31m[ERROR]\033[0m $1" >&2 + exit 1 +} + +trap 'error "An unexpected error occurred. Please check the logs."' ERR + +# basePath="C:/Users/$(whoami)/azscripts/azscriptinput" +# echo "${basePath}" + +# === Step 1: Copy KB files === +echo "Running copy_kb_files.sh" +bash infra/scripts/copy_kb_files.sh "$STORAGE_ACCOUNT_NAME" "$CONTAINER_NAME" "$MANAGED_IDENTITY_CLIENT_ID" +if [ $? -ne 0 ]; then + echo "Error: copy_kb_files.sh failed." + exit 1 +fi +echo "copy_kb_files.sh completed successfully." + +# === Step 2: Run create index scripts === +log "Creating indexes..." +echo "Running run_create_index_scripts.sh" +bash infra/scripts/run_create_index_scripts.sh "$KEY_VAULT_NAME" "$MANAGED_IDENTITY_CLIENT_ID" "$SQL_SERVER_NAME" "$RG_NAME" +if [ $? -ne 0 ]; then + echo "Error: run_create_index_scripts.sh failed." + exit 1 +fi +echo "run_create_index_scripts.sh completed successfully." + + +# curl -s -o create-sql-user-and-role.ps1 "${BASE_URL}infra/scripts/add_user_scripts/create-sql-user-and-role.ps1" +# chmod +x create-sql-user-and-role.ps1 + +# Note: You'll need to pass user info (client ID, display name, role) via environment vars or args. +# Here is a sample with hardcoded values for demo: + +# === Step 3: SQL User & Role Setup === +log "Setting up SQL users and roles..." + +pwsh -File ./infra/scripts/add_user_scripts/create-sql-user-and-role.ps1 \ + -SqlServerName "$SQL_SERVER_NAME" \ + -SqlDatabaseName "$SQL_DB_NAME" \ + -ClientId "$MANAGED_IDENTITY_CLIENT_ID" \ + -DisplayName "script-user" \ + -ManagedIdentityClientId "$MANAGED_IDENTITY_CLIENT_ID" \ + -DatabaseRole "db_datawriter" \ + +log "Sample data processing completed successfully!" diff --git a/infra/scripts/run_create_index_scripts.sh b/infra/scripts/run_create_index_scripts.sh index 483c7f7c2..68a246de3 100644 --- a/infra/scripts/run_create_index_scripts.sh +++ b/infra/scripts/run_create_index_scripts.sh @@ -2,52 +2,146 @@ echo "started the script" # Variables -baseUrl="$1" -keyvaultName="$2" -managedIdentityClientId="$3" -requirementFile="requirements.txt" -requirementFileUrl=${baseUrl}"infra/scripts/index_scripts/requirements.txt" +# baseUrl="$1" +keyvaultName="$1" +managedIdentityClientId="$2" +serverName="$3" +resourceGroup="$4" +# requirementFile="requirements.txt" +# requirementFileUrl=${baseUrl}"infra/scripts/index_scripts/requirements.txt" echo "Script Started" -# Download the create_index and create table python files -curl --output "01_create_search_index.py" ${baseUrl}"infra/scripts/index_scripts/01_create_search_index.py" -curl --output "02_create_cu_template_text.py" ${baseUrl}"infra/scripts/index_scripts/02_create_cu_template_text.py" -curl --output "02_create_cu_template_audio.py" ${baseUrl}"infra/scripts/index_scripts/02_create_cu_template_audio.py" -curl --output "03_cu_process_data_text.py" ${baseUrl}"infra/scripts/index_scripts/03_cu_process_data_text.py" -curl --output "content_understanding_client.py" ${baseUrl}"infra/scripts/index_scripts/content_understanding_client.py" -curl --output "ckm-analyzer_config_text.json" ${baseUrl}"infra/data/ckm-analyzer_config_text.json" -curl --output "ckm-analyzer_config_audio.json" ${baseUrl}"infra/data/ckm-analyzer_config_audio.json" +# Authenticate with Azure +if az account show &> /dev/null; then + echo "Already authenticated with Azure." +else + if [ -n "$managedIdentityClientId" ]; then + # Use managed identity if running in Azure + echo "Authenticating with Managed Identity..." + az login --identity --client-id ${managedIdentityClientId} + else + # Use Azure CLI login if running locally + echo "Authenticating with Azure CLI..." + az login + fi + echo "Not authenticated with Azure. Attempting to authenticate..." +fi -curl --output "sample_processed_data.json" ${baseUrl}"infra/data/sample_processed_data.json" -curl --output "sample_processed_data_key_phrases.json" ${baseUrl}"infra/data/sample_processed_data_key_phrases.json" -curl --output "sample_search_index_data.json" ${baseUrl}"infra/data/sample_search_index_data.json" +echo "Getting signed in user id" +signed_user_id=$(az ad signed-in-user show --query id --output tsv) -# RUN apt-get update -# RUN apt-get install python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev -# apk add python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev - -# # RUN apt-get install python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev -# pip install pyodbc +echo "Getting key vault resource id" +key_vault_resource_id=$(az keyvault show --name $keyvaultName --query id --output tsv) -# Download the requirement file -curl --output "$requirementFile" "$requirementFileUrl" +echo "Checking if user has the Key Vault Administrator role" +role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --assignee $signed_user_id --role "Key Vault Administrator" --scope $key_vault_resource_id --query "[].roleDefinitionId" -o tsv) +if [ -z "$role_assignment" ]; then + echo "User does not have the Key Vault Administrator role. Assigning the role." + MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role "Key Vault Administrator" --scope $key_vault_resource_id --output none + if [ $? -eq 0 ]; then + echo "Key Vault Administrator role assigned successfully." + else + echo "Failed to assign Key Vault Administrator role." + exit 1 + fi +else + echo "User already has the Key Vault Administrator role." +fi + + +# RUN apt-get install python3 python3-dev g++ unixodbc-dev unixodbc libpq-dev +pip install pyodbc echo "Download completed" #Replace key vault name sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "01_create_search_index.py" -sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "01_create_search_index.py" sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "02_create_cu_template_text.py" -sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "02_create_cu_template_text.py" sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "02_create_cu_template_audio.py" -sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "02_create_cu_template_audio.py" sed -i "s/kv_to-be-replaced/${keyvaultName}/g" "03_cu_process_data_text.py" -sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "03_cu_process_data_text.py" +if [ -n "$managedIdentityClientId" ]; then + sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "01_create_search_index.py" + sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "02_create_cu_template_text.py" + sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "02_create_cu_template_audio.py" + sed -i "s/mici_to-be-replaced/${managedIdentityClientId}/g" "03_cu_process_data_text.py" +fi + +# create virtual environment +# Check if the virtual environment already exists +if [ -d "infra/scripts/scriptenv" ]; then + echo "Virtual environment already exists. Skipping creation." +else + echo "Creating virtual environment" + python3 -m venv infra/scripts/scriptenv +fi + +# handling virtual environment activation for different OS +activate_env_output=$(source infra/scripts/scriptenv/bin/activate 2>&1) +if [ -n "$activate_env_output" ]; then + source infra/scripts/scriptenv/Scripts/activate +fi + +# Install the requirements +echo "Installing requirements" +pip install --quiet -r infra/scripts/index_scripts/requirements.txt +echo "Requirements installed" + +echo "Running the python scripts" +echo "Creating the search index" +python infra/scripts/index_scripts/01_create_search_index.py "$keyvaultName" "$managedIdentityClientId" +if [ $? -ne 0 ]; then + echo "Error: 01_create_search_index.py failed." + exit 1 +fi + +echo "Processing the data" +python infra/scripts/index_scripts/02_create_cu_template_text.py "$keyvaultName" "$managedIdentityClientId" +if [ $? -ne 0 ]; then + echo "Error: 02_create_cu_template_text.py failed." + exit 1 +fi + +echo "Processing the data" +python infra/scripts/index_scripts/02_create_cu_template_audio.py "$keyvaultName" "$managedIdentityClientId" +if [ $? -ne 0 ]; then + echo "Error: 02_create_cu_template_audio.py failed." + exit 1 +fi + +echo "Processing the data" + +# Authenticate with Azure +if az account show &> /dev/null; then + echo "Already authenticated with Azure." +else + if [ -n "$managedIdentityClientId" ]; then + echo "Authenticating with Managed Identity..." + az login --identity --client-id ${managedIdentityClientId} + else + echo "Authenticating with Azure CLI..." + az login --use-device-code + fi + echo "Not authenticated with Azure. Attempting to authenticate..." +fi + +user=$(az account show --query user.name --output tsv) + +# Get the signed-in user's object ID +objectId=$(az ad signed-in-user show --query id --output tsv) + +az sql server ad-admin create \ + --resource-group "$resourceGroup" \ + --server "$serverName" \ + --display-name "$user" \ + --object-id "$objectId" + +echo "✅ Set $user as Azure SQL Server AAD admin." -pip install -r requirements.txt +python infra/scripts/index_scripts/03_cu_process_data_text.py "$keyvaultName" "$managedIdentityClientId" +if [ $? -ne 0 ]; then + echo "Error: 03_cu_process_data_text.py failed." + exit 1 +fi -python 01_create_search_index.py -python 02_create_cu_template_text.py -python 02_create_cu_template_audio.py -python 03_cu_process_data_text.py \ No newline at end of file +echo "Scripts completed" \ No newline at end of file