diff --git a/documents/DeploymentGuide.md b/documents/DeploymentGuide.md index c2af4c3ea..16d295a6e 100644 --- a/documents/DeploymentGuide.md +++ b/documents/DeploymentGuide.md @@ -147,7 +147,7 @@ If you're not using one of the above options for opening the project, then you'l 1. Make sure the following tools are installed: - [PowerShell](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-7.5) (v7.0+) - available for Windows, macOS, and Linux. - [Azure Developer CLI (azd)](https://aka.ms/install-azd) (v1.18.0+) - version - - [Python 3.9 to 3.11](https://www.python.org/downloads/) + - [Python 3.9+](https://www.python.org/downloads/) - [Docker Desktop](https://www.docker.com/products/docker-desktop/) - [Git](https://git-scm.com/downloads) - [Microsoft ODBC Driver 18](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16) for SQL Server. @@ -248,7 +248,32 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain - This deployment generally takes **7-10 minutes** to provision the resources in your account and set up the solution. - If you encounter an error or timeout during deployment, changing the location may help, as there could be availability constraints for the resources. -5. Once the deployment has completed successfully and you would like to use the **sample data**, please open a **Git Bash terminal** and run the bash command printed below. The bash command will look like the following: +5. Once the deployment has completed successfully, copy the bash command from terminal: (ex: `bash ./infra/scripts/process_sample_data.sh`) for later use. + +> **Note**: If you are running this deployment in GitHub Codespaces or VS Code Dev Container or Visual Studio Code (WEB) skip to step 7. + +6. Create and activate a virtual environment in bash terminal: + + ```shell + python -m venv .venv + ``` + + ```shell + source .venv/Scripts/activate + ``` + +7. Login to Azure: + + ```shell + az login + ``` + + Alternatively, login to Azure using a device code (recommended when using VS Code Web): + + ```shell + az login --use-device-code + ``` +8. Run the bash script from the output of the azd deployment. The script will look like the following: ```bash bash ./infra/scripts/process_sample_data.sh @@ -267,9 +292,9 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain ``` -6. Once the deployment has completed successfully, open the [Azure Portal](https://portal.azure.com/), go to the deployed resource group, find the App Service, and get the app URL from `Default domain`. +9. Once the deployment has completed successfully, open the [Azure Portal](https://portal.azure.com/), go to the deployed resource group, find the App Service, and get the app URL from `Default domain`. -7. You can now delete the resources by running `azd down`, if you are done trying out the application. +10. You can now delete the resources by running `azd down`, if you are done trying out the application. > **Note:** If you deployed with `enableRedundancy=true` and Log Analytics workspace replication is enabled, you must first disable replication before running `azd down` else resource group delete will fail. Follow the steps in [Handling Log Analytics Workspace Deletion with Replication Enabled](./LogAnalyticsReplicationDisable.md), wait until replication returns `false`, then run `azd down`. ### šŸ› ļø Troubleshooting diff --git a/documents/LocalDebuggingSetup.md b/documents/LocalDebuggingSetup.md index 298f692c7..91e9716bb 100644 --- a/documents/LocalDebuggingSetup.md +++ b/documents/LocalDebuggingSetup.md @@ -11,7 +11,7 @@ Install these tools before you start: - [Azure Tools](https://marketplace.visualstudio.com/items?itemName=ms-vscode.vscode-node-azure-pack) - [Bicep](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-bicep) - [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python) -- [Python 3.9 to 3.11](https://www.python.org/downloads/). **Important:** Check "Add Python to PATH" during installation. +- [Python 3.9+](https://www.python.org/downloads/). **Important:** Check "Add Python to PATH" during installation. - [PowerShell 7.0+](https://github.com/PowerShell/PowerShell#get-powershell). - [Node.js (LTS)](https://nodejs.org/en). - [Git](https://git-scm.com/downloads). diff --git a/infra/scripts/add_user_scripts/assign_sql_roles.py b/infra/scripts/add_user_scripts/assign_sql_roles.py index 2cc14416d..2c42e7a15 100644 --- a/infra/scripts/add_user_scripts/assign_sql_roles.py +++ b/infra/scripts/add_user_scripts/assign_sql_roles.py @@ -55,12 +55,9 @@ def assign_sql_roles(server, database, roles_json): credential = AzureCliCredential() # Connect to SQL Server - print(f"Connecting to {server}/{database}...") conn = connect_with_token(server, database, credential) cursor = conn.cursor() - print("Connected successfully.") - # Process each role assignment for role_assignment in roles: client_id = role_assignment.get("clientId") @@ -68,11 +65,8 @@ def assign_sql_roles(server, database, roles_json): role = role_assignment.get("role") if not client_id or not display_name or not role: - print(f"Skipping invalid role assignment: {role_assignment}") continue - print(f"\nProcessing: {display_name} -> {role}") - # Check if user already exists check_user_sql = f"SELECT COUNT(*) FROM sys.database_principals WHERE name = '{display_name}'" cursor.execute(check_user_sql) @@ -81,16 +75,13 @@ def assign_sql_roles(server, database, roles_json): if not user_exists: # Create user from external provider create_user_sql = f"CREATE USER [{display_name}] FROM EXTERNAL PROVIDER" - print(f" Creating user: {display_name}") try: cursor.execute(create_user_sql) conn.commit() - print(" āœ“ User created successfully") + print(f"āœ“ Created user: {display_name}") except Exception as e: - print(f" āœ— Failed to create user: {e}") + print(f"āœ— Failed to create user: {e}") continue - else: - print(f" User already exists: {display_name}") # Check if user already has the role check_role_sql = f""" @@ -106,21 +97,17 @@ def assign_sql_roles(server, database, roles_json): if not has_role: # Add user to role add_role_sql = f"ALTER ROLE [{role}] ADD MEMBER [{display_name}]" - print(f" Assigning role: {role}") try: cursor.execute(add_role_sql) conn.commit() - print(" āœ“ Role assigned successfully") + print(f"āœ“ Assigned {role} to {display_name}") except Exception as e: - print(f" āœ— Failed to assign role: {e}") + print(f"āœ— Failed to assign {role}: {e}") continue - else: - print(f" User already has role: {role}") # Close connection cursor.close() conn.close() - print("\nāœ“ All role assignments completed successfully") return 0 except Exception as e: diff --git a/infra/scripts/copy_kb_files.sh b/infra/scripts/copy_kb_files.sh index c3638363e..fadc44cda 100644 --- a/infra/scripts/copy_kb_files.sh +++ b/infra/scripts/copy_kb_files.sh @@ -11,11 +11,6 @@ extractedFolder1="call_transcripts" zipFileName2="infra/data/audio_data.zip" extractedFolder2="audio_data" -echo "Script Started" -echo "Storage Account: $storageAccountName" -echo "Container Name: $containerName" -echo "Resource Group: $resourceGroupName" - # Validate required parameters if [ -z "$storageAccountName" ] || [ -z "$containerName" ] || [ -z "$resourceGroupName" ]; then echo "Error: Missing required parameters." @@ -25,102 +20,76 @@ fi # Extract zip files if they exist if [ -f "$zipFileName1" ]; then - echo "Extracting $zipFileName1..." - unzip -o "$zipFileName1" -d "$extractedFolder1" -else - echo "Warning: $zipFileName1 not found. Skipping extraction." + unzip -q -o "$zipFileName1" -d "$extractedFolder1" fi if [ -f "$zipFileName2" ]; then - echo "Extracting $zipFileName2..." - unzip -o "$zipFileName2" -d "$extractedFolder2" -else - echo "Warning: $zipFileName2 not found. Skipping extraction." + unzip -q -o "$zipFileName2" -d "$extractedFolder2" fi # Authenticate with Azure -if az account show &> /dev/null; then - echo "Already authenticated with Azure." -else +if ! az account show &> /dev/null; then echo "Authenticating with Azure CLI..." - az login + az login --use-device-code fi # Check and assign Storage Blob Data Contributor role to current user -echo "Checking Storage Blob Data Contributor role assignment..." signed_user_id=$(az ad signed-in-user show --query id --output tsv) storage_resource_id=$(az storage account show --name "$storageAccountName" --resource-group "$resourceGroupName" --query id --output tsv) role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --assignee $signed_user_id --role "Storage Blob Data Contributor" --scope $storage_resource_id --query "[].roleDefinitionId" -o tsv) if [ -z "$role_assignment" ]; then - echo "Assigning Storage Blob Data Contributor role to current user..." + echo "āœ“ Assigning Storage Blob Data Contributor role" MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role "Storage Blob Data Contributor" --scope $storage_resource_id --output none - if [ $? -eq 0 ]; then - echo "Storage Blob Data Contributor role assigned successfully." - # Wait a bit for role assignment to propagate - echo "Waiting for role assignment to propagate..." - sleep 10 - else - echo "Failed to assign Storage Blob Data Contributor role." + if [ $? -ne 0 ]; then + echo "āœ— Failed to assign Storage Blob Data Contributor role" exit 1 fi -else - echo "User already has Storage Blob Data Contributor role." + sleep 10 fi # Upload files to storage account -# Using az storage blob upload-batch to upload files with Azure CLI authentication -echo "Uploading call transcripts to storage account..." if [ -d "$extractedFolder1" ]; then + echo "āœ“ Uploading call transcripts" az storage blob upload-batch \ --account-name "$storageAccountName" \ --destination "$containerName/$extractedFolder1" \ --source "$extractedFolder1" \ --auth-mode login \ --pattern '*' \ - --overwrite - if [ $? -eq 0 ]; then - echo "āœ“ Call transcripts uploaded successfully" - else + --overwrite \ + --output none + if [ $? -ne 0 ]; then echo "āœ— Failed to upload call transcripts" exit 1 fi -else - echo "Warning: $extractedFolder1 directory not found. Skipping upload." fi -echo "Uploading audio data to storage account..." if [ -d "$extractedFolder2" ]; then + echo "āœ“ Uploading audio data" az storage blob upload-batch \ --account-name "$storageAccountName" \ --destination "$containerName/$extractedFolder2" \ --source "$extractedFolder2" \ --auth-mode login \ --pattern '*' \ - --overwrite - if [ $? -eq 0 ]; then - echo "āœ“ Audio data uploaded successfully" - else + --overwrite \ + --output none + if [ $? -ne 0 ]; then echo "āœ— Failed to upload audio data" exit 1 fi -else - echo "Warning: $extractedFolder2 directory not found. Skipping upload." fi # Create custom data directories for user uploads -echo "Creating custom data directories..." az storage fs directory create \ --account-name "$storageAccountName" \ --file-system "$containerName" \ --name custom_audiodata \ - --auth-mode login 2>/dev/null || echo "custom_audiodata directory may already exist" + --auth-mode login --output none 2>/dev/null az storage fs directory create \ --account-name "$storageAccountName" \ --file-system "$containerName" \ --name custom_transcripts \ - --auth-mode login 2>/dev/null || echo "custom_transcripts directory may already exist" - -echo "āœ“ Custom data directories created successfully" -echo "Script completed successfully" \ No newline at end of file + --auth-mode login --output none 2>/dev/null \ No newline at end of file diff --git a/infra/scripts/index_scripts/01_create_search_index.py b/infra/scripts/index_scripts/01_create_search_index.py index 1deaaa4a1..ccb2f5b82 100644 --- a/infra/scripts/index_scripts/01_create_search_index.py +++ b/infra/scripts/index_scripts/01_create_search_index.py @@ -30,9 +30,6 @@ INDEX_NAME = "call_transcripts_index" -print("calling create_search_index()....") - - def create_search_index(): """ Creates or updates an Azure Cognitive Search index configured for: @@ -105,7 +102,7 @@ def create_search_index(): ) result = index_client.create_or_update_index(index) - print(f"Search index '{result.name}' created or updated successfully.") + print(f"āœ“ Search index '{result.name}' created") create_search_index() \ No newline at end of file diff --git a/infra/scripts/index_scripts/02_create_cu_template_audio.py b/infra/scripts/index_scripts/02_create_cu_template_audio.py index da46664ad..72279c29d 100644 --- a/infra/scripts/index_scripts/02_create_cu_template_audio.py +++ b/infra/scripts/index_scripts/02_create_cu_template_audio.py @@ -36,9 +36,9 @@ analyzer = client.get_analyzer_detail_by_id(ANALYZER_ID) if analyzer is not None: client.delete_analyzer(ANALYZER_ID) -except Exception as e: - print(f"Analyzer with ID {ANALYZER_ID} was not found. Proceeding to create a new one.") +except Exception: + pass response = client.begin_create_analyzer(ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE_FILE) result = client.poll_result(response) -print(f"Analyzer with ID {ANALYZER_ID} created successfully.") +print(f"āœ“ Analyzer '{ANALYZER_ID}' created") diff --git a/infra/scripts/index_scripts/02_create_cu_template_text.py b/infra/scripts/index_scripts/02_create_cu_template_text.py index df128f31e..a9080f2ed 100644 --- a/infra/scripts/index_scripts/02_create_cu_template_text.py +++ b/infra/scripts/index_scripts/02_create_cu_template_text.py @@ -31,9 +31,9 @@ analyzer = client.get_analyzer_detail_by_id(ANALYZER_ID) if analyzer is not None: client.delete_analyzer(ANALYZER_ID) -except Exception as e: - print(f"Analyzer with ID {ANALYZER_ID} was not found. Proceeding to create a new one.") +except Exception: + pass response = client.begin_create_analyzer(ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE_FILE) result = client.poll_result(response) -print(f"Analyzer with ID {ANALYZER_ID} created successfully.") +print(f"āœ“ Analyzer '{ANALYZER_ID}' created") diff --git a/infra/scripts/index_scripts/03_cu_process_data_text.py b/infra/scripts/index_scripts/03_cu_process_data_text.py index a16757b72..32715e614 100644 --- a/infra/scripts/index_scripts/03_cu_process_data_text.py +++ b/infra/scripts/index_scripts/03_cu_process_data_text.py @@ -49,8 +49,6 @@ SAMPLE_PROCESSED_DATA_FILE = 'infra/data/sample_processed_data.json' SAMPLE_PROCESSED_DATA_KEY_PHRASES_FILE = 'infra/data/sample_processed_data_key_phrases.json' -print("Parameters received.") - # Azure DataLake setup account_url = f"https://{STORAGE_ACCOUNT_NAME}.dfs.core.windows.net" credential = AzureCliCredential(process_timeout=30) @@ -58,13 +56,11 @@ file_system_client = service_client.get_file_system_client(FILE_SYSTEM_CLIENT_NAME) directory_name = DIRECTORY paths = list(file_system_client.get_paths(path=directory_name)) -print("Azure DataLake setup complete.") # Azure Search setup search_credential = AzureCliCredential(process_timeout=30) search_client = SearchClient(SEARCH_ENDPOINT, INDEX_NAME, search_credential) index_client = SearchIndexClient(endpoint=SEARCH_ENDPOINT, credential=search_credential) -print("Azure Search setup complete.") # SQL Server setup driver = "{ODBC Driver 18 for SQL Server}" @@ -74,7 +70,6 @@ connection_string = f"DRIVER={driver};SERVER={SQL_SERVER};DATABASE={SQL_DATABASE};" conn = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) cursor = conn.cursor() -print("SQL Server connection established.") # SQL data type mapping for pandas to SQL conversion sql_data_types = { @@ -170,7 +165,6 @@ def generate_sql_insert_script(df, table_name, columns, sql_file_name): token_provider=cu_token_provider ) ANALYZER_ID = "ckm-json" -print("Content Understanding client initialized.") # Azure AI Foundry (Inference) clients (Managed Identity) inference_endpoint = f"https://{urlparse(AI_PROJECT_ENDPOINT).netloc}/models" @@ -289,7 +283,7 @@ def create_tables(): StartTime varchar(255) );""") conn.commit() - print("Database tables created.") + create_tables() @@ -332,12 +326,10 @@ def create_tables(): if docs != [] and counter % 10 == 0: result = search_client.upload_documents(documents=docs) docs = [] - print(f'{counter} uploaded to Azure Search.') if docs: search_client.upload_documents(documents=docs) - print(f'Final batch uploaded to Azure Search.') -print("File processing and DB/Search insertion complete.") +print(f"āœ“ Processed {counter} files") # Load sample data to search index and database def bulk_import_json_to_table(json_file, table_name): @@ -345,7 +337,6 @@ def bulk_import_json_to_table(json_file, table_name): data = json.load(f) if not data: - print(f"No data to import into {table_name}.") return df = pd.DataFrame(data) @@ -356,11 +347,10 @@ def bulk_import_json_to_table(json_file, table_name): documents = json.load(file) batch = [{"@search.action": "upload", **doc} for doc in documents] search_client.upload_documents(documents=batch) -print(f'Successfully uploaded {len(documents)} sample index data records to search index {INDEX_NAME}.') bulk_import_json_to_table(SAMPLE_PROCESSED_DATA_FILE, 'processed_data') bulk_import_json_to_table(SAMPLE_PROCESSED_DATA_KEY_PHRASES_FILE, 'processed_data_key_phrases') -print("Sample data loaded to DB and Search.") +print(f"āœ“ Loaded {len(documents)} sample records") # Topic mining and mapping cursor.execute('SELECT distinct topic FROM processed_data') @@ -374,7 +364,6 @@ def bulk_import_json_to_table(json_file, table_name): );""") conn.commit() topics_str = ', '.join(df['topic'].tolist()) -print("Topic mining table prepared.") def call_gpt4(topics_str1, client): @@ -410,7 +399,6 @@ def call_gpt4(topics_str1, client): for object1 in res['topics']: cursor.execute("INSERT INTO km_mined_topics (label, description) VALUES (?,?)", (object1['label'], object1['description'])) conn.commit() -print("Topics mined and inserted into km_mined_topics.") cursor.execute('SELECT label FROM km_mined_topics') rows = [tuple(row) for row in cursor.fetchall()] @@ -418,7 +406,7 @@ def call_gpt4(topics_str1, client): df_topics = pd.DataFrame(rows, columns=column_names) mined_topics_list = df_topics['label'].tolist() mined_topics = ", ".join(mined_topics_list) -print("Mined topics loaded.") +print(f"āœ“ Mined {len(mined_topics_list)} topics") def get_mined_topic_mapping(input_text, list_of_topics): @@ -445,7 +433,6 @@ def get_mined_topic_mapping(input_text, list_of_topics): mined_topic_str = get_mined_topic_mapping(row['topic'], str(mined_topics_list)) cursor.execute("UPDATE processed_data SET mined_topic = ? WHERE ConversationId = ?", (mined_topic_str, row['ConversationId'])) conn.commit() -print("Processed data mapped to mined topics.") # Update processed data for RAG cursor.execute('DROP TABLE IF EXISTS km_processed_data') @@ -472,7 +459,6 @@ def get_mined_topic_mapping(input_text, list_of_topics): generate_sql_insert_script(df_km, 'km_processed_data', columns, 'km_processed_data_insert.sql') # Update processed_data_key_phrases table -print("Updating processed_data_key_phrases table") cursor.execute('''select ConversationId, key_phrases, sentiment, mined_topic as topic, StartTime from processed_data''') rows = [tuple(row) for row in cursor.fetchall()] column_names = [i[0] for i in cursor.description] @@ -485,7 +471,6 @@ def get_mined_topic_mapping(input_text, list_of_topics): cursor.execute("INSERT INTO processed_data_key_phrases (ConversationId, key_phrase, sentiment, topic, StartTime) VALUES (?,?,?,?,?)", (row['ConversationId'], key_phrase, row['sentiment'], row['topic'], row['StartTime'])) conn.commit() -print("processed_data_key_phrases table updated.") # Adjust dates to current date today = datetime.today() @@ -496,8 +481,7 @@ def get_mined_topic_mapping(input_text, list_of_topics): cursor.execute("UPDATE [dbo].[km_processed_data] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss'), EndTime = FORMAT(DATEADD(DAY, ?, EndTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference, days_difference)) cursor.execute("UPDATE [dbo].[processed_data_key_phrases] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference,)) conn.commit() -print("Dates adjusted to current date.") cursor.close() conn.close() -print("All steps completed. Connection closed.") +print("āœ“ Data processing completed") diff --git a/infra/scripts/index_scripts/04_cu_process_custom_data.py b/infra/scripts/index_scripts/04_cu_process_custom_data.py index 244e22741..ad44f7b0b 100644 --- a/infra/scripts/index_scripts/04_cu_process_custom_data.py +++ b/infra/scripts/index_scripts/04_cu_process_custom_data.py @@ -65,8 +65,6 @@ CU_ENDPOINT = args.cu_endpoint CU_API_VERSION = args.cu_api_version -print("Command-line arguments parsed.") - # Azure DataLake setup account_url = f"https://{STORAGE_ACCOUNT_NAME}.dfs.core.windows.net" credential = AzureCliCredential(process_timeout=30) @@ -74,13 +72,11 @@ file_system_client = service_client.get_file_system_client(FILE_SYSTEM_CLIENT_NAME) directory_name = DIRECTORY paths = list(file_system_client.get_paths(path=directory_name)) -print("Azure DataLake setup complete.") # Azure Search setup search_credential = AzureCliCredential(process_timeout=30) search_client = SearchClient(SEARCH_ENDPOINT, INDEX_NAME, search_credential) index_client = SearchIndexClient(endpoint=SEARCH_ENDPOINT, credential=search_credential) -print("Azure Search setup complete.") # Azure AI Foundry (Inference) clients (Managed Identity) inference_endpoint = f"https://{urlparse(AI_PROJECT_ENDPOINT).netloc}/models" @@ -170,7 +166,7 @@ def create_search_index(): ) result = index_client.create_or_update_index(index) - print(f"Search index '{result.name}' created or updated successfully.") + print(f"āœ“ Search index '{result.name}' created") create_search_index() @@ -182,7 +178,6 @@ def create_search_index(): connection_string = f"DRIVER={DRIVER};SERVER={SQL_SERVER};DATABASE={SQL_DATABASE};" conn = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) cursor = conn.cursor() -print("SQL Server connection established.") # Content Understanding client cu_credential = AzureCliCredential(process_timeout=30) @@ -192,7 +187,6 @@ def create_search_index(): api_version=CU_API_VERSION, token_provider=cu_token_provider ) -print("Content Understanding client initialized.") # Utility functions def get_embeddings(text: str): @@ -273,7 +267,6 @@ def generate_sql_insert_script(df, table_name, columns, sql_file_name): conn.commit() record_count = len(df) - print(f"Inserted {record_count} records into {table_name} using optimized SQL script.") return record_count def clean_spaces_with_regex(text): @@ -346,7 +339,6 @@ def create_tables(): StartTime varchar(255) );""") conn.commit() - print("Database tables created.") create_tables() @@ -389,19 +381,16 @@ def create_tables(): if docs != [] and counter % 10 == 0: result = search_client.upload_documents(documents=docs) docs = [] - print(f'{counter} uploaded to Azure Search.') if docs: search_client.upload_documents(documents=docs) - print(f'Final batch uploaded to Azure Search - transcripts.') -print("File processing and DB/Search insertion complete - transcripts.") +print(f"āœ“ Processed {counter} transcript files") # Process files for audio data ANALYZER_ID = "ckm-audio" directory_name = AUDIO_DIRECTORY paths = list(file_system_client.get_paths(path=directory_name)) -print("Processing audio files") docs = [] counter = 0 # process and upload audio files to search index - audio data @@ -443,22 +432,18 @@ def create_tables(): docs.extend(prepare_search_doc(content, document_id, path.name)) counter += 1 - print(f"Processed file {path.name} successfully.") except Exception as e: - print(f"Error processing file {path.name}: {e}") pass if docs != [] and counter % 10 == 0: result = search_client.upload_documents(documents=docs) docs = [] - print(f' {str(counter)} uploaded') # upload the last batch if docs != []: search_client.upload_documents(documents=docs) - print(f'Final batch uploaded to Azure Search - audio data.') -print("File processing and DB/Search insertion complete - audio data.") +print(f"āœ“ Processed {counter} audio files") # Topic mining and mapping cursor.execute('SELECT distinct topic FROM processed_data') @@ -472,7 +457,6 @@ def create_tables(): );""") conn.commit() topics_str = ', '.join(df['topic'].tolist()) -print("Topic mining table prepared.") def call_gpt4(topics_str1, client): topic_prompt = f""" @@ -506,7 +490,6 @@ def call_gpt4(topics_str1, client): for object1 in res['topics']: cursor.execute("INSERT INTO km_mined_topics (label, description) VALUES (?,?)", (object1['label'], object1['description'])) conn.commit() -print("Topics mined and inserted into km_mined_topics.") cursor.execute('SELECT label FROM km_mined_topics') rows = [tuple(row) for row in cursor.fetchall()] @@ -514,7 +497,6 @@ def call_gpt4(topics_str1, client): df_topics = pd.DataFrame(rows, columns=column_names) mined_topics_list = df_topics['label'].tolist() mined_topics = ", ".join(mined_topics_list) -print("Mined topics loaded.") def get_mined_topic_mapping(input_text, list_of_topics): @@ -541,7 +523,6 @@ def get_mined_topic_mapping(input_text, list_of_topics): mined_topic_str = get_mined_topic_mapping(row['topic'], str(mined_topics_list)) cursor.execute("UPDATE processed_data SET mined_topic = ? WHERE ConversationId = ?", (mined_topic_str, row['ConversationId'])) conn.commit() -print("Processed data mapped to mined topics.") # Update processed data for RAG cursor.execute('DROP TABLE IF EXISTS km_processed_data') @@ -565,11 +546,10 @@ def get_mined_topic_mapping(input_text, list_of_topics): "keyphrases", "complaint", "topic"] df_km = pd.DataFrame([list(row) for row in rows], columns=columns) -generate_sql_insert_script(df_km, 'km_processed_data', columns, 'km_processed_data_insert.sql') -print("km_processed_data table updated.") +record_count = generate_sql_insert_script(df_km, 'km_processed_data', columns, 'km_processed_data_insert.sql') +print(f"āœ“ Loaded {record_count} sample records") # Update processed_data_key_phrases table -print("Updating processed_data_key_phrases table") cursor.execute('''select ConversationId, key_phrases, sentiment, mined_topic as topic, StartTime from processed_data''') rows = [tuple(row) for row in cursor.fetchall()] column_names = [i[0] for i in cursor.description] @@ -582,7 +562,6 @@ def get_mined_topic_mapping(input_text, list_of_topics): cursor.execute("INSERT INTO processed_data_key_phrases (ConversationId, key_phrase, sentiment, topic, StartTime) VALUES (?,?,?,?,?)", (row['ConversationId'], key_phrase, row['sentiment'], row['topic'], row['StartTime'])) conn.commit() -print("processed_data_key_phrases table updated.") # Adjust dates to current date today = datetime.today() @@ -593,8 +572,7 @@ def get_mined_topic_mapping(input_text, list_of_topics): cursor.execute("UPDATE [dbo].[km_processed_data] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss'), EndTime = FORMAT(DATEADD(DAY, ?, EndTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference, days_difference)) cursor.execute("UPDATE [dbo].[processed_data_key_phrases] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference,)) conn.commit() -print("Dates adjusted to current date.") cursor.close() conn.close() -print("All steps completed. Connection closed.") +print("āœ“ Data processing completed") diff --git a/infra/scripts/index_scripts/requirements.txt b/infra/scripts/index_scripts/requirements.txt index b2735a540..3fa8d865d 100644 --- a/infra/scripts/index_scripts/requirements.txt +++ b/infra/scripts/index_scripts/requirements.txt @@ -6,7 +6,6 @@ azure-ai-inference==1.0.0b9 pypdf==5.6.0 # pyodbc tiktoken==0.9.0 -msal[broker]==1.32.3 azure-identity==1.23.0 azure-ai-textanalytics==5.3.0 azure-search-documents==11.5.2 diff --git a/infra/scripts/process_custom_data.sh b/infra/scripts/process_custom_data.sh index b6f3f4771..24ff86f25 100644 --- a/infra/scripts/process_custom_data.sh +++ b/infra/scripts/process_custom_data.sh @@ -49,10 +49,8 @@ original_full_range_rule_present="false" # Function to enable public network access temporarily enable_public_access() { - echo "=== Temporarily enabling public network access for services ===" # Enable public access for Storage Account - echo "Enabling public access for Storage Account: $storageAccountName" original_storage_public_access=$(az storage account show \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ @@ -65,37 +63,30 @@ enable_public_access() { -o tsv) if [ "$original_storage_public_access" != "Enabled" ]; then + echo "āœ“ Enabling Storage Account public access" az storage account update \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ --public-network-access Enabled \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account public access enabled" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to enable Storage Account public access" return 1 fi - else - echo "āœ“ Storage Account public access already enabled" fi # Also ensure the default network action allows access if [ "$original_storage_default_action" != "Allow" ]; then - echo "Setting Storage Account network default action to Allow" + echo "āœ“ Setting Storage Account network default action to Allow" az storage account update \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ --default-action Allow \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account network default action set to Allow" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to set Storage Account network default action" return 1 fi - else - echo "āœ“ Storage Account network default action already set to Allow" fi # Enable public access for AI Foundry @@ -113,21 +104,16 @@ enable_public_access() { --output tsv) if [ -z "$original_foundry_public_access" ] || [ "$original_foundry_public_access" = "null" ]; then - echo "⚠ Info: Could not retrieve AI Foundry network access status." + echo "⚠ Could not retrieve AI Foundry network access status" elif [ "$original_foundry_public_access" != "Enabled" ]; then - echo "Current AI Foundry public access: $original_foundry_public_access" - echo "Enabling public access for AI Foundry resource: $aif_resource_name (Resource Group: $aif_resource_group)" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Enabling AI Foundry public access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$aif_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \ --output none; then - echo "āœ“ AI Foundry public access enabled" - else - echo "⚠ Warning: Failed to enable AI Foundry public access automatically." + echo "⚠ Failed to enable AI Foundry public access" fi - else - echo "āœ“ AI Foundry public access already enabled" fi fi @@ -146,26 +132,20 @@ enable_public_access() { --output tsv) if [ -z "$original_cu_foundry_public_access" ] || [ "$original_cu_foundry_public_access" = "null" ]; then - echo "⚠ Info: Could not retrieve CU Foundry network access status." + echo "⚠ Could not retrieve CU Foundry network access status" elif [ "$original_cu_foundry_public_access" != "Enabled" ]; then - echo "Current CU Foundry public access: $original_cu_foundry_public_access" - echo "Enabling public access for CU Foundry resource: $cu_resource_name (Resource Group: $cu_resource_group)" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Enabling CU Foundry public access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$cu_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \ --output none; then - echo "āœ“ CU Foundry public access enabled" - else - echo "⚠ Warning: Failed to enable CU Foundry public access automatically." + echo "⚠ Failed to enable CU Foundry public access" fi - else - echo "āœ“ CU Foundry public access already enabled" fi fi # Enable public access for SQL Server - echo "Checking SQL Server public access: $sqlServerName" original_sql_public_access=$(az sql server show \ --name "$sqlServerName" \ --resource-group "$resourceGroupName" \ @@ -173,15 +153,16 @@ enable_public_access() { -o tsv) if [ "$original_sql_public_access" != "Enabled" ]; then - echo "Enabling public access for SQL Server" + echo "āœ“ Enabling SQL Server public access" az sql server update \ --name "$sqlServerName" \ --resource-group "$resourceGroupName" \ --enable-public-network true \ --output none - echo "āœ“ SQL Server public access enabled" - else - echo "āœ“ SQL Server public access already enabled" + if [ $? -ne 0 ]; then + echo "āœ— Failed to enable SQL Server public access" + return 1 + fi fi # Create temporary allow-all firewall rule for SQL Server @@ -204,43 +185,34 @@ enable_public_access() { --query "[?name=='${sql_allow_all_rule_name}'] | [0].name" \ -o tsv 2>/dev/null) - if [ -z "$existing_allow_all_rule" ]; then - if [ -n "$pre_existing_full_range_rule" ]; then - echo "āœ“ Existing rule ($pre_existing_full_range_rule) already allows full IP range." - else - echo "Creating temporary allow-all firewall rule ($sql_allow_all_rule_name)..." - if az sql server firewall-rule create \ - --resource-group "$resourceGroupName" \ - --server "$sqlServerName" \ - --name "$sql_allow_all_rule_name" \ - --start-ip-address 0.0.0.0 \ - --end-ip-address 255.255.255.255 \ - --output none; then - created_sql_allow_all_firewall_rule="true" - echo "āœ“ Temporary allow-all firewall rule created" - else - echo "⚠ Warning: Failed to create allow-all firewall rule" - fi - fi + if [ -z "$existing_allow_all_rule" ] && [ -z "$pre_existing_full_range_rule" ]; then + echo "āœ“ Creating temporary SQL firewall rule" + if az sql server firewall-rule create \ + --resource-group "$resourceGroupName" \ + --server "$sqlServerName" \ + --name "$sql_allow_all_rule_name" \ + --start-ip-address 0.0.0.0 \ + --end-ip-address 255.255.255.255 \ + --output none; then + created_sql_allow_all_firewall_rule="true" + else + echo "⚠ Failed to create firewall rule" + fi else - echo "āœ“ Temporary allow-all firewall rule already present" - original_full_range_rule_present="true" + original_full_range_rule_present="true" fi # Wait a bit for changes to take effect - echo "Waiting for network access changes to propagate..." sleep 10 - echo "=== Public network access enabled successfully ===" return 0 } # Function to restore original network access settings restore_network_access() { - echo "=== Restoring original network access settings ===" # Restore Storage Account access if [ -n "$original_storage_public_access" ] && [ "$original_storage_public_access" != "Enabled" ]; then - echo "Restoring Storage Account public access to: $original_storage_public_access" + echo "āœ“ Restoring Storage Account access" case "$original_storage_public_access" in "enabled"|"Enabled") restore_value="Enabled" ;; "disabled"|"Disabled") restore_value="Disabled" ;; @@ -251,74 +223,56 @@ restore_network_access() { --resource-group "$resourceGroupName" \ --public-network-access "$restore_value" \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account access restored" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to restore Storage Account access" fi - else - echo "Storage Account access unchanged (already at desired state)" fi # Restore Storage Account network default action if [ -n "$original_storage_default_action" ] && [ "$original_storage_default_action" != "Allow" ]; then - echo "Restoring Storage Account network default action to: $original_storage_default_action" + echo "āœ“ Restoring Storage Account network default action" az storage account update \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ --default-action "$original_storage_default_action" \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account network default action restored" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to restore Storage Account network default action" fi - else - echo "Storage Account network default action unchanged (already at desired state)" fi # Restore AI Foundry access if [ -n "$original_foundry_public_access" ] && [ "$original_foundry_public_access" != "Enabled" ]; then - echo "Restoring AI Foundry public access to: $original_foundry_public_access" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Restoring AI Foundry access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$aif_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess="$original_foundry_public_access" \ --set properties.apiProperties.qnaAzureSearchEndpointKey="" \ --set properties.networkAcls.bypass="AzureServices" \ --output none 2>/dev/null; then - echo "āœ“ AI Foundry access restored" - else - echo "⚠ Warning: Failed to restore AI Foundry access automatically." - echo " Please manually restore network access in the Azure portal if needed." + echo "⚠ Failed to restore AI Foundry access - please check Azure portal" fi - else - echo "AI Foundry access unchanged (already at desired state)" fi # Restore CU Foundry access if [ -n "$original_cu_foundry_public_access" ] && [ "$original_cu_foundry_public_access" != "Enabled" ]; then - echo "Restoring CU Foundry public access to: $original_cu_foundry_public_access" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Restoring CU Foundry access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$cu_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess="$original_cu_foundry_public_access" \ --set properties.apiProperties.qnaAzureSearchEndpointKey="" \ --set properties.networkAcls.bypass="AzureServices" \ --output none 2>/dev/null; then - echo "āœ“ CU Foundry access restored" - else - echo "⚠ Warning: Failed to restore CU Foundry access automatically." - echo " Please manually restore network access in the Azure portal if needed." + echo "⚠ Failed to restore CU Foundry access - please check Azure portal" fi - else - echo "CU Foundry access unchanged (already at desired state)" fi # Restore SQL Server public access if [ -n "$original_sql_public_access" ] && [ "$original_sql_public_access" != "Enabled" ]; then - echo "Restoring SQL Server public access to: $original_sql_public_access" + echo "āœ“ Restoring SQL Server access" case "$original_sql_public_access" in "enabled"|"Enabled") restore_value=true ;; "disabled"|"Disabled") restore_value=false ;; @@ -329,27 +283,10 @@ restore_network_access() { --resource-group "$resourceGroupName" \ --enable-public-network $restore_value \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ SQL Server access restored" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to restore SQL Server access" fi - else - echo "SQL Server access unchanged (already at desired state)" fi - - # Remove temporary allow-all firewall rule if we created it - if [ "$created_sql_allow_all_firewall_rule" = "true" ] && [ "$original_full_range_rule_present" != "true" ]; then - echo "Removing temporary allow-all firewall rule..." - az sql server firewall-rule delete \ - --resource-group "$resourceGroupName" \ - --server "$sqlServerName" \ - --name "TempAllowAll" \ - --output none 2>/dev/null - echo "āœ“ Temporary firewall rule removed" - fi - - echo "=== Network access restoration completed ===" } # Function to handle script cleanup on exit @@ -357,11 +294,9 @@ cleanup_on_exit() { exit_code=$? echo "" if [ $exit_code -ne 0 ]; then - echo "āŒ Script failed with exit code $exit_code" - echo "Restoring network access settings before exit..." + echo "āŒ Script failed" else echo "āœ… Script completed successfully" - echo "Restoring network access settings..." fi restore_network_access exit $exit_code @@ -380,7 +315,6 @@ check_azd_installed() { } get_values_from_azd_env() { - echo "Getting values from azd environment variables..." # Use grep with a regex to ensure we're only capturing sanitized values to avoid command injection resourceGroupName=$(azd env get-value RESOURCE_GROUP_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$') storageAccountName=$(azd env get-value STORAGE_ACCOUNT_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$') @@ -407,19 +341,19 @@ get_values_from_azd_env() { if [ -z "$resourceGroupName" ] || [ -z "$storageAccountName" ] || [ -z "$fileSystem" ] || [ -z "$sqlServerName" ] || [ -z "$SqlDatabaseName" ] || [ -z "$backendUserMidClientId" ] || [ -z "$backendUserMidDisplayName" ] || [ -z "$aiSearchName" ] || [ -z "$aif_resource_id" ]; then echo "Error: One or more required values could not be retrieved from azd environment." return 1 - else - echo "All values retrieved successfully from azd environment." - return 0 fi + return 0 } - - -# Use Azure CLI login if running locally -echo "" -echo "Authenticating with Azure CLI..." -az login --use-device-code -echo "" +# Check if user is logged in to Azure +echo "Checking Azure authentication..." +if az account show &> /dev/null; then + echo "Already authenticated with Azure." +else + # Use Azure CLI login if running locally + echo "Authenticating with Azure CLI..." + az login --use-device-code +fi if check_azd_installed; then azSubscriptionId=$(azd env get-value AZURE_SUBSCRIPTION_ID) || azSubscriptionId="$AZURE_SUBSCRIPTION_ID" || azSubscriptionId="" @@ -507,61 +441,31 @@ if [ $? -ne 0 ]; then exit 1 fi -# Determine the correct Python command -if command -v python3 &> /dev/null; then - PYTHON_CMD="python3" -elif command -v python &> /dev/null; then - PYTHON_CMD="python" -else - echo "Python is not installed on this system. Or it is not added in the PATH." - exit 1 -fi - pythonScriptPath="infra/scripts/index_scripts/" -# create virtual environment -# Check if the virtual environment already exists -if [ -d $pythonScriptPath"scriptenv" ]; then - echo "Virtual environment already exists. Skipping creation." -else - echo "Creating virtual environment" - $PYTHON_CMD -m venv $pythonScriptPath"scriptenv" -fi - -# Activate the virtual environment -if [ -f $pythonScriptPath"scriptenv/bin/activate" ]; then - echo "Activating virtual environment (Linux/macOS)" - source $pythonScriptPath"scriptenv/bin/activate" -elif [ -f $pythonScriptPath"scriptenv/Scripts/activate" ]; then - echo "Activating virtual environment (Windows)" - source $pythonScriptPath"scriptenv/Scripts/activate" -else - echo "Error activating virtual environment. Requirements may be installed globally." -fi # Install the requirements -echo "Installing requirements" pip install --quiet -r ${pythonScriptPath}requirements.txt -echo "Requirements installed" +if [ $? -ne 0 ]; then + echo "Error: Failed to install Python requirements." + exit 1 +fi # Create Content Understanding analyzers -echo "Creating Content Understanding analyzer templates..." -echo "Running 02_create_cu_template_text.py" +echo "āœ“ Creating Content Understanding analyzer templates" python infra/scripts/index_scripts/02_create_cu_template_text.py --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion" if [ $? -ne 0 ]; then echo "Error: 02_create_cu_template_text.py failed." exit 1 fi -echo "Running 02_create_cu_template_audio.py" python infra/scripts/index_scripts/02_create_cu_template_audio.py --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion" if [ $? -ne 0 ]; then echo "Error: 02_create_cu_template_audio.py failed." exit 1 fi -echo "Content Understanding analyzers created successfully." # Run 04_cu_process_custom_data.py -echo "Running 04_cu_process_custom_data.py..." +echo "āœ“ Processing custom data" sql_server_fqdn="$sqlServerName.database.windows.net" python infra/scripts/index_scripts/04_cu_process_custom_data.py \ --search_endpoint "$searchEndpoint" \ @@ -579,4 +483,3 @@ if [ $? -ne 0 ]; then echo "Error: 04_cu_process_custom_data.py failed." exit 1 fi -echo "04_cu_process_custom_data.py completed successfully." diff --git a/infra/scripts/process_sample_data.sh b/infra/scripts/process_sample_data.sh index 9dbbcece2..d91e106d3 100644 --- a/infra/scripts/process_sample_data.sh +++ b/infra/scripts/process_sample_data.sh @@ -49,10 +49,8 @@ original_full_range_rule_present="false" # Function to enable public network access temporarily enable_public_access() { - echo "=== Temporarily enabling public network access for services ===" # Enable public access for Storage Account - echo "Enabling public access for Storage Account: $storageAccountName" original_storage_public_access=$(az storage account show \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ @@ -65,37 +63,30 @@ enable_public_access() { -o tsv) if [ "$original_storage_public_access" != "Enabled" ]; then + echo "āœ“ Enabling Storage Account public access" az storage account update \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ --public-network-access Enabled \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account public access enabled" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to enable Storage Account public access" return 1 fi - else - echo "āœ“ Storage Account public access already enabled" fi # Also ensure the default network action allows access if [ "$original_storage_default_action" != "Allow" ]; then - echo "Setting Storage Account network default action to Allow" + echo "āœ“ Setting Storage Account network default action to Allow" az storage account update \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ --default-action Allow \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account network default action set to Allow" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to set Storage Account network default action" return 1 fi - else - echo "āœ“ Storage Account network default action already set to Allow" fi # Enable public access for AI Foundry @@ -113,21 +104,16 @@ enable_public_access() { --output tsv) if [ -z "$original_foundry_public_access" ] || [ "$original_foundry_public_access" = "null" ]; then - echo "⚠ Info: Could not retrieve AI Foundry network access status." + echo "⚠ Could not retrieve AI Foundry network access status" elif [ "$original_foundry_public_access" != "Enabled" ]; then - echo "Current AI Foundry public access: $original_foundry_public_access" - echo "Enabling public access for AI Foundry resource: $aif_resource_name (Resource Group: $aif_resource_group)" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Enabling AI Foundry public access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$aif_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \ --output none; then - echo "āœ“ AI Foundry public access enabled" - else - echo "⚠ Warning: Failed to enable AI Foundry public access automatically." + echo "⚠ Failed to enable AI Foundry public access" fi - else - echo "āœ“ AI Foundry public access already enabled" fi fi @@ -146,26 +132,20 @@ enable_public_access() { --output tsv) if [ -z "$original_cu_foundry_public_access" ] || [ "$original_cu_foundry_public_access" = "null" ]; then - echo "⚠ Info: Could not retrieve CU Foundry network access status." + echo "⚠ Could not retrieve CU Foundry network access status" elif [ "$original_cu_foundry_public_access" != "Enabled" ]; then - echo "Current CU Foundry public access: $original_cu_foundry_public_access" - echo "Enabling public access for CU Foundry resource: $cu_resource_name (Resource Group: $cu_resource_group)" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Enabling CU Foundry public access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$cu_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \ --output none; then - echo "āœ“ CU Foundry public access enabled" - else - echo "⚠ Warning: Failed to enable CU Foundry public access automatically." + echo "⚠ Failed to enable CU Foundry public access" fi - else - echo "āœ“ CU Foundry public access already enabled" fi fi # Enable public access for SQL Server - echo "Checking SQL Server public access: $sqlServerName" original_sql_public_access=$(az sql server show \ --name "$sqlServerName" \ --resource-group "$resourceGroupName" \ @@ -173,20 +153,16 @@ enable_public_access() { -o tsv) if [ "$original_sql_public_access" != "Enabled" ]; then - echo "Enabling public access for SQL Server" + echo "āœ“ Enabling SQL Server public access" az sql server update \ --name "$sqlServerName" \ --resource-group "$resourceGroupName" \ --enable-public-network true \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ SQL Server public access enabled" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to enable SQL Server public access" return 1 fi - else - echo "āœ“ SQL Server public access already enabled" fi # Create temporary allow-all firewall rule for SQL Server @@ -209,43 +185,34 @@ enable_public_access() { --query "[?name=='${sql_allow_all_rule_name}'] | [0].name" \ -o tsv 2>/dev/null) - if [ -z "$existing_allow_all_rule" ]; then - if [ -n "$pre_existing_full_range_rule" ]; then - echo "āœ“ Existing rule ($pre_existing_full_range_rule) already allows full IP range." - else - echo "Creating temporary allow-all firewall rule ($sql_allow_all_rule_name)..." - if az sql server firewall-rule create \ - --resource-group "$resourceGroupName" \ - --server "$sqlServerName" \ - --name "$sql_allow_all_rule_name" \ - --start-ip-address 0.0.0.0 \ - --end-ip-address 255.255.255.255 \ - --output none; then - created_sql_allow_all_firewall_rule="true" - echo "āœ“ Temporary allow-all firewall rule created" - else - echo "⚠ Warning: Failed to create allow-all firewall rule" - fi - fi + if [ -z "$existing_allow_all_rule" ] && [ -z "$pre_existing_full_range_rule" ]; then + echo "āœ“ Creating temporary SQL firewall rule" + if az sql server firewall-rule create \ + --resource-group "$resourceGroupName" \ + --server "$sqlServerName" \ + --name "$sql_allow_all_rule_name" \ + --start-ip-address 0.0.0.0 \ + --end-ip-address 255.255.255.255 \ + --output none; then + created_sql_allow_all_firewall_rule="true" + else + echo "⚠ Failed to create firewall rule" + fi else - echo "āœ“ Temporary allow-all firewall rule already present" - original_full_range_rule_present="true" + original_full_range_rule_present="true" fi # Wait a bit for changes to take effect - echo "Waiting for network access changes to propagate..." sleep 10 - echo "=== Public network access enabled successfully ===" return 0 } # Function to restore original network access settings restore_network_access() { - echo "=== Restoring original network access settings ===" # Restore Storage Account access if [ -n "$original_storage_public_access" ] && [ "$original_storage_public_access" != "Enabled" ]; then - echo "Restoring Storage Account public access to: $original_storage_public_access" + echo "āœ“ Restoring Storage Account access" case "$original_storage_public_access" in "enabled"|"Enabled") restore_value="Enabled" ;; "disabled"|"Disabled") restore_value="Disabled" ;; @@ -256,74 +223,56 @@ restore_network_access() { --resource-group "$resourceGroupName" \ --public-network-access "$restore_value" \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account access restored" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to restore Storage Account access" fi - else - echo "Storage Account access unchanged (already at desired state)" fi # Restore Storage Account network default action if [ -n "$original_storage_default_action" ] && [ "$original_storage_default_action" != "Allow" ]; then - echo "Restoring Storage Account network default action to: $original_storage_default_action" + echo "āœ“ Restoring Storage Account network default action" az storage account update \ --name "$storageAccountName" \ --resource-group "$resourceGroupName" \ --default-action "$original_storage_default_action" \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ Storage Account network default action restored" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to restore Storage Account network default action" fi - else - echo "Storage Account network default action unchanged (already at desired state)" fi # Restore AI Foundry access if [ -n "$original_foundry_public_access" ] && [ "$original_foundry_public_access" != "Enabled" ]; then - echo "Restoring AI Foundry public access to: $original_foundry_public_access" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Restoring AI Foundry access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$aif_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess="$original_foundry_public_access" \ --set properties.apiProperties.qnaAzureSearchEndpointKey="" \ --set properties.networkAcls.bypass="AzureServices" \ --output none 2>/dev/null; then - echo "āœ“ AI Foundry access restored" - else - echo "⚠ Warning: Failed to restore AI Foundry access automatically." - echo " Please manually restore network access in the Azure portal if needed." + echo "⚠ Failed to restore AI Foundry access - please check Azure portal" fi - else - echo "AI Foundry access unchanged (already at desired state)" fi # Restore CU Foundry access if [ -n "$original_cu_foundry_public_access" ] && [ "$original_cu_foundry_public_access" != "Enabled" ]; then - echo "Restoring CU Foundry public access to: $original_cu_foundry_public_access" - if MSYS_NO_PATHCONV=1 az resource update \ + echo "āœ“ Restoring CU Foundry access" + if ! MSYS_NO_PATHCONV=1 az resource update \ --ids "$cu_account_resource_id" \ --api-version 2024-10-01 \ --set properties.publicNetworkAccess="$original_cu_foundry_public_access" \ --set properties.apiProperties.qnaAzureSearchEndpointKey="" \ --set properties.networkAcls.bypass="AzureServices" \ --output none 2>/dev/null; then - echo "āœ“ CU Foundry access restored" - else - echo "⚠ Warning: Failed to restore CU Foundry access automatically." - echo " Please manually restore network access in the Azure portal if needed." + echo "⚠ Failed to restore CU Foundry access - please check Azure portal" fi - else - echo "CU Foundry access unchanged (already at desired state)" fi # Restore SQL Server public access if [ -n "$original_sql_public_access" ] && [ "$original_sql_public_access" != "Enabled" ]; then - echo "Restoring SQL Server public access to: $original_sql_public_access" + echo "āœ“ Restoring SQL Server access" case "$original_sql_public_access" in "enabled"|"Enabled") restore_value=true ;; "disabled"|"Disabled") restore_value=false ;; @@ -334,16 +283,10 @@ restore_network_access() { --resource-group "$resourceGroupName" \ --enable-public-network $restore_value \ --output none - if [ $? -eq 0 ]; then - echo "āœ“ SQL Server access restored" - else + if [ $? -ne 0 ]; then echo "āœ— Failed to restore SQL Server access" fi - else - echo "SQL Server access unchanged (already at desired state)" fi - - echo "=== Network access restoration completed ===" } # Function to handle script cleanup on exit @@ -351,11 +294,9 @@ cleanup_on_exit() { exit_code=$? echo "" if [ $exit_code -ne 0 ]; then - echo "āŒ Script failed with exit code $exit_code" - echo "Restoring network access settings before exit..." + echo "āŒ Script failed" else echo "āœ… Script completed successfully" - echo "Restoring network access settings..." fi restore_network_access exit $exit_code @@ -374,7 +315,6 @@ check_azd_installed() { } get_values_from_azd_env() { - echo "Getting values from azd environment variables..." # Use grep with a regex to ensure we're only capturing sanitized values to avoid command injection resourceGroupName=$(azd env get-value RESOURCE_GROUP_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$') storageAccountName=$(azd env get-value STORAGE_ACCOUNT_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$') @@ -401,19 +341,21 @@ get_values_from_azd_env() { if [ -z "$resourceGroupName" ] || [ -z "$storageAccountName" ] || [ -z "$fileSystem" ] || [ -z "$sqlServerName" ] || [ -z "$SqlDatabaseName" ] || [ -z "$backendUserMidClientId" ] || [ -z "$backendUserMidDisplayName" ] || [ -z "$aiSearchName" ] || [ -z "$aif_resource_id" ]; then echo "Error: One or more required values could not be retrieved from azd environment." return 1 - else - echo "All values retrieved successfully from azd environment." - return 0 fi + return 0 } -# Use Azure CLI login if running locally -echo "" -echo "Authenticating with Azure CLI..." -az login --use-device-code -echo "" +# Check if user is logged in to Azure +echo "Checking Azure authentication..." +if az account show &> /dev/null; then + echo "Already authenticated with Azure." +else + # Use Azure CLI login if running locally + echo "Authenticating with Azure CLI..." + az login --use-device-code +fi if check_azd_installed; then azSubscriptionId=$(azd env get-value AZURE_SUBSCRIPTION_ID) || azSubscriptionId="$AZURE_SUBSCRIPTION_ID" || azSubscriptionId="" @@ -520,8 +462,4 @@ if [ $? -ne 0 ]; then fi echo "run_create_index_scripts.sh completed successfully." -## SQL role assignment now centralized in run_create_index_scripts.sh; removed local duplicate block. - echo "All scripts executed successfully." -echo "Network access will be restored to original settings..." -# Note: cleanup_on_exit will be called automatically via the trap diff --git a/infra/scripts/run_create_index_scripts.sh b/infra/scripts/run_create_index_scripts.sh index 868d74a34..d43e1d044 100644 --- a/infra/scripts/run_create_index_scripts.sh +++ b/infra/scripts/run_create_index_scripts.sh @@ -20,20 +20,13 @@ ai_agent_endpoint="${16}" pythonScriptPath="infra/scripts/index_scripts/" -echo "Script Started" - # Authenticate with Azure -if az account show &> /dev/null; then - echo "Already authenticated with Azure." -else - echo "Not authenticated with Azure. Attempting to authenticate..." - # Use Azure CLI login +if ! az account show &> /dev/null; then echo "Authenticating with Azure CLI..." - az login + az login --use-device-code fi # Get signed in user and store the output -echo "Getting signed in user id and display name" signed_user=$(az ad signed-in-user show --query "{id:id, displayName:displayName}" -o json) signed_user_id=$(echo "$signed_user" | grep -o '"id": *"[^"]*"' | head -1 | sed 's/"id": *"\([^"]*\)"/\1/') signed_user_display_name=$(echo "$signed_user" | grep -o '"displayName": *"[^"]*"' | sed 's/"displayName": *"\([^"]*\)"/\1/') @@ -41,147 +34,88 @@ signed_user_display_name=$(echo "$signed_user" | grep -o '"displayName": *"[^"]* # Note: Environment variables are now passed as parameters from process_sample_data.sh ### Assign Azure AI User role to the signed in user for AI Foundry ### - -# Check if the user has the Azure AI User role -echo "Checking if user has the Azure AI User role for AI Foundry" role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $aif_resource_id --assignee $signed_user_id --query "[].roleDefinitionId" -o tsv) if [ -z "$role_assignment" ]; then - echo "User does not have the Azure AI User role for AI Foundry. Assigning the role." + echo "āœ“ Assigning Azure AI User role for AI Foundry" MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $aif_resource_id --output none - if [ $? -eq 0 ]; then - echo "Azure AI User role for AI Foundry assigned successfully." - else - echo "Failed to assign Azure AI User role for AI Foundry." + if [ $? -ne 0 ]; then + echo "āœ— Failed to assign Azure AI User role for AI Foundry" exit 1 fi -else - echo "User already has the Azure AI User role for AI Foundry." fi ### Assign Azure AI User role to the signed in user for CU Foundry ### - if [ -n "$cu_foundry_resource_id" ] && [ "$cu_foundry_resource_id" != "null" ]; then - echo "Checking if user has the Azure AI User role for CU Foundry" role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $cu_foundry_resource_id --assignee $signed_user_id --query "[].roleDefinitionId" -o tsv) if [ -z "$role_assignment" ]; then - echo "User does not have the Azure AI User role for CU Foundry. Assigning the role." + echo "āœ“ Assigning Azure AI User role for CU Foundry" MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $cu_foundry_resource_id --output none - if [ $? -eq 0 ]; then - echo "Azure AI User role for CU Foundry assigned successfully." - else - echo "Failed to assign Azure AI User role for CU Foundry." + if [ $? -ne 0 ]; then + echo "āœ— Failed to assign Azure AI User role for CU Foundry" exit 1 fi - else - echo "User already has the Azure AI User role for CU Foundry." fi fi ### Assign Search Index Data Contributor role to the signed in user ### - -echo "Getting Azure Search resource id" search_resource_id=$(az search service show --name $aiSearchName --resource-group $resourceGroupName --query id --output tsv) -echo "Checking if user has the Search Index Data Contributor role" role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --assignee $signed_user_id --role "Search Index Data Contributor" --scope $search_resource_id --query "[].roleDefinitionId" -o tsv) if [ -z "$role_assignment" ]; then - echo "User does not have the Search Index Data Contributor role. Assigning the role." + echo "āœ“ Assigning Search Index Data Contributor role" MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role "Search Index Data Contributor" --scope $search_resource_id --output none - if [ $? -eq 0 ]; then - echo "Search Index Data Contributor role assigned successfully." - else - echo "Failed to assign Search Index Data Contributor role." + if [ $? -ne 0 ]; then + echo "āœ— Failed to assign Search Index Data Contributor role" exit 1 fi -else - echo "User already has the Search Index Data Contributor role." fi ### Assign signed in user as SQL Server Admin ### - -echo "Getting Azure SQL Server resource id" sql_server_resource_id=$(az sql server show --name $sqlServerName --resource-group $resourceGroupName --query id --output tsv) - -# Check if the user is Azure SQL Server Admin -echo "Checking if user is Azure SQL Server Admin" admin=$(MSYS_NO_PATHCONV=1 az sql server ad-admin list --ids $sql_server_resource_id --query "[?sid == '$signed_user_id']" -o tsv) -# Check if the role exists -if [ -n "$admin" ]; then - echo "User is already Azure SQL Server Admin" -else - echo "User is not Azure SQL Server Admin. Assigning the role." +if [ -z "$admin" ]; then + echo "āœ“ Assigning user as SQL Server Admin" MSYS_NO_PATHCONV=1 az sql server ad-admin create --display-name "$signed_user_display_name" --object-id $signed_user_id --resource-group $resourceGroupName --server $sqlServerName --output none - if [ $? -eq 0 ]; then - echo "Assigned user as Azure SQL Server Admin." - else - echo "Failed to assign Azure SQL Server Admin role." + if [ $? -ne 0 ]; then + echo "āœ— Failed to assign SQL Server Admin role" exit 1 fi fi - -# Determine the correct Python command -if command -v python3 &> /dev/null; then - PYTHON_CMD="python3" -elif command -v python &> /dev/null; then - PYTHON_CMD="python" -else - echo "Python is not installed on this system. Or it is not added in the PATH." - exit 1 -fi - -# create virtual environment -# Check if the virtual environment already exists -if [ -d $pythonScriptPath"scriptenv" ]; then - echo "Virtual environment already exists. Skipping creation." -else - echo "Creating virtual environment" - $PYTHON_CMD -m venv $pythonScriptPath"scriptenv" -fi - -# Activate the virtual environment -if [ -f $pythonScriptPath"scriptenv/bin/activate" ]; then - echo "Activating virtual environment (Linux/macOS)" - source $pythonScriptPath"scriptenv/bin/activate" -elif [ -f $pythonScriptPath"scriptenv/Scripts/activate" ]; then - echo "Activating virtual environment (Windows)" - source $pythonScriptPath"scriptenv/Scripts/activate" -else - echo "Error activating virtual environment. Requirements may be installed globally." -fi - # Install the requirements echo "Installing requirements" pip install --quiet -r ${pythonScriptPath}requirements.txt -echo "Requirements installed" +if [ $? -ne 0 ]; then + echo "Error: Failed to install Python requirements." + exit 1 +fi error_flag=false -echo "Running the python scripts" -echo "Creating the search index" +echo "āœ“ Creating search index" python ${pythonScriptPath}01_create_search_index.py --search_endpoint="$search_endpoint" --openai_endpoint="$openai_endpoint" --embedding_model="$embedding_model" if [ $? -ne 0 ]; then echo "Error: 01_create_search_index.py failed." error_flag=true fi -echo "Creating CU template for text" +echo "āœ“ Creating CU template for text" python ${pythonScriptPath}02_create_cu_template_text.py --cu_endpoint="$cu_endpoint" --cu_api_version="$cu_api_version" if [ $? -ne 0 ]; then echo "Error: 02_create_cu_template_text.py failed." error_flag=true fi -echo "Creating CU template for audio" +echo "āœ“ Creating CU template for audio" python ${pythonScriptPath}02_create_cu_template_audio.py --cu_endpoint="$cu_endpoint" --cu_api_version="$cu_api_version" if [ $? -ne 0 ]; then echo "Error: 02_create_cu_template_audio.py failed." error_flag=true fi -echo "Processing data with CU" +echo "āœ“ Processing data with CU" sql_server_fqdn="$sqlServerName.database.windows.net" python ${pythonScriptPath}03_cu_process_data_text.py --search_endpoint="$search_endpoint" --ai_project_endpoint="$ai_agent_endpoint" --deployment_model="$deployment_model" --embedding_model="$embedding_model" --storage_account_name="$storageAccountName" --sql_server="$sql_server_fqdn" --sql_database="$sqlDatabaseName" --cu_endpoint="$cu_endpoint" --cu_api_version="$cu_api_version" if [ $? -ne 0 ]; then @@ -194,27 +128,19 @@ if [ -n "$backendManagedIdentityClientId" ] && [ -n "$backendManagedIdentityDisp mi_display_name="$backendManagedIdentityDisplayName" server_fqdn="$sqlServerName.database.windows.net" roles_json="[{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datareader\"},{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datawriter\"}]" - echo "[RoleAssign] Invoking assign_sql_roles.py for roles: db_datareader, db_datawriter" if [ -f "infra/scripts/add_user_scripts/assign_sql_roles.py" ]; then + echo "āœ“ Assigning SQL roles to managed identity" python infra/scripts/add_user_scripts/assign_sql_roles.py --server "$server_fqdn" --database "$sqlDatabaseName" --roles-json "$roles_json" if [ $? -ne 0 ]; then - echo "[RoleAssign] Warning: SQL role assignment failed." + echo "⚠ SQL role assignment failed" error_flag=true - else - echo "[RoleAssign] SQL roles assignment completed successfully." fi - else - echo "[RoleAssign] assign_sql_roles.py not found. Skipping SQL role assignment." fi -else - echo "[RoleAssign] Skipped SQL role assignment due to missing required values." fi # Check for any errors and exit if any occurred if [ "$error_flag" = true ]; then echo "One or more scripts failed. Please check the logs above." exit 1 -fi - -echo "Scripts completed successfully" \ No newline at end of file +fi \ No newline at end of file