diff --git a/documents/DeploymentGuide.md b/documents/DeploymentGuide.md
index c2af4c3ea..16d295a6e 100644
--- a/documents/DeploymentGuide.md
+++ b/documents/DeploymentGuide.md
@@ -147,7 +147,7 @@ If you're not using one of the above options for opening the project, then you'l
1. Make sure the following tools are installed:
- [PowerShell](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell?view=powershell-7.5) (v7.0+) - available for Windows, macOS, and Linux.
- [Azure Developer CLI (azd)](https://aka.ms/install-azd) (v1.18.0+) - version
- - [Python 3.9 to 3.11](https://www.python.org/downloads/)
+ - [Python 3.9+](https://www.python.org/downloads/)
- [Docker Desktop](https://www.docker.com/products/docker-desktop/)
- [Git](https://git-scm.com/downloads)
- [Microsoft ODBC Driver 18](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16) for SQL Server.
@@ -248,7 +248,32 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain
- This deployment generally takes **7-10 minutes** to provision the resources in your account and set up the solution.
- If you encounter an error or timeout during deployment, changing the location may help, as there could be availability constraints for the resources.
-5. Once the deployment has completed successfully and you would like to use the **sample data**, please open a **Git Bash terminal** and run the bash command printed below. The bash command will look like the following:
+5. Once the deployment has completed successfully, copy the bash command from terminal: (ex: `bash ./infra/scripts/process_sample_data.sh`) for later use.
+
+> **Note**: If you are running this deployment in GitHub Codespaces or VS Code Dev Container or Visual Studio Code (WEB) skip to step 7.
+
+6. Create and activate a virtual environment in bash terminal:
+
+ ```shell
+ python -m venv .venv
+ ```
+
+ ```shell
+ source .venv/Scripts/activate
+ ```
+
+7. Login to Azure:
+
+ ```shell
+ az login
+ ```
+
+ Alternatively, login to Azure using a device code (recommended when using VS Code Web):
+
+ ```shell
+ az login --use-device-code
+ ```
+8. Run the bash script from the output of the azd deployment. The script will look like the following:
```bash
bash ./infra/scripts/process_sample_data.sh
@@ -267,9 +292,9 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain
```
-6. Once the deployment has completed successfully, open the [Azure Portal](https://portal.azure.com/), go to the deployed resource group, find the App Service, and get the app URL from `Default domain`.
+9. Once the deployment has completed successfully, open the [Azure Portal](https://portal.azure.com/), go to the deployed resource group, find the App Service, and get the app URL from `Default domain`.
-7. You can now delete the resources by running `azd down`, if you are done trying out the application.
+10. You can now delete the resources by running `azd down`, if you are done trying out the application.
> **Note:** If you deployed with `enableRedundancy=true` and Log Analytics workspace replication is enabled, you must first disable replication before running `azd down` else resource group delete will fail. Follow the steps in [Handling Log Analytics Workspace Deletion with Replication Enabled](./LogAnalyticsReplicationDisable.md), wait until replication returns `false`, then run `azd down`.
### š ļø Troubleshooting
diff --git a/documents/LocalDebuggingSetup.md b/documents/LocalDebuggingSetup.md
index 298f692c7..91e9716bb 100644
--- a/documents/LocalDebuggingSetup.md
+++ b/documents/LocalDebuggingSetup.md
@@ -11,7 +11,7 @@ Install these tools before you start:
- [Azure Tools](https://marketplace.visualstudio.com/items?itemName=ms-vscode.vscode-node-azure-pack)
- [Bicep](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-bicep)
- [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python)
-- [Python 3.9 to 3.11](https://www.python.org/downloads/). **Important:** Check "Add Python to PATH" during installation.
+- [Python 3.9+](https://www.python.org/downloads/). **Important:** Check "Add Python to PATH" during installation.
- [PowerShell 7.0+](https://github.com/PowerShell/PowerShell#get-powershell).
- [Node.js (LTS)](https://nodejs.org/en).
- [Git](https://git-scm.com/downloads).
diff --git a/infra/scripts/add_user_scripts/assign_sql_roles.py b/infra/scripts/add_user_scripts/assign_sql_roles.py
index 2cc14416d..2c42e7a15 100644
--- a/infra/scripts/add_user_scripts/assign_sql_roles.py
+++ b/infra/scripts/add_user_scripts/assign_sql_roles.py
@@ -55,12 +55,9 @@ def assign_sql_roles(server, database, roles_json):
credential = AzureCliCredential()
# Connect to SQL Server
- print(f"Connecting to {server}/{database}...")
conn = connect_with_token(server, database, credential)
cursor = conn.cursor()
- print("Connected successfully.")
-
# Process each role assignment
for role_assignment in roles:
client_id = role_assignment.get("clientId")
@@ -68,11 +65,8 @@ def assign_sql_roles(server, database, roles_json):
role = role_assignment.get("role")
if not client_id or not display_name or not role:
- print(f"Skipping invalid role assignment: {role_assignment}")
continue
- print(f"\nProcessing: {display_name} -> {role}")
-
# Check if user already exists
check_user_sql = f"SELECT COUNT(*) FROM sys.database_principals WHERE name = '{display_name}'"
cursor.execute(check_user_sql)
@@ -81,16 +75,13 @@ def assign_sql_roles(server, database, roles_json):
if not user_exists:
# Create user from external provider
create_user_sql = f"CREATE USER [{display_name}] FROM EXTERNAL PROVIDER"
- print(f" Creating user: {display_name}")
try:
cursor.execute(create_user_sql)
conn.commit()
- print(" ā User created successfully")
+ print(f"ā Created user: {display_name}")
except Exception as e:
- print(f" ā Failed to create user: {e}")
+ print(f"ā Failed to create user: {e}")
continue
- else:
- print(f" User already exists: {display_name}")
# Check if user already has the role
check_role_sql = f"""
@@ -106,21 +97,17 @@ def assign_sql_roles(server, database, roles_json):
if not has_role:
# Add user to role
add_role_sql = f"ALTER ROLE [{role}] ADD MEMBER [{display_name}]"
- print(f" Assigning role: {role}")
try:
cursor.execute(add_role_sql)
conn.commit()
- print(" ā Role assigned successfully")
+ print(f"ā Assigned {role} to {display_name}")
except Exception as e:
- print(f" ā Failed to assign role: {e}")
+ print(f"ā Failed to assign {role}: {e}")
continue
- else:
- print(f" User already has role: {role}")
# Close connection
cursor.close()
conn.close()
- print("\nā All role assignments completed successfully")
return 0
except Exception as e:
diff --git a/infra/scripts/copy_kb_files.sh b/infra/scripts/copy_kb_files.sh
index c3638363e..fadc44cda 100644
--- a/infra/scripts/copy_kb_files.sh
+++ b/infra/scripts/copy_kb_files.sh
@@ -11,11 +11,6 @@ extractedFolder1="call_transcripts"
zipFileName2="infra/data/audio_data.zip"
extractedFolder2="audio_data"
-echo "Script Started"
-echo "Storage Account: $storageAccountName"
-echo "Container Name: $containerName"
-echo "Resource Group: $resourceGroupName"
-
# Validate required parameters
if [ -z "$storageAccountName" ] || [ -z "$containerName" ] || [ -z "$resourceGroupName" ]; then
echo "Error: Missing required parameters."
@@ -25,102 +20,76 @@ fi
# Extract zip files if they exist
if [ -f "$zipFileName1" ]; then
- echo "Extracting $zipFileName1..."
- unzip -o "$zipFileName1" -d "$extractedFolder1"
-else
- echo "Warning: $zipFileName1 not found. Skipping extraction."
+ unzip -q -o "$zipFileName1" -d "$extractedFolder1"
fi
if [ -f "$zipFileName2" ]; then
- echo "Extracting $zipFileName2..."
- unzip -o "$zipFileName2" -d "$extractedFolder2"
-else
- echo "Warning: $zipFileName2 not found. Skipping extraction."
+ unzip -q -o "$zipFileName2" -d "$extractedFolder2"
fi
# Authenticate with Azure
-if az account show &> /dev/null; then
- echo "Already authenticated with Azure."
-else
+if ! az account show &> /dev/null; then
echo "Authenticating with Azure CLI..."
- az login
+ az login --use-device-code
fi
# Check and assign Storage Blob Data Contributor role to current user
-echo "Checking Storage Blob Data Contributor role assignment..."
signed_user_id=$(az ad signed-in-user show --query id --output tsv)
storage_resource_id=$(az storage account show --name "$storageAccountName" --resource-group "$resourceGroupName" --query id --output tsv)
role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --assignee $signed_user_id --role "Storage Blob Data Contributor" --scope $storage_resource_id --query "[].roleDefinitionId" -o tsv)
if [ -z "$role_assignment" ]; then
- echo "Assigning Storage Blob Data Contributor role to current user..."
+ echo "ā Assigning Storage Blob Data Contributor role"
MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role "Storage Blob Data Contributor" --scope $storage_resource_id --output none
- if [ $? -eq 0 ]; then
- echo "Storage Blob Data Contributor role assigned successfully."
- # Wait a bit for role assignment to propagate
- echo "Waiting for role assignment to propagate..."
- sleep 10
- else
- echo "Failed to assign Storage Blob Data Contributor role."
+ if [ $? -ne 0 ]; then
+ echo "ā Failed to assign Storage Blob Data Contributor role"
exit 1
fi
-else
- echo "User already has Storage Blob Data Contributor role."
+ sleep 10
fi
# Upload files to storage account
-# Using az storage blob upload-batch to upload files with Azure CLI authentication
-echo "Uploading call transcripts to storage account..."
if [ -d "$extractedFolder1" ]; then
+ echo "ā Uploading call transcripts"
az storage blob upload-batch \
--account-name "$storageAccountName" \
--destination "$containerName/$extractedFolder1" \
--source "$extractedFolder1" \
--auth-mode login \
--pattern '*' \
- --overwrite
- if [ $? -eq 0 ]; then
- echo "ā Call transcripts uploaded successfully"
- else
+ --overwrite \
+ --output none
+ if [ $? -ne 0 ]; then
echo "ā Failed to upload call transcripts"
exit 1
fi
-else
- echo "Warning: $extractedFolder1 directory not found. Skipping upload."
fi
-echo "Uploading audio data to storage account..."
if [ -d "$extractedFolder2" ]; then
+ echo "ā Uploading audio data"
az storage blob upload-batch \
--account-name "$storageAccountName" \
--destination "$containerName/$extractedFolder2" \
--source "$extractedFolder2" \
--auth-mode login \
--pattern '*' \
- --overwrite
- if [ $? -eq 0 ]; then
- echo "ā Audio data uploaded successfully"
- else
+ --overwrite \
+ --output none
+ if [ $? -ne 0 ]; then
echo "ā Failed to upload audio data"
exit 1
fi
-else
- echo "Warning: $extractedFolder2 directory not found. Skipping upload."
fi
# Create custom data directories for user uploads
-echo "Creating custom data directories..."
az storage fs directory create \
--account-name "$storageAccountName" \
--file-system "$containerName" \
--name custom_audiodata \
- --auth-mode login 2>/dev/null || echo "custom_audiodata directory may already exist"
+ --auth-mode login --output none 2>/dev/null
az storage fs directory create \
--account-name "$storageAccountName" \
--file-system "$containerName" \
--name custom_transcripts \
- --auth-mode login 2>/dev/null || echo "custom_transcripts directory may already exist"
-
-echo "ā Custom data directories created successfully"
-echo "Script completed successfully"
\ No newline at end of file
+ --auth-mode login --output none 2>/dev/null
\ No newline at end of file
diff --git a/infra/scripts/index_scripts/01_create_search_index.py b/infra/scripts/index_scripts/01_create_search_index.py
index 1deaaa4a1..ccb2f5b82 100644
--- a/infra/scripts/index_scripts/01_create_search_index.py
+++ b/infra/scripts/index_scripts/01_create_search_index.py
@@ -30,9 +30,6 @@
INDEX_NAME = "call_transcripts_index"
-print("calling create_search_index()....")
-
-
def create_search_index():
"""
Creates or updates an Azure Cognitive Search index configured for:
@@ -105,7 +102,7 @@ def create_search_index():
)
result = index_client.create_or_update_index(index)
- print(f"Search index '{result.name}' created or updated successfully.")
+ print(f"ā Search index '{result.name}' created")
create_search_index()
\ No newline at end of file
diff --git a/infra/scripts/index_scripts/02_create_cu_template_audio.py b/infra/scripts/index_scripts/02_create_cu_template_audio.py
index da46664ad..72279c29d 100644
--- a/infra/scripts/index_scripts/02_create_cu_template_audio.py
+++ b/infra/scripts/index_scripts/02_create_cu_template_audio.py
@@ -36,9 +36,9 @@
analyzer = client.get_analyzer_detail_by_id(ANALYZER_ID)
if analyzer is not None:
client.delete_analyzer(ANALYZER_ID)
-except Exception as e:
- print(f"Analyzer with ID {ANALYZER_ID} was not found. Proceeding to create a new one.")
+except Exception:
+ pass
response = client.begin_create_analyzer(ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE_FILE)
result = client.poll_result(response)
-print(f"Analyzer with ID {ANALYZER_ID} created successfully.")
+print(f"ā Analyzer '{ANALYZER_ID}' created")
diff --git a/infra/scripts/index_scripts/02_create_cu_template_text.py b/infra/scripts/index_scripts/02_create_cu_template_text.py
index df128f31e..a9080f2ed 100644
--- a/infra/scripts/index_scripts/02_create_cu_template_text.py
+++ b/infra/scripts/index_scripts/02_create_cu_template_text.py
@@ -31,9 +31,9 @@
analyzer = client.get_analyzer_detail_by_id(ANALYZER_ID)
if analyzer is not None:
client.delete_analyzer(ANALYZER_ID)
-except Exception as e:
- print(f"Analyzer with ID {ANALYZER_ID} was not found. Proceeding to create a new one.")
+except Exception:
+ pass
response = client.begin_create_analyzer(ANALYZER_ID, analyzer_template_path=ANALYZER_TEMPLATE_FILE)
result = client.poll_result(response)
-print(f"Analyzer with ID {ANALYZER_ID} created successfully.")
+print(f"ā Analyzer '{ANALYZER_ID}' created")
diff --git a/infra/scripts/index_scripts/03_cu_process_data_text.py b/infra/scripts/index_scripts/03_cu_process_data_text.py
index a16757b72..32715e614 100644
--- a/infra/scripts/index_scripts/03_cu_process_data_text.py
+++ b/infra/scripts/index_scripts/03_cu_process_data_text.py
@@ -49,8 +49,6 @@
SAMPLE_PROCESSED_DATA_FILE = 'infra/data/sample_processed_data.json'
SAMPLE_PROCESSED_DATA_KEY_PHRASES_FILE = 'infra/data/sample_processed_data_key_phrases.json'
-print("Parameters received.")
-
# Azure DataLake setup
account_url = f"https://{STORAGE_ACCOUNT_NAME}.dfs.core.windows.net"
credential = AzureCliCredential(process_timeout=30)
@@ -58,13 +56,11 @@
file_system_client = service_client.get_file_system_client(FILE_SYSTEM_CLIENT_NAME)
directory_name = DIRECTORY
paths = list(file_system_client.get_paths(path=directory_name))
-print("Azure DataLake setup complete.")
# Azure Search setup
search_credential = AzureCliCredential(process_timeout=30)
search_client = SearchClient(SEARCH_ENDPOINT, INDEX_NAME, search_credential)
index_client = SearchIndexClient(endpoint=SEARCH_ENDPOINT, credential=search_credential)
-print("Azure Search setup complete.")
# SQL Server setup
driver = "{ODBC Driver 18 for SQL Server}"
@@ -74,7 +70,6 @@
connection_string = f"DRIVER={driver};SERVER={SQL_SERVER};DATABASE={SQL_DATABASE};"
conn = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct})
cursor = conn.cursor()
-print("SQL Server connection established.")
# SQL data type mapping for pandas to SQL conversion
sql_data_types = {
@@ -170,7 +165,6 @@ def generate_sql_insert_script(df, table_name, columns, sql_file_name):
token_provider=cu_token_provider
)
ANALYZER_ID = "ckm-json"
-print("Content Understanding client initialized.")
# Azure AI Foundry (Inference) clients (Managed Identity)
inference_endpoint = f"https://{urlparse(AI_PROJECT_ENDPOINT).netloc}/models"
@@ -289,7 +283,7 @@ def create_tables():
StartTime varchar(255)
);""")
conn.commit()
- print("Database tables created.")
+
create_tables()
@@ -332,12 +326,10 @@ def create_tables():
if docs != [] and counter % 10 == 0:
result = search_client.upload_documents(documents=docs)
docs = []
- print(f'{counter} uploaded to Azure Search.')
if docs:
search_client.upload_documents(documents=docs)
- print(f'Final batch uploaded to Azure Search.')
-print("File processing and DB/Search insertion complete.")
+print(f"ā Processed {counter} files")
# Load sample data to search index and database
def bulk_import_json_to_table(json_file, table_name):
@@ -345,7 +337,6 @@ def bulk_import_json_to_table(json_file, table_name):
data = json.load(f)
if not data:
- print(f"No data to import into {table_name}.")
return
df = pd.DataFrame(data)
@@ -356,11 +347,10 @@ def bulk_import_json_to_table(json_file, table_name):
documents = json.load(file)
batch = [{"@search.action": "upload", **doc} for doc in documents]
search_client.upload_documents(documents=batch)
-print(f'Successfully uploaded {len(documents)} sample index data records to search index {INDEX_NAME}.')
bulk_import_json_to_table(SAMPLE_PROCESSED_DATA_FILE, 'processed_data')
bulk_import_json_to_table(SAMPLE_PROCESSED_DATA_KEY_PHRASES_FILE, 'processed_data_key_phrases')
-print("Sample data loaded to DB and Search.")
+print(f"ā Loaded {len(documents)} sample records")
# Topic mining and mapping
cursor.execute('SELECT distinct topic FROM processed_data')
@@ -374,7 +364,6 @@ def bulk_import_json_to_table(json_file, table_name):
);""")
conn.commit()
topics_str = ', '.join(df['topic'].tolist())
-print("Topic mining table prepared.")
def call_gpt4(topics_str1, client):
@@ -410,7 +399,6 @@ def call_gpt4(topics_str1, client):
for object1 in res['topics']:
cursor.execute("INSERT INTO km_mined_topics (label, description) VALUES (?,?)", (object1['label'], object1['description']))
conn.commit()
-print("Topics mined and inserted into km_mined_topics.")
cursor.execute('SELECT label FROM km_mined_topics')
rows = [tuple(row) for row in cursor.fetchall()]
@@ -418,7 +406,7 @@ def call_gpt4(topics_str1, client):
df_topics = pd.DataFrame(rows, columns=column_names)
mined_topics_list = df_topics['label'].tolist()
mined_topics = ", ".join(mined_topics_list)
-print("Mined topics loaded.")
+print(f"ā Mined {len(mined_topics_list)} topics")
def get_mined_topic_mapping(input_text, list_of_topics):
@@ -445,7 +433,6 @@ def get_mined_topic_mapping(input_text, list_of_topics):
mined_topic_str = get_mined_topic_mapping(row['topic'], str(mined_topics_list))
cursor.execute("UPDATE processed_data SET mined_topic = ? WHERE ConversationId = ?", (mined_topic_str, row['ConversationId']))
conn.commit()
-print("Processed data mapped to mined topics.")
# Update processed data for RAG
cursor.execute('DROP TABLE IF EXISTS km_processed_data')
@@ -472,7 +459,6 @@ def get_mined_topic_mapping(input_text, list_of_topics):
generate_sql_insert_script(df_km, 'km_processed_data', columns, 'km_processed_data_insert.sql')
# Update processed_data_key_phrases table
-print("Updating processed_data_key_phrases table")
cursor.execute('''select ConversationId, key_phrases, sentiment, mined_topic as topic, StartTime from processed_data''')
rows = [tuple(row) for row in cursor.fetchall()]
column_names = [i[0] for i in cursor.description]
@@ -485,7 +471,6 @@ def get_mined_topic_mapping(input_text, list_of_topics):
cursor.execute("INSERT INTO processed_data_key_phrases (ConversationId, key_phrase, sentiment, topic, StartTime) VALUES (?,?,?,?,?)",
(row['ConversationId'], key_phrase, row['sentiment'], row['topic'], row['StartTime']))
conn.commit()
-print("processed_data_key_phrases table updated.")
# Adjust dates to current date
today = datetime.today()
@@ -496,8 +481,7 @@ def get_mined_topic_mapping(input_text, list_of_topics):
cursor.execute("UPDATE [dbo].[km_processed_data] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss'), EndTime = FORMAT(DATEADD(DAY, ?, EndTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference, days_difference))
cursor.execute("UPDATE [dbo].[processed_data_key_phrases] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference,))
conn.commit()
-print("Dates adjusted to current date.")
cursor.close()
conn.close()
-print("All steps completed. Connection closed.")
+print("ā Data processing completed")
diff --git a/infra/scripts/index_scripts/04_cu_process_custom_data.py b/infra/scripts/index_scripts/04_cu_process_custom_data.py
index 244e22741..ad44f7b0b 100644
--- a/infra/scripts/index_scripts/04_cu_process_custom_data.py
+++ b/infra/scripts/index_scripts/04_cu_process_custom_data.py
@@ -65,8 +65,6 @@
CU_ENDPOINT = args.cu_endpoint
CU_API_VERSION = args.cu_api_version
-print("Command-line arguments parsed.")
-
# Azure DataLake setup
account_url = f"https://{STORAGE_ACCOUNT_NAME}.dfs.core.windows.net"
credential = AzureCliCredential(process_timeout=30)
@@ -74,13 +72,11 @@
file_system_client = service_client.get_file_system_client(FILE_SYSTEM_CLIENT_NAME)
directory_name = DIRECTORY
paths = list(file_system_client.get_paths(path=directory_name))
-print("Azure DataLake setup complete.")
# Azure Search setup
search_credential = AzureCliCredential(process_timeout=30)
search_client = SearchClient(SEARCH_ENDPOINT, INDEX_NAME, search_credential)
index_client = SearchIndexClient(endpoint=SEARCH_ENDPOINT, credential=search_credential)
-print("Azure Search setup complete.")
# Azure AI Foundry (Inference) clients (Managed Identity)
inference_endpoint = f"https://{urlparse(AI_PROJECT_ENDPOINT).netloc}/models"
@@ -170,7 +166,7 @@ def create_search_index():
)
result = index_client.create_or_update_index(index)
- print(f"Search index '{result.name}' created or updated successfully.")
+ print(f"ā Search index '{result.name}' created")
create_search_index()
@@ -182,7 +178,6 @@ def create_search_index():
connection_string = f"DRIVER={DRIVER};SERVER={SQL_SERVER};DATABASE={SQL_DATABASE};"
conn = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct})
cursor = conn.cursor()
-print("SQL Server connection established.")
# Content Understanding client
cu_credential = AzureCliCredential(process_timeout=30)
@@ -192,7 +187,6 @@ def create_search_index():
api_version=CU_API_VERSION,
token_provider=cu_token_provider
)
-print("Content Understanding client initialized.")
# Utility functions
def get_embeddings(text: str):
@@ -273,7 +267,6 @@ def generate_sql_insert_script(df, table_name, columns, sql_file_name):
conn.commit()
record_count = len(df)
- print(f"Inserted {record_count} records into {table_name} using optimized SQL script.")
return record_count
def clean_spaces_with_regex(text):
@@ -346,7 +339,6 @@ def create_tables():
StartTime varchar(255)
);""")
conn.commit()
- print("Database tables created.")
create_tables()
@@ -389,19 +381,16 @@ def create_tables():
if docs != [] and counter % 10 == 0:
result = search_client.upload_documents(documents=docs)
docs = []
- print(f'{counter} uploaded to Azure Search.')
if docs:
search_client.upload_documents(documents=docs)
- print(f'Final batch uploaded to Azure Search - transcripts.')
-print("File processing and DB/Search insertion complete - transcripts.")
+print(f"ā Processed {counter} transcript files")
# Process files for audio data
ANALYZER_ID = "ckm-audio"
directory_name = AUDIO_DIRECTORY
paths = list(file_system_client.get_paths(path=directory_name))
-print("Processing audio files")
docs = []
counter = 0
# process and upload audio files to search index - audio data
@@ -443,22 +432,18 @@ def create_tables():
docs.extend(prepare_search_doc(content, document_id, path.name))
counter += 1
- print(f"Processed file {path.name} successfully.")
except Exception as e:
- print(f"Error processing file {path.name}: {e}")
pass
if docs != [] and counter % 10 == 0:
result = search_client.upload_documents(documents=docs)
docs = []
- print(f' {str(counter)} uploaded')
# upload the last batch
if docs != []:
search_client.upload_documents(documents=docs)
- print(f'Final batch uploaded to Azure Search - audio data.')
-print("File processing and DB/Search insertion complete - audio data.")
+print(f"ā Processed {counter} audio files")
# Topic mining and mapping
cursor.execute('SELECT distinct topic FROM processed_data')
@@ -472,7 +457,6 @@ def create_tables():
);""")
conn.commit()
topics_str = ', '.join(df['topic'].tolist())
-print("Topic mining table prepared.")
def call_gpt4(topics_str1, client):
topic_prompt = f"""
@@ -506,7 +490,6 @@ def call_gpt4(topics_str1, client):
for object1 in res['topics']:
cursor.execute("INSERT INTO km_mined_topics (label, description) VALUES (?,?)", (object1['label'], object1['description']))
conn.commit()
-print("Topics mined and inserted into km_mined_topics.")
cursor.execute('SELECT label FROM km_mined_topics')
rows = [tuple(row) for row in cursor.fetchall()]
@@ -514,7 +497,6 @@ def call_gpt4(topics_str1, client):
df_topics = pd.DataFrame(rows, columns=column_names)
mined_topics_list = df_topics['label'].tolist()
mined_topics = ", ".join(mined_topics_list)
-print("Mined topics loaded.")
def get_mined_topic_mapping(input_text, list_of_topics):
@@ -541,7 +523,6 @@ def get_mined_topic_mapping(input_text, list_of_topics):
mined_topic_str = get_mined_topic_mapping(row['topic'], str(mined_topics_list))
cursor.execute("UPDATE processed_data SET mined_topic = ? WHERE ConversationId = ?", (mined_topic_str, row['ConversationId']))
conn.commit()
-print("Processed data mapped to mined topics.")
# Update processed data for RAG
cursor.execute('DROP TABLE IF EXISTS km_processed_data')
@@ -565,11 +546,10 @@ def get_mined_topic_mapping(input_text, list_of_topics):
"keyphrases", "complaint", "topic"]
df_km = pd.DataFrame([list(row) for row in rows], columns=columns)
-generate_sql_insert_script(df_km, 'km_processed_data', columns, 'km_processed_data_insert.sql')
-print("km_processed_data table updated.")
+record_count = generate_sql_insert_script(df_km, 'km_processed_data', columns, 'km_processed_data_insert.sql')
+print(f"ā Loaded {record_count} sample records")
# Update processed_data_key_phrases table
-print("Updating processed_data_key_phrases table")
cursor.execute('''select ConversationId, key_phrases, sentiment, mined_topic as topic, StartTime from processed_data''')
rows = [tuple(row) for row in cursor.fetchall()]
column_names = [i[0] for i in cursor.description]
@@ -582,7 +562,6 @@ def get_mined_topic_mapping(input_text, list_of_topics):
cursor.execute("INSERT INTO processed_data_key_phrases (ConversationId, key_phrase, sentiment, topic, StartTime) VALUES (?,?,?,?,?)",
(row['ConversationId'], key_phrase, row['sentiment'], row['topic'], row['StartTime']))
conn.commit()
-print("processed_data_key_phrases table updated.")
# Adjust dates to current date
today = datetime.today()
@@ -593,8 +572,7 @@ def get_mined_topic_mapping(input_text, list_of_topics):
cursor.execute("UPDATE [dbo].[km_processed_data] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss'), EndTime = FORMAT(DATEADD(DAY, ?, EndTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference, days_difference))
cursor.execute("UPDATE [dbo].[processed_data_key_phrases] SET StartTime = FORMAT(DATEADD(DAY, ?, StartTime), 'yyyy-MM-dd HH:mm:ss')", (days_difference,))
conn.commit()
-print("Dates adjusted to current date.")
cursor.close()
conn.close()
-print("All steps completed. Connection closed.")
+print("ā Data processing completed")
diff --git a/infra/scripts/index_scripts/requirements.txt b/infra/scripts/index_scripts/requirements.txt
index b2735a540..3fa8d865d 100644
--- a/infra/scripts/index_scripts/requirements.txt
+++ b/infra/scripts/index_scripts/requirements.txt
@@ -6,7 +6,6 @@ azure-ai-inference==1.0.0b9
pypdf==5.6.0
# pyodbc
tiktoken==0.9.0
-msal[broker]==1.32.3
azure-identity==1.23.0
azure-ai-textanalytics==5.3.0
azure-search-documents==11.5.2
diff --git a/infra/scripts/process_custom_data.sh b/infra/scripts/process_custom_data.sh
index b6f3f4771..24ff86f25 100644
--- a/infra/scripts/process_custom_data.sh
+++ b/infra/scripts/process_custom_data.sh
@@ -49,10 +49,8 @@ original_full_range_rule_present="false"
# Function to enable public network access temporarily
enable_public_access() {
- echo "=== Temporarily enabling public network access for services ==="
# Enable public access for Storage Account
- echo "Enabling public access for Storage Account: $storageAccountName"
original_storage_public_access=$(az storage account show \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
@@ -65,37 +63,30 @@ enable_public_access() {
-o tsv)
if [ "$original_storage_public_access" != "Enabled" ]; then
+ echo "ā Enabling Storage Account public access"
az storage account update \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
--public-network-access Enabled \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account public access enabled"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to enable Storage Account public access"
return 1
fi
- else
- echo "ā Storage Account public access already enabled"
fi
# Also ensure the default network action allows access
if [ "$original_storage_default_action" != "Allow" ]; then
- echo "Setting Storage Account network default action to Allow"
+ echo "ā Setting Storage Account network default action to Allow"
az storage account update \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
--default-action Allow \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account network default action set to Allow"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to set Storage Account network default action"
return 1
fi
- else
- echo "ā Storage Account network default action already set to Allow"
fi
# Enable public access for AI Foundry
@@ -113,21 +104,16 @@ enable_public_access() {
--output tsv)
if [ -z "$original_foundry_public_access" ] || [ "$original_foundry_public_access" = "null" ]; then
- echo "ā Info: Could not retrieve AI Foundry network access status."
+ echo "ā Could not retrieve AI Foundry network access status"
elif [ "$original_foundry_public_access" != "Enabled" ]; then
- echo "Current AI Foundry public access: $original_foundry_public_access"
- echo "Enabling public access for AI Foundry resource: $aif_resource_name (Resource Group: $aif_resource_group)"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Enabling AI Foundry public access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$aif_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \
--output none; then
- echo "ā AI Foundry public access enabled"
- else
- echo "ā Warning: Failed to enable AI Foundry public access automatically."
+ echo "ā Failed to enable AI Foundry public access"
fi
- else
- echo "ā AI Foundry public access already enabled"
fi
fi
@@ -146,26 +132,20 @@ enable_public_access() {
--output tsv)
if [ -z "$original_cu_foundry_public_access" ] || [ "$original_cu_foundry_public_access" = "null" ]; then
- echo "ā Info: Could not retrieve CU Foundry network access status."
+ echo "ā Could not retrieve CU Foundry network access status"
elif [ "$original_cu_foundry_public_access" != "Enabled" ]; then
- echo "Current CU Foundry public access: $original_cu_foundry_public_access"
- echo "Enabling public access for CU Foundry resource: $cu_resource_name (Resource Group: $cu_resource_group)"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Enabling CU Foundry public access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$cu_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \
--output none; then
- echo "ā CU Foundry public access enabled"
- else
- echo "ā Warning: Failed to enable CU Foundry public access automatically."
+ echo "ā Failed to enable CU Foundry public access"
fi
- else
- echo "ā CU Foundry public access already enabled"
fi
fi
# Enable public access for SQL Server
- echo "Checking SQL Server public access: $sqlServerName"
original_sql_public_access=$(az sql server show \
--name "$sqlServerName" \
--resource-group "$resourceGroupName" \
@@ -173,15 +153,16 @@ enable_public_access() {
-o tsv)
if [ "$original_sql_public_access" != "Enabled" ]; then
- echo "Enabling public access for SQL Server"
+ echo "ā Enabling SQL Server public access"
az sql server update \
--name "$sqlServerName" \
--resource-group "$resourceGroupName" \
--enable-public-network true \
--output none
- echo "ā SQL Server public access enabled"
- else
- echo "ā SQL Server public access already enabled"
+ if [ $? -ne 0 ]; then
+ echo "ā Failed to enable SQL Server public access"
+ return 1
+ fi
fi
# Create temporary allow-all firewall rule for SQL Server
@@ -204,43 +185,34 @@ enable_public_access() {
--query "[?name=='${sql_allow_all_rule_name}'] | [0].name" \
-o tsv 2>/dev/null)
- if [ -z "$existing_allow_all_rule" ]; then
- if [ -n "$pre_existing_full_range_rule" ]; then
- echo "ā Existing rule ($pre_existing_full_range_rule) already allows full IP range."
- else
- echo "Creating temporary allow-all firewall rule ($sql_allow_all_rule_name)..."
- if az sql server firewall-rule create \
- --resource-group "$resourceGroupName" \
- --server "$sqlServerName" \
- --name "$sql_allow_all_rule_name" \
- --start-ip-address 0.0.0.0 \
- --end-ip-address 255.255.255.255 \
- --output none; then
- created_sql_allow_all_firewall_rule="true"
- echo "ā Temporary allow-all firewall rule created"
- else
- echo "ā Warning: Failed to create allow-all firewall rule"
- fi
- fi
+ if [ -z "$existing_allow_all_rule" ] && [ -z "$pre_existing_full_range_rule" ]; then
+ echo "ā Creating temporary SQL firewall rule"
+ if az sql server firewall-rule create \
+ --resource-group "$resourceGroupName" \
+ --server "$sqlServerName" \
+ --name "$sql_allow_all_rule_name" \
+ --start-ip-address 0.0.0.0 \
+ --end-ip-address 255.255.255.255 \
+ --output none; then
+ created_sql_allow_all_firewall_rule="true"
+ else
+ echo "ā Failed to create firewall rule"
+ fi
else
- echo "ā Temporary allow-all firewall rule already present"
- original_full_range_rule_present="true"
+ original_full_range_rule_present="true"
fi
# Wait a bit for changes to take effect
- echo "Waiting for network access changes to propagate..."
sleep 10
- echo "=== Public network access enabled successfully ==="
return 0
}
# Function to restore original network access settings
restore_network_access() {
- echo "=== Restoring original network access settings ==="
# Restore Storage Account access
if [ -n "$original_storage_public_access" ] && [ "$original_storage_public_access" != "Enabled" ]; then
- echo "Restoring Storage Account public access to: $original_storage_public_access"
+ echo "ā Restoring Storage Account access"
case "$original_storage_public_access" in
"enabled"|"Enabled") restore_value="Enabled" ;;
"disabled"|"Disabled") restore_value="Disabled" ;;
@@ -251,74 +223,56 @@ restore_network_access() {
--resource-group "$resourceGroupName" \
--public-network-access "$restore_value" \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account access restored"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to restore Storage Account access"
fi
- else
- echo "Storage Account access unchanged (already at desired state)"
fi
# Restore Storage Account network default action
if [ -n "$original_storage_default_action" ] && [ "$original_storage_default_action" != "Allow" ]; then
- echo "Restoring Storage Account network default action to: $original_storage_default_action"
+ echo "ā Restoring Storage Account network default action"
az storage account update \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
--default-action "$original_storage_default_action" \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account network default action restored"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to restore Storage Account network default action"
fi
- else
- echo "Storage Account network default action unchanged (already at desired state)"
fi
# Restore AI Foundry access
if [ -n "$original_foundry_public_access" ] && [ "$original_foundry_public_access" != "Enabled" ]; then
- echo "Restoring AI Foundry public access to: $original_foundry_public_access"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Restoring AI Foundry access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$aif_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess="$original_foundry_public_access" \
--set properties.apiProperties.qnaAzureSearchEndpointKey="" \
--set properties.networkAcls.bypass="AzureServices" \
--output none 2>/dev/null; then
- echo "ā AI Foundry access restored"
- else
- echo "ā Warning: Failed to restore AI Foundry access automatically."
- echo " Please manually restore network access in the Azure portal if needed."
+ echo "ā Failed to restore AI Foundry access - please check Azure portal"
fi
- else
- echo "AI Foundry access unchanged (already at desired state)"
fi
# Restore CU Foundry access
if [ -n "$original_cu_foundry_public_access" ] && [ "$original_cu_foundry_public_access" != "Enabled" ]; then
- echo "Restoring CU Foundry public access to: $original_cu_foundry_public_access"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Restoring CU Foundry access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$cu_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess="$original_cu_foundry_public_access" \
--set properties.apiProperties.qnaAzureSearchEndpointKey="" \
--set properties.networkAcls.bypass="AzureServices" \
--output none 2>/dev/null; then
- echo "ā CU Foundry access restored"
- else
- echo "ā Warning: Failed to restore CU Foundry access automatically."
- echo " Please manually restore network access in the Azure portal if needed."
+ echo "ā Failed to restore CU Foundry access - please check Azure portal"
fi
- else
- echo "CU Foundry access unchanged (already at desired state)"
fi
# Restore SQL Server public access
if [ -n "$original_sql_public_access" ] && [ "$original_sql_public_access" != "Enabled" ]; then
- echo "Restoring SQL Server public access to: $original_sql_public_access"
+ echo "ā Restoring SQL Server access"
case "$original_sql_public_access" in
"enabled"|"Enabled") restore_value=true ;;
"disabled"|"Disabled") restore_value=false ;;
@@ -329,27 +283,10 @@ restore_network_access() {
--resource-group "$resourceGroupName" \
--enable-public-network $restore_value \
--output none
- if [ $? -eq 0 ]; then
- echo "ā SQL Server access restored"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to restore SQL Server access"
fi
- else
- echo "SQL Server access unchanged (already at desired state)"
fi
-
- # Remove temporary allow-all firewall rule if we created it
- if [ "$created_sql_allow_all_firewall_rule" = "true" ] && [ "$original_full_range_rule_present" != "true" ]; then
- echo "Removing temporary allow-all firewall rule..."
- az sql server firewall-rule delete \
- --resource-group "$resourceGroupName" \
- --server "$sqlServerName" \
- --name "TempAllowAll" \
- --output none 2>/dev/null
- echo "ā Temporary firewall rule removed"
- fi
-
- echo "=== Network access restoration completed ==="
}
# Function to handle script cleanup on exit
@@ -357,11 +294,9 @@ cleanup_on_exit() {
exit_code=$?
echo ""
if [ $exit_code -ne 0 ]; then
- echo "ā Script failed with exit code $exit_code"
- echo "Restoring network access settings before exit..."
+ echo "ā Script failed"
else
echo "ā
Script completed successfully"
- echo "Restoring network access settings..."
fi
restore_network_access
exit $exit_code
@@ -380,7 +315,6 @@ check_azd_installed() {
}
get_values_from_azd_env() {
- echo "Getting values from azd environment variables..."
# Use grep with a regex to ensure we're only capturing sanitized values to avoid command injection
resourceGroupName=$(azd env get-value RESOURCE_GROUP_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$')
storageAccountName=$(azd env get-value STORAGE_ACCOUNT_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$')
@@ -407,19 +341,19 @@ get_values_from_azd_env() {
if [ -z "$resourceGroupName" ] || [ -z "$storageAccountName" ] || [ -z "$fileSystem" ] || [ -z "$sqlServerName" ] || [ -z "$SqlDatabaseName" ] || [ -z "$backendUserMidClientId" ] || [ -z "$backendUserMidDisplayName" ] || [ -z "$aiSearchName" ] || [ -z "$aif_resource_id" ]; then
echo "Error: One or more required values could not be retrieved from azd environment."
return 1
- else
- echo "All values retrieved successfully from azd environment."
- return 0
fi
+ return 0
}
-
-
-# Use Azure CLI login if running locally
-echo ""
-echo "Authenticating with Azure CLI..."
-az login --use-device-code
-echo ""
+# Check if user is logged in to Azure
+echo "Checking Azure authentication..."
+if az account show &> /dev/null; then
+ echo "Already authenticated with Azure."
+else
+ # Use Azure CLI login if running locally
+ echo "Authenticating with Azure CLI..."
+ az login --use-device-code
+fi
if check_azd_installed; then
azSubscriptionId=$(azd env get-value AZURE_SUBSCRIPTION_ID) || azSubscriptionId="$AZURE_SUBSCRIPTION_ID" || azSubscriptionId=""
@@ -507,61 +441,31 @@ if [ $? -ne 0 ]; then
exit 1
fi
-# Determine the correct Python command
-if command -v python3 &> /dev/null; then
- PYTHON_CMD="python3"
-elif command -v python &> /dev/null; then
- PYTHON_CMD="python"
-else
- echo "Python is not installed on this system. Or it is not added in the PATH."
- exit 1
-fi
-
pythonScriptPath="infra/scripts/index_scripts/"
-# create virtual environment
-# Check if the virtual environment already exists
-if [ -d $pythonScriptPath"scriptenv" ]; then
- echo "Virtual environment already exists. Skipping creation."
-else
- echo "Creating virtual environment"
- $PYTHON_CMD -m venv $pythonScriptPath"scriptenv"
-fi
-
-# Activate the virtual environment
-if [ -f $pythonScriptPath"scriptenv/bin/activate" ]; then
- echo "Activating virtual environment (Linux/macOS)"
- source $pythonScriptPath"scriptenv/bin/activate"
-elif [ -f $pythonScriptPath"scriptenv/Scripts/activate" ]; then
- echo "Activating virtual environment (Windows)"
- source $pythonScriptPath"scriptenv/Scripts/activate"
-else
- echo "Error activating virtual environment. Requirements may be installed globally."
-fi
# Install the requirements
-echo "Installing requirements"
pip install --quiet -r ${pythonScriptPath}requirements.txt
-echo "Requirements installed"
+if [ $? -ne 0 ]; then
+ echo "Error: Failed to install Python requirements."
+ exit 1
+fi
# Create Content Understanding analyzers
-echo "Creating Content Understanding analyzer templates..."
-echo "Running 02_create_cu_template_text.py"
+echo "ā Creating Content Understanding analyzer templates"
python infra/scripts/index_scripts/02_create_cu_template_text.py --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion"
if [ $? -ne 0 ]; then
echo "Error: 02_create_cu_template_text.py failed."
exit 1
fi
-echo "Running 02_create_cu_template_audio.py"
python infra/scripts/index_scripts/02_create_cu_template_audio.py --cu_endpoint="$cuEndpoint" --cu_api_version="$cuApiVersion"
if [ $? -ne 0 ]; then
echo "Error: 02_create_cu_template_audio.py failed."
exit 1
fi
-echo "Content Understanding analyzers created successfully."
# Run 04_cu_process_custom_data.py
-echo "Running 04_cu_process_custom_data.py..."
+echo "ā Processing custom data"
sql_server_fqdn="$sqlServerName.database.windows.net"
python infra/scripts/index_scripts/04_cu_process_custom_data.py \
--search_endpoint "$searchEndpoint" \
@@ -579,4 +483,3 @@ if [ $? -ne 0 ]; then
echo "Error: 04_cu_process_custom_data.py failed."
exit 1
fi
-echo "04_cu_process_custom_data.py completed successfully."
diff --git a/infra/scripts/process_sample_data.sh b/infra/scripts/process_sample_data.sh
index 9dbbcece2..d91e106d3 100644
--- a/infra/scripts/process_sample_data.sh
+++ b/infra/scripts/process_sample_data.sh
@@ -49,10 +49,8 @@ original_full_range_rule_present="false"
# Function to enable public network access temporarily
enable_public_access() {
- echo "=== Temporarily enabling public network access for services ==="
# Enable public access for Storage Account
- echo "Enabling public access for Storage Account: $storageAccountName"
original_storage_public_access=$(az storage account show \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
@@ -65,37 +63,30 @@ enable_public_access() {
-o tsv)
if [ "$original_storage_public_access" != "Enabled" ]; then
+ echo "ā Enabling Storage Account public access"
az storage account update \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
--public-network-access Enabled \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account public access enabled"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to enable Storage Account public access"
return 1
fi
- else
- echo "ā Storage Account public access already enabled"
fi
# Also ensure the default network action allows access
if [ "$original_storage_default_action" != "Allow" ]; then
- echo "Setting Storage Account network default action to Allow"
+ echo "ā Setting Storage Account network default action to Allow"
az storage account update \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
--default-action Allow \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account network default action set to Allow"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to set Storage Account network default action"
return 1
fi
- else
- echo "ā Storage Account network default action already set to Allow"
fi
# Enable public access for AI Foundry
@@ -113,21 +104,16 @@ enable_public_access() {
--output tsv)
if [ -z "$original_foundry_public_access" ] || [ "$original_foundry_public_access" = "null" ]; then
- echo "ā Info: Could not retrieve AI Foundry network access status."
+ echo "ā Could not retrieve AI Foundry network access status"
elif [ "$original_foundry_public_access" != "Enabled" ]; then
- echo "Current AI Foundry public access: $original_foundry_public_access"
- echo "Enabling public access for AI Foundry resource: $aif_resource_name (Resource Group: $aif_resource_group)"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Enabling AI Foundry public access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$aif_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \
--output none; then
- echo "ā AI Foundry public access enabled"
- else
- echo "ā Warning: Failed to enable AI Foundry public access automatically."
+ echo "ā Failed to enable AI Foundry public access"
fi
- else
- echo "ā AI Foundry public access already enabled"
fi
fi
@@ -146,26 +132,20 @@ enable_public_access() {
--output tsv)
if [ -z "$original_cu_foundry_public_access" ] || [ "$original_cu_foundry_public_access" = "null" ]; then
- echo "ā Info: Could not retrieve CU Foundry network access status."
+ echo "ā Could not retrieve CU Foundry network access status"
elif [ "$original_cu_foundry_public_access" != "Enabled" ]; then
- echo "Current CU Foundry public access: $original_cu_foundry_public_access"
- echo "Enabling public access for CU Foundry resource: $cu_resource_name (Resource Group: $cu_resource_group)"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Enabling CU Foundry public access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$cu_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess=Enabled properties.apiProperties="{}" \
--output none; then
- echo "ā CU Foundry public access enabled"
- else
- echo "ā Warning: Failed to enable CU Foundry public access automatically."
+ echo "ā Failed to enable CU Foundry public access"
fi
- else
- echo "ā CU Foundry public access already enabled"
fi
fi
# Enable public access for SQL Server
- echo "Checking SQL Server public access: $sqlServerName"
original_sql_public_access=$(az sql server show \
--name "$sqlServerName" \
--resource-group "$resourceGroupName" \
@@ -173,20 +153,16 @@ enable_public_access() {
-o tsv)
if [ "$original_sql_public_access" != "Enabled" ]; then
- echo "Enabling public access for SQL Server"
+ echo "ā Enabling SQL Server public access"
az sql server update \
--name "$sqlServerName" \
--resource-group "$resourceGroupName" \
--enable-public-network true \
--output none
- if [ $? -eq 0 ]; then
- echo "ā SQL Server public access enabled"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to enable SQL Server public access"
return 1
fi
- else
- echo "ā SQL Server public access already enabled"
fi
# Create temporary allow-all firewall rule for SQL Server
@@ -209,43 +185,34 @@ enable_public_access() {
--query "[?name=='${sql_allow_all_rule_name}'] | [0].name" \
-o tsv 2>/dev/null)
- if [ -z "$existing_allow_all_rule" ]; then
- if [ -n "$pre_existing_full_range_rule" ]; then
- echo "ā Existing rule ($pre_existing_full_range_rule) already allows full IP range."
- else
- echo "Creating temporary allow-all firewall rule ($sql_allow_all_rule_name)..."
- if az sql server firewall-rule create \
- --resource-group "$resourceGroupName" \
- --server "$sqlServerName" \
- --name "$sql_allow_all_rule_name" \
- --start-ip-address 0.0.0.0 \
- --end-ip-address 255.255.255.255 \
- --output none; then
- created_sql_allow_all_firewall_rule="true"
- echo "ā Temporary allow-all firewall rule created"
- else
- echo "ā Warning: Failed to create allow-all firewall rule"
- fi
- fi
+ if [ -z "$existing_allow_all_rule" ] && [ -z "$pre_existing_full_range_rule" ]; then
+ echo "ā Creating temporary SQL firewall rule"
+ if az sql server firewall-rule create \
+ --resource-group "$resourceGroupName" \
+ --server "$sqlServerName" \
+ --name "$sql_allow_all_rule_name" \
+ --start-ip-address 0.0.0.0 \
+ --end-ip-address 255.255.255.255 \
+ --output none; then
+ created_sql_allow_all_firewall_rule="true"
+ else
+ echo "ā Failed to create firewall rule"
+ fi
else
- echo "ā Temporary allow-all firewall rule already present"
- original_full_range_rule_present="true"
+ original_full_range_rule_present="true"
fi
# Wait a bit for changes to take effect
- echo "Waiting for network access changes to propagate..."
sleep 10
- echo "=== Public network access enabled successfully ==="
return 0
}
# Function to restore original network access settings
restore_network_access() {
- echo "=== Restoring original network access settings ==="
# Restore Storage Account access
if [ -n "$original_storage_public_access" ] && [ "$original_storage_public_access" != "Enabled" ]; then
- echo "Restoring Storage Account public access to: $original_storage_public_access"
+ echo "ā Restoring Storage Account access"
case "$original_storage_public_access" in
"enabled"|"Enabled") restore_value="Enabled" ;;
"disabled"|"Disabled") restore_value="Disabled" ;;
@@ -256,74 +223,56 @@ restore_network_access() {
--resource-group "$resourceGroupName" \
--public-network-access "$restore_value" \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account access restored"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to restore Storage Account access"
fi
- else
- echo "Storage Account access unchanged (already at desired state)"
fi
# Restore Storage Account network default action
if [ -n "$original_storage_default_action" ] && [ "$original_storage_default_action" != "Allow" ]; then
- echo "Restoring Storage Account network default action to: $original_storage_default_action"
+ echo "ā Restoring Storage Account network default action"
az storage account update \
--name "$storageAccountName" \
--resource-group "$resourceGroupName" \
--default-action "$original_storage_default_action" \
--output none
- if [ $? -eq 0 ]; then
- echo "ā Storage Account network default action restored"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to restore Storage Account network default action"
fi
- else
- echo "Storage Account network default action unchanged (already at desired state)"
fi
# Restore AI Foundry access
if [ -n "$original_foundry_public_access" ] && [ "$original_foundry_public_access" != "Enabled" ]; then
- echo "Restoring AI Foundry public access to: $original_foundry_public_access"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Restoring AI Foundry access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$aif_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess="$original_foundry_public_access" \
--set properties.apiProperties.qnaAzureSearchEndpointKey="" \
--set properties.networkAcls.bypass="AzureServices" \
--output none 2>/dev/null; then
- echo "ā AI Foundry access restored"
- else
- echo "ā Warning: Failed to restore AI Foundry access automatically."
- echo " Please manually restore network access in the Azure portal if needed."
+ echo "ā Failed to restore AI Foundry access - please check Azure portal"
fi
- else
- echo "AI Foundry access unchanged (already at desired state)"
fi
# Restore CU Foundry access
if [ -n "$original_cu_foundry_public_access" ] && [ "$original_cu_foundry_public_access" != "Enabled" ]; then
- echo "Restoring CU Foundry public access to: $original_cu_foundry_public_access"
- if MSYS_NO_PATHCONV=1 az resource update \
+ echo "ā Restoring CU Foundry access"
+ if ! MSYS_NO_PATHCONV=1 az resource update \
--ids "$cu_account_resource_id" \
--api-version 2024-10-01 \
--set properties.publicNetworkAccess="$original_cu_foundry_public_access" \
--set properties.apiProperties.qnaAzureSearchEndpointKey="" \
--set properties.networkAcls.bypass="AzureServices" \
--output none 2>/dev/null; then
- echo "ā CU Foundry access restored"
- else
- echo "ā Warning: Failed to restore CU Foundry access automatically."
- echo " Please manually restore network access in the Azure portal if needed."
+ echo "ā Failed to restore CU Foundry access - please check Azure portal"
fi
- else
- echo "CU Foundry access unchanged (already at desired state)"
fi
# Restore SQL Server public access
if [ -n "$original_sql_public_access" ] && [ "$original_sql_public_access" != "Enabled" ]; then
- echo "Restoring SQL Server public access to: $original_sql_public_access"
+ echo "ā Restoring SQL Server access"
case "$original_sql_public_access" in
"enabled"|"Enabled") restore_value=true ;;
"disabled"|"Disabled") restore_value=false ;;
@@ -334,16 +283,10 @@ restore_network_access() {
--resource-group "$resourceGroupName" \
--enable-public-network $restore_value \
--output none
- if [ $? -eq 0 ]; then
- echo "ā SQL Server access restored"
- else
+ if [ $? -ne 0 ]; then
echo "ā Failed to restore SQL Server access"
fi
- else
- echo "SQL Server access unchanged (already at desired state)"
fi
-
- echo "=== Network access restoration completed ==="
}
# Function to handle script cleanup on exit
@@ -351,11 +294,9 @@ cleanup_on_exit() {
exit_code=$?
echo ""
if [ $exit_code -ne 0 ]; then
- echo "ā Script failed with exit code $exit_code"
- echo "Restoring network access settings before exit..."
+ echo "ā Script failed"
else
echo "ā
Script completed successfully"
- echo "Restoring network access settings..."
fi
restore_network_access
exit $exit_code
@@ -374,7 +315,6 @@ check_azd_installed() {
}
get_values_from_azd_env() {
- echo "Getting values from azd environment variables..."
# Use grep with a regex to ensure we're only capturing sanitized values to avoid command injection
resourceGroupName=$(azd env get-value RESOURCE_GROUP_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$')
storageAccountName=$(azd env get-value STORAGE_ACCOUNT_NAME 2>&1 | grep -E '^[a-zA-Z0-9._/-]+$')
@@ -401,19 +341,21 @@ get_values_from_azd_env() {
if [ -z "$resourceGroupName" ] || [ -z "$storageAccountName" ] || [ -z "$fileSystem" ] || [ -z "$sqlServerName" ] || [ -z "$SqlDatabaseName" ] || [ -z "$backendUserMidClientId" ] || [ -z "$backendUserMidDisplayName" ] || [ -z "$aiSearchName" ] || [ -z "$aif_resource_id" ]; then
echo "Error: One or more required values could not be retrieved from azd environment."
return 1
- else
- echo "All values retrieved successfully from azd environment."
- return 0
fi
+ return 0
}
-# Use Azure CLI login if running locally
-echo ""
-echo "Authenticating with Azure CLI..."
-az login --use-device-code
-echo ""
+# Check if user is logged in to Azure
+echo "Checking Azure authentication..."
+if az account show &> /dev/null; then
+ echo "Already authenticated with Azure."
+else
+ # Use Azure CLI login if running locally
+ echo "Authenticating with Azure CLI..."
+ az login --use-device-code
+fi
if check_azd_installed; then
azSubscriptionId=$(azd env get-value AZURE_SUBSCRIPTION_ID) || azSubscriptionId="$AZURE_SUBSCRIPTION_ID" || azSubscriptionId=""
@@ -520,8 +462,4 @@ if [ $? -ne 0 ]; then
fi
echo "run_create_index_scripts.sh completed successfully."
-## SQL role assignment now centralized in run_create_index_scripts.sh; removed local duplicate block.
-
echo "All scripts executed successfully."
-echo "Network access will be restored to original settings..."
-# Note: cleanup_on_exit will be called automatically via the trap
diff --git a/infra/scripts/run_create_index_scripts.sh b/infra/scripts/run_create_index_scripts.sh
index 868d74a34..d43e1d044 100644
--- a/infra/scripts/run_create_index_scripts.sh
+++ b/infra/scripts/run_create_index_scripts.sh
@@ -20,20 +20,13 @@ ai_agent_endpoint="${16}"
pythonScriptPath="infra/scripts/index_scripts/"
-echo "Script Started"
-
# Authenticate with Azure
-if az account show &> /dev/null; then
- echo "Already authenticated with Azure."
-else
- echo "Not authenticated with Azure. Attempting to authenticate..."
- # Use Azure CLI login
+if ! az account show &> /dev/null; then
echo "Authenticating with Azure CLI..."
- az login
+ az login --use-device-code
fi
# Get signed in user and store the output
-echo "Getting signed in user id and display name"
signed_user=$(az ad signed-in-user show --query "{id:id, displayName:displayName}" -o json)
signed_user_id=$(echo "$signed_user" | grep -o '"id": *"[^"]*"' | head -1 | sed 's/"id": *"\([^"]*\)"/\1/')
signed_user_display_name=$(echo "$signed_user" | grep -o '"displayName": *"[^"]*"' | sed 's/"displayName": *"\([^"]*\)"/\1/')
@@ -41,147 +34,88 @@ signed_user_display_name=$(echo "$signed_user" | grep -o '"displayName": *"[^"]*
# Note: Environment variables are now passed as parameters from process_sample_data.sh
### Assign Azure AI User role to the signed in user for AI Foundry ###
-
-# Check if the user has the Azure AI User role
-echo "Checking if user has the Azure AI User role for AI Foundry"
role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $aif_resource_id --assignee $signed_user_id --query "[].roleDefinitionId" -o tsv)
if [ -z "$role_assignment" ]; then
- echo "User does not have the Azure AI User role for AI Foundry. Assigning the role."
+ echo "ā Assigning Azure AI User role for AI Foundry"
MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $aif_resource_id --output none
- if [ $? -eq 0 ]; then
- echo "Azure AI User role for AI Foundry assigned successfully."
- else
- echo "Failed to assign Azure AI User role for AI Foundry."
+ if [ $? -ne 0 ]; then
+ echo "ā Failed to assign Azure AI User role for AI Foundry"
exit 1
fi
-else
- echo "User already has the Azure AI User role for AI Foundry."
fi
### Assign Azure AI User role to the signed in user for CU Foundry ###
-
if [ -n "$cu_foundry_resource_id" ] && [ "$cu_foundry_resource_id" != "null" ]; then
- echo "Checking if user has the Azure AI User role for CU Foundry"
role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $cu_foundry_resource_id --assignee $signed_user_id --query "[].roleDefinitionId" -o tsv)
if [ -z "$role_assignment" ]; then
- echo "User does not have the Azure AI User role for CU Foundry. Assigning the role."
+ echo "ā Assigning Azure AI User role for CU Foundry"
MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role 53ca6127-db72-4b80-b1b0-d745d6d5456d --scope $cu_foundry_resource_id --output none
- if [ $? -eq 0 ]; then
- echo "Azure AI User role for CU Foundry assigned successfully."
- else
- echo "Failed to assign Azure AI User role for CU Foundry."
+ if [ $? -ne 0 ]; then
+ echo "ā Failed to assign Azure AI User role for CU Foundry"
exit 1
fi
- else
- echo "User already has the Azure AI User role for CU Foundry."
fi
fi
### Assign Search Index Data Contributor role to the signed in user ###
-
-echo "Getting Azure Search resource id"
search_resource_id=$(az search service show --name $aiSearchName --resource-group $resourceGroupName --query id --output tsv)
-echo "Checking if user has the Search Index Data Contributor role"
role_assignment=$(MSYS_NO_PATHCONV=1 az role assignment list --assignee $signed_user_id --role "Search Index Data Contributor" --scope $search_resource_id --query "[].roleDefinitionId" -o tsv)
if [ -z "$role_assignment" ]; then
- echo "User does not have the Search Index Data Contributor role. Assigning the role."
+ echo "ā Assigning Search Index Data Contributor role"
MSYS_NO_PATHCONV=1 az role assignment create --assignee $signed_user_id --role "Search Index Data Contributor" --scope $search_resource_id --output none
- if [ $? -eq 0 ]; then
- echo "Search Index Data Contributor role assigned successfully."
- else
- echo "Failed to assign Search Index Data Contributor role."
+ if [ $? -ne 0 ]; then
+ echo "ā Failed to assign Search Index Data Contributor role"
exit 1
fi
-else
- echo "User already has the Search Index Data Contributor role."
fi
### Assign signed in user as SQL Server Admin ###
-
-echo "Getting Azure SQL Server resource id"
sql_server_resource_id=$(az sql server show --name $sqlServerName --resource-group $resourceGroupName --query id --output tsv)
-
-# Check if the user is Azure SQL Server Admin
-echo "Checking if user is Azure SQL Server Admin"
admin=$(MSYS_NO_PATHCONV=1 az sql server ad-admin list --ids $sql_server_resource_id --query "[?sid == '$signed_user_id']" -o tsv)
-# Check if the role exists
-if [ -n "$admin" ]; then
- echo "User is already Azure SQL Server Admin"
-else
- echo "User is not Azure SQL Server Admin. Assigning the role."
+if [ -z "$admin" ]; then
+ echo "ā Assigning user as SQL Server Admin"
MSYS_NO_PATHCONV=1 az sql server ad-admin create --display-name "$signed_user_display_name" --object-id $signed_user_id --resource-group $resourceGroupName --server $sqlServerName --output none
- if [ $? -eq 0 ]; then
- echo "Assigned user as Azure SQL Server Admin."
- else
- echo "Failed to assign Azure SQL Server Admin role."
+ if [ $? -ne 0 ]; then
+ echo "ā Failed to assign SQL Server Admin role"
exit 1
fi
fi
-
-# Determine the correct Python command
-if command -v python3 &> /dev/null; then
- PYTHON_CMD="python3"
-elif command -v python &> /dev/null; then
- PYTHON_CMD="python"
-else
- echo "Python is not installed on this system. Or it is not added in the PATH."
- exit 1
-fi
-
-# create virtual environment
-# Check if the virtual environment already exists
-if [ -d $pythonScriptPath"scriptenv" ]; then
- echo "Virtual environment already exists. Skipping creation."
-else
- echo "Creating virtual environment"
- $PYTHON_CMD -m venv $pythonScriptPath"scriptenv"
-fi
-
-# Activate the virtual environment
-if [ -f $pythonScriptPath"scriptenv/bin/activate" ]; then
- echo "Activating virtual environment (Linux/macOS)"
- source $pythonScriptPath"scriptenv/bin/activate"
-elif [ -f $pythonScriptPath"scriptenv/Scripts/activate" ]; then
- echo "Activating virtual environment (Windows)"
- source $pythonScriptPath"scriptenv/Scripts/activate"
-else
- echo "Error activating virtual environment. Requirements may be installed globally."
-fi
-
# Install the requirements
echo "Installing requirements"
pip install --quiet -r ${pythonScriptPath}requirements.txt
-echo "Requirements installed"
+if [ $? -ne 0 ]; then
+ echo "Error: Failed to install Python requirements."
+ exit 1
+fi
error_flag=false
-echo "Running the python scripts"
-echo "Creating the search index"
+echo "ā Creating search index"
python ${pythonScriptPath}01_create_search_index.py --search_endpoint="$search_endpoint" --openai_endpoint="$openai_endpoint" --embedding_model="$embedding_model"
if [ $? -ne 0 ]; then
echo "Error: 01_create_search_index.py failed."
error_flag=true
fi
-echo "Creating CU template for text"
+echo "ā Creating CU template for text"
python ${pythonScriptPath}02_create_cu_template_text.py --cu_endpoint="$cu_endpoint" --cu_api_version="$cu_api_version"
if [ $? -ne 0 ]; then
echo "Error: 02_create_cu_template_text.py failed."
error_flag=true
fi
-echo "Creating CU template for audio"
+echo "ā Creating CU template for audio"
python ${pythonScriptPath}02_create_cu_template_audio.py --cu_endpoint="$cu_endpoint" --cu_api_version="$cu_api_version"
if [ $? -ne 0 ]; then
echo "Error: 02_create_cu_template_audio.py failed."
error_flag=true
fi
-echo "Processing data with CU"
+echo "ā Processing data with CU"
sql_server_fqdn="$sqlServerName.database.windows.net"
python ${pythonScriptPath}03_cu_process_data_text.py --search_endpoint="$search_endpoint" --ai_project_endpoint="$ai_agent_endpoint" --deployment_model="$deployment_model" --embedding_model="$embedding_model" --storage_account_name="$storageAccountName" --sql_server="$sql_server_fqdn" --sql_database="$sqlDatabaseName" --cu_endpoint="$cu_endpoint" --cu_api_version="$cu_api_version"
if [ $? -ne 0 ]; then
@@ -194,27 +128,19 @@ if [ -n "$backendManagedIdentityClientId" ] && [ -n "$backendManagedIdentityDisp
mi_display_name="$backendManagedIdentityDisplayName"
server_fqdn="$sqlServerName.database.windows.net"
roles_json="[{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datareader\"},{\"clientId\":\"$backendManagedIdentityClientId\",\"displayName\":\"$mi_display_name\",\"role\":\"db_datawriter\"}]"
- echo "[RoleAssign] Invoking assign_sql_roles.py for roles: db_datareader, db_datawriter"
if [ -f "infra/scripts/add_user_scripts/assign_sql_roles.py" ]; then
+ echo "ā Assigning SQL roles to managed identity"
python infra/scripts/add_user_scripts/assign_sql_roles.py --server "$server_fqdn" --database "$sqlDatabaseName" --roles-json "$roles_json"
if [ $? -ne 0 ]; then
- echo "[RoleAssign] Warning: SQL role assignment failed."
+ echo "ā SQL role assignment failed"
error_flag=true
- else
- echo "[RoleAssign] SQL roles assignment completed successfully."
fi
- else
- echo "[RoleAssign] assign_sql_roles.py not found. Skipping SQL role assignment."
fi
-else
- echo "[RoleAssign] Skipped SQL role assignment due to missing required values."
fi
# Check for any errors and exit if any occurred
if [ "$error_flag" = true ]; then
echo "One or more scripts failed. Please check the logs above."
exit 1
-fi
-
-echo "Scripts completed successfully"
\ No newline at end of file
+fi
\ No newline at end of file