Skip to content

Commit 20f017f

Browse files
dantelmomsftdantelmomsft
committed
adding python script for rag evaluation using azure ai evaluation sdk and ragas
1 parent f9de3df commit 20f017f

26 files changed

Lines changed: 838761 additions & 41 deletions

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ ipython_config.py
9595
# intended to run in multiple environments; otherwise, check them in:
9696
# .python-version
9797

98+
.evalenv
99+
.env
98100
# pipenv
99101
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
100102
# However, in case of collaboration, if having platform-specific dependencies or dependencies

‎.vscode/launch.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,36 @@
88
"hostName": "127.0.0.1",
99
"port": 5005,
1010
"preLaunchTask": "func: host start"
11+
},
12+
{
13+
"name": "Debug evaluate.py",
14+
"type": "debugpy",
15+
"request": "launch",
16+
"program": "${workspaceFolder}/evals/evaluate.py",
17+
"python": "${workspaceFolder}/.evalenv/bin/python",
18+
"cwd": "${workspaceFolder}",
19+
"args": [
20+
"--env-file-path", "./deploy/aca"
21+
],
22+
"console": "integratedTerminal",
23+
"justMyCode": false,
24+
"stopOnEntry": false
25+
},
26+
{
27+
"name": "Debug safety_evaluation.py",
28+
"type": "debugpy",
29+
"request": "launch",
30+
"program": "${workspaceFolder}/evals/safety_evaluation.py",
31+
"python": "${workspaceFolder}/.evalenv/bin/python",
32+
"cwd": "${workspaceFolder}",
33+
"args": [
34+
"--env-file-path", "./deploy/aca",
35+
"--target_url", "http://172.24.128.1:8080/api/chat",
36+
"--max_simulations", "1"
37+
],
38+
"console": "integratedTerminal",
39+
"justMyCode": false,
40+
"stopOnEntry": false
1141
}
1242
]
1343
}

‎deploy/aca/infra/main.bicep‎

Lines changed: 113 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,13 @@ param tenantId string = tenant().tenantId
1313
param authTenantId string = ''
1414

1515
var tenantIdForAuth = !empty(authTenantId) ? authTenantId : tenantId
16-
var authenticationIssuerUri = '${environment().authentication.loginEndpoint}${tenantIdForAuth}/v2.0'
16+
1717

1818
// Used for the optional login and document level access control system
1919
param useAuthentication bool = true
20-
param enableUnauthenticatedAccess bool = false
21-
2220
param serverAppId string = ''
23-
@secure()
24-
param serverAppSecret string = ''
2521
param clientAppId string = ''
26-
@secure()
27-
param clientAppSecret string = ''
22+
2823

2924
@allowed(['None', 'AzureServices'])
3025
@description('If allowedIp is set, whether azure services are allowed to bypass the storage and AI services firewall.')
@@ -69,6 +64,9 @@ param enableGlobalDocumentAccess bool = true
6964
@description('Use Service Bus for indexing documents requests')
7065
param useServiceBusIndexing bool = false
7166

67+
param useEval bool = false
68+
param useSafetyEval bool = false
69+
7270
@allowed(['free', 'provisioned', 'serverless'])
7371
param cosmosDbSkuName string // Set in main.parameters.json
7472
param cosmodDbResourceGroupName string = ''
@@ -104,11 +102,34 @@ param documentIntelligenceResourceGroupLocation string = location
104102

105103
param documentIntelligenceSkuName string = 'S0'
106104

107-
param chatGptDeploymentName string // Set in main.parameters.json
108-
param chatGptDeploymentCapacity int = 80
109-
param chatGptDeploymentSkuName string= 'Standard'
110-
param chatGptModelName string = 'gpt-4o-mini'
111-
param chatGptModelVersion string = '2024-07-18'
105+
param azureOpenAiDisableKeys bool = true
106+
param chatGptModelName string = ''
107+
param chatGptDeploymentName string = ''
108+
param chatGptDeploymentVersion string = ''
109+
param chatGptDeploymentSkuName string = ''
110+
param chatGptDeploymentCapacity int = 0
111+
112+
var chatGpt = {
113+
modelName: !empty(chatGptModelName) ? chatGptModelName : 'gpt-4o-mini'
114+
deploymentName: !empty(chatGptDeploymentName) ? chatGptDeploymentName : 'gpt-4o-mini'
115+
deploymentVersion: !empty(chatGptDeploymentVersion) ? chatGptDeploymentVersion : '2024-07-18'
116+
deploymentSkuName: !empty(chatGptDeploymentSkuName) ? chatGptDeploymentSkuName : 'GlobalStandard' // Not backward-compatible
117+
deploymentCapacity: chatGptDeploymentCapacity != 0 ? chatGptDeploymentCapacity : 30
118+
}
119+
120+
param evalModelName string = ''
121+
param evalDeploymentName string = ''
122+
param evalModelVersion string = ''
123+
param evalDeploymentSkuName string = ''
124+
param evalDeploymentCapacity int = 0
125+
126+
var eval = {
127+
modelName: !empty(evalModelName) ? evalModelName : 'gpt-4o'
128+
deploymentName: !empty(evalDeploymentName) ? evalDeploymentName : 'gpt-4o'
129+
deploymentVersion: !empty(evalModelVersion) ? evalModelVersion : '2024-08-06'
130+
deploymentSkuName: !empty(evalDeploymentSkuName) ? evalDeploymentSkuName : 'GlobalStandard' // Not backward-compatible
131+
deploymentCapacity: evalDeploymentCapacity != 0 ? evalDeploymentCapacity : 30
132+
}
112133

113134
param embeddingModelName string = ''
114135
param embeddingDeploymentName string = ''
@@ -419,43 +440,72 @@ module web './app/web.bicep' = {
419440
}
420441
}
421442

422-
423-
module openAi '../../shared/ai/cognitiveservices.bicep' = {
424-
name: 'openai'
425-
scope: openAiResourceGroup
426-
params: {
427-
name: !empty(openAiServiceName) ? openAiServiceName : '${abbrs.cognitiveServicesAccounts}${resourceToken}'
428-
location: !empty(customOpenAiResourceGroupLocation) ? customOpenAiResourceGroupLocation : openAiResourceGroupLocation
429-
tags: tags
443+
var defaultOpenAiDeployments = [
444+
{
445+
name: chatGpt.deploymentName
446+
model: {
447+
format: 'OpenAI'
448+
name: chatGpt.modelName
449+
version: chatGpt.deploymentVersion
450+
}
430451
sku: {
431-
name: openAiSkuName
452+
name: chatGpt.deploymentSkuName
453+
capacity: chatGpt.deploymentCapacity
432454
}
433-
deployments: [
434-
{
435-
name: chatGptDeploymentName
436-
model: {
437-
format: 'OpenAI'
438-
name: chatGptModelName
439-
version: chatGptModelVersion
440-
}
441-
sku: {
442-
name: chatGptDeploymentSkuName
443-
capacity: chatGptDeploymentCapacity
444-
}
445-
}
455+
}
456+
{
457+
name: embedding.deploymentName
458+
model: {
459+
format: 'OpenAI'
460+
name: embedding.modelName
461+
version: embedding.deploymentVersion
462+
}
463+
sku: {
464+
name: embedding.deploymentSkuName
465+
capacity: embedding.deploymentCapacity
466+
}
467+
}
468+
]
469+
470+
var openAiDeployments = concat(
471+
defaultOpenAiDeployments,
472+
useEval
473+
? [
446474
{
447-
name: embedding.deploymentName
475+
name: eval.deploymentName
448476
model: {
449477
format: 'OpenAI'
450-
name: embedding.modelName
451-
version: embedding.deploymentVersion
478+
name: eval.modelName
479+
version: eval.deploymentVersion
452480
}
453481
sku: {
454-
name: embedding.deploymentSkuName
455-
capacity: embedding.deploymentCapacity
482+
name: eval.deploymentSkuName
483+
capacity: eval.deploymentCapacity
456484
}
457485
}
458-
]
486+
] : []
487+
)
488+
489+
490+
module openAi 'br/public:avm/res/cognitive-services/account:0.7.2' = {
491+
name: 'openai'
492+
scope: openAiResourceGroup
493+
params: {
494+
name: !empty(openAiServiceName) ? openAiServiceName : '${abbrs.cognitiveServicesAccounts}${resourceToken}'
495+
location: !empty(customOpenAiResourceGroupLocation) ? customOpenAiResourceGroupLocation : openAiResourceGroupLocation
496+
tags: tags
497+
kind: 'OpenAI'
498+
customSubDomainName: !empty(openAiServiceName)
499+
? openAiServiceName
500+
: '${abbrs.cognitiveServicesAccounts}${resourceToken}'
501+
publicNetworkAccess: publicNetworkAccess
502+
networkAcls: {
503+
defaultAction: 'Allow'
504+
bypass: bypass
505+
}
506+
sku: openAiSkuName
507+
deployments: openAiDeployments
508+
disableLocalAuth: azureOpenAiDisableKeys
459509
}
460510
}
461511

@@ -501,7 +551,7 @@ module storage '../../shared/storage/storage-account.bicep' = {
501551
tags: tags
502552
allowBlobPublicAccess: false
503553
publicNetworkAccess: 'Enabled'
504-
isHnsEnabled: true
554+
isHnsEnabled: false
505555
sku: {
506556
name: storageSkuName
507557
}
@@ -610,6 +660,21 @@ module cosmosDb 'br/public:avm/res/document-db/database-account:0.6.1' = if (use
610660
}
611661
}
612662

663+
module ai '../../shared/ai/ai-environment.bicep' = if (useSafetyEval) {
664+
name: 'ai'
665+
scope: resourceGroup
666+
params: {
667+
// Limited region support: https://learn.microsoft.com/azure/ai-foundry/how-to/develop/evaluate-sdk#region-support
668+
location: 'eastus2'
669+
tags: tags
670+
hubName: 'aihub-${resourceToken}'
671+
projectName: 'aiproj-${resourceToken}'
672+
storageAccountId: storage.outputs.id
673+
applicationInsightsId: !useApplicationInsights ? '' : monitoring.outputs.applicationInsightsId
674+
}
675+
}
676+
677+
613678

614679
// USER ROLES
615680
module openAiRoleUser '../../shared/security/role.bicep' = {
@@ -835,6 +900,12 @@ output AZURE_OPENAI_EMB_DEPLOYMENT string = embedding.deploymentName
835900
output AZURE_OPENAI_EMB_DEPLOYMENT_VERSION string = embedding.deploymentVersion
836901
output AZURE_OPENAI_EMB_DEPLOYMENT_SKU string = embedding.deploymentSkuName
837902

903+
// Specific to Azure OpenAI with eval
904+
output AZURE_OPENAI_EVAL_DEPLOYMENT string = useEval ? eval.deploymentName : ''
905+
output AZURE_OPENAI_EVAL_DEPLOYMENT_VERSION string = useEval ? eval.deploymentVersion : ''
906+
output AZURE_OPENAI_EVAL_DEPLOYMENT_SKU string = useEval ? eval.deploymentSkuName : ''
907+
output AZURE_OPENAI_EVAL_MODEL string = useEval ? eval.modelName : ''
908+
838909
// Used only with non-Azure OpenAI deployments
839910
output OPENAI_API_KEY string = openAiApiKey
840911
output OPENAI_ORGANIZATION string = openAiApiOrganization
@@ -865,6 +936,7 @@ output USE_CHAT_HISTORY_BROWSER bool = useChatHistoryBrowser
865936
output USE_CHAT_HISTORY_COSMOS bool = useChatHistoryCosmos
866937
output USE_SERVICEBUS_INDEXING bool = useServiceBusIndexing
867938

939+
output AZURE_AI_PROJECT string = useSafetyEval ? ai.outputs.projectName : ''
868940
output WEB_URI string = web.outputs.SERVICE_WEB_URI
869941
output INDEXER_URI string = indexer.outputs.SERVICE_INDEXER_URI
870942
// output INDEXER_FUNCTIONAPP_NAME string = indexer.outputs.name

‎deploy/aca/infra/main.parameters.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,21 @@
8383
"chatGptDeploymentSkuName": {
8484
"value": "${AZURE_OPENAI_CHATGPT_DEPLOYMENT_SKU_NAME=Standard}"
8585
},
86+
"evalModelName":{
87+
"value": "${AZURE_OPENAI_EVAL_MODEL}"
88+
},
89+
"evalModelVersion":{
90+
"value": "${AZURE_OPENAI_EVAL_MODEL_VERSION}"
91+
},
92+
"evalDeploymentName": {
93+
"value": "${AZURE_OPENAI_EVAL_DEPLOYMENT}"
94+
},
95+
"evalDeploymentSkuName":{
96+
"value": "${AZURE_OPENAI_EVAL_DEPLOYMENT_SKU}"
97+
},
98+
"evalDeploymentCapacity":{
99+
"value": "${AZURE_OPENAI_EVAL_DEPLOYMENT_CAPACITY}"
100+
},
86101
"embeddingModelName": {
87102
"value": "${AZURE_OPENAI_EMB_MODEL}"
88103
},
@@ -147,6 +162,12 @@
147162
"useServiceBusIndexing": {
148163
"value": "${USE_SERVICEBUS_INDEXING=true}"
149164
},
165+
"useEval": {
166+
"value": "${USE_EVAL=false}"
167+
},
168+
"useSafetyEval": {
169+
"value": "${USE_SAFETY_EVAL=false}"
170+
},
150171
"apiAppExists": {
151172
"value": false
152173
},

‎deploy/shared/ai/ai-environment.bicep‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
@minLength(1)
2+
@description('Primary location for all resources')
3+
param location string
4+
5+
@description('The AI Hub resource name.')
6+
param hubName string
7+
@description('The AI Project resource name.')
8+
param projectName string
9+
@description('The Storage Account resource ID.')
10+
param storageAccountId string
11+
@description('The Application Insights resource ID.')
12+
param applicationInsightsId string = ''
13+
@description('The Azure Search resource name.')
14+
param searchServiceName string = ''
15+
@description('The Azure Search connection name.')
16+
param searchConnectionName string = ''
17+
param tags object = {}
18+
19+
module hub './hub.bicep' = {
20+
name: 'hub'
21+
params: {
22+
location: location
23+
tags: tags
24+
name: hubName
25+
displayName: hubName
26+
storageAccountId: storageAccountId
27+
containerRegistryId: null
28+
applicationInsightsId: applicationInsightsId
29+
aiSearchName: searchServiceName
30+
aiSearchConnectionName: searchConnectionName
31+
}
32+
}
33+
34+
module project './project.bicep' = {
35+
name: 'project'
36+
params: {
37+
location: location
38+
tags: tags
39+
name: projectName
40+
displayName: projectName
41+
hubName: hub.outputs.name
42+
}
43+
}
44+
45+
46+
output projectName string = project.outputs.name

0 commit comments

Comments
 (0)