Skip to content

Commit 448ca84

Browse files
antriksh30antriksh30
authored
Update hosted agent cloud e2e pipeline toolbox endpoint and remove cleanup and role grant access step (microsoft-foundry#405)
1 parent 40d3607 commit 448ca84

1 file changed

Lines changed: 4 additions & 211 deletions

File tree

‎.github/workflows/hosted-agents-cloud-e2e.yml‎

Lines changed: 4 additions & 211 deletions
Original file line numberDiff line numberDiff line change
@@ -86,9 +86,9 @@ jobs:
8686
- name: Find samples to test
8787
id: find-samples
8888
env:
89-
TOOLBOX_ENDPOINT_LIST: ${{ vars.TOOLBOX_ENDPOINT }}
89+
TOOLBOX_ENDPOINT_LIST: ${{ vars.TOOLBOX_ENDPOINT_NCUS }}
9090
run: |
91-
# ── Parse TOOLBOX_ENDPOINT repo variable into a list ───────────
91+
# ── Parse TOOLBOX_ENDPOINT_NCUS repo variable into a list ──────
9292
# Format: one entry per line, "label=https://...|optional query text"
9393
# Used to expand toolbox samples into a cartesian product.
9494
toolboxes=()
@@ -137,25 +137,9 @@ jobs:
137137
case "$sample_dir" in
138138
samples/python/hosted-agents/*toolbox*|samples/csharp/hosted-agents/*toolbox*) is_toolbox="true" ;;
139139
esac
140-
# Per-sample project override. All toolbox-consuming samples are
141-
# routed to the westus2 project because (a) the agent-framework
142-
# samples resolve their toolbox server-side and need the agent to
143-
# live in the project that hosts the toolbox, and (b) the BYO
144-
# samples invoke the toolbox MCP URL with their own runtime MI —
145-
# co-locating the agent with the toolbox avoids cross-account
146-
# auth. The MCP dot-prefix tool-name bug also still reproduces on
147-
# the ncus project for Foundry-hosted toolboxes (web-search,
148-
# ms-learn, github-mcp), while westus2 has the fix.
140+
# Toolbox samples use the default ncus project, so the matrix is
141+
# expanded from the TOOLBOX_ENDPOINT_NCUS repo variable.
149142
local use_westus2="false"
150-
case "$sample_dir" in
151-
samples/python/hosted-agents/agent-framework/responses/04-foundry-toolbox|\
152-
samples/csharp/hosted-agents/agent-framework/foundry-toolbox-server-side|\
153-
samples/python/hosted-agents/bring-your-own/invocations/toolbox|\
154-
samples/python/hosted-agents/bring-your-own/responses/bring-your-own-toolbox|\
155-
samples/python/hosted-agents/bring-your-own/responses/langgraph-toolbox|\
156-
samples/python/hosted-agents/bring-your-own/responses/langgraph-toolbox-user-identity)
157-
use_westus2="true" ;;
158-
esac
159143
if [ "$is_toolbox" = "true" ]; then
160144
if [ "$toolboxes_json" = "[]" ]; then
161145
echo "::warning::Skipping toolbox sample $sample_dir — TOOLBOX_ENDPOINT repo variable is empty" >&2
@@ -803,107 +787,6 @@ jobs:
803787
echo "::warning::Agent may not be active yet — proceeding with invoke anyway"
804788
timeout-minutes: 3
805789

806-
# ── Grant model access to agent identity (toolbox samples) ──────
807-
# Each azd deploy creates a new ephemeral AgentIdentityBlueprint SP
808-
# for the agent runtime. Toolbox samples that call AzureChatOpenAI
809-
# directly need that SP to have 'Cognitive Services OpenAI User' on
810-
# the Foundry account hosting the model deployment. Discover the
811-
# current identity from `azd ai agent show` and grant the role
812-
# before invoking.
813-
- name: Grant model access to agent identity
814-
if: matrix.is_toolbox == 'true'
815-
working-directory: /tmp/ci-${{ matrix.combo_id }}
816-
env:
817-
TOOLBOX_PROJECT_ID: ${{ matrix.use_westus2 == 'true' && vars.TOOLBOX_PROJECT_ID_WESTUS2 || vars.TOOLBOX_PROJECT_ID }}
818-
AZURE_SUBSCRIPTION_ID: ${{ matrix.use_westus2 == 'true' && vars.TOOLBOX_SUBSCRIPTION_ID_WESTUS2 || vars.AZURE_SUBSCRIPTION_ID }}
819-
run: |
820-
set +e
821-
# Derive the Cognitive Services account scope by stripping the
822-
# `/projects/<name>` suffix from TOOLBOX_PROJECT_ID.
823-
AGENT_TARGET_ACCOUNT_SCOPE=$(echo "$TOOLBOX_PROJECT_ID" | sed -E 's|/projects/[^/]+$||')
824-
if [ -z "$AGENT_TARGET_ACCOUNT_SCOPE" ]; then
825-
echo "::warning::Could not derive account scope from TOOLBOX_PROJECT_ID, skipping role grant"
826-
exit 0
827-
fi
828-
echo "Account scope: $AGENT_TARGET_ACCOUNT_SCOPE"
829-
# Switch az CLI context to the subscription that owns the target account
830-
# so role-assignment writes land in the right tenant ABAC scope.
831-
az account set --subscription "$AZURE_SUBSCRIPTION_ID" 2>&1 | head -5 || true
832-
833-
# ── Cleanup orphaned role assignments ──
834-
# Each `azd deploy` creates a new ephemeral AgentIdentityBlueprint SP
835-
# and we grant up to 7 role assignments per SP. Across many parallel
836-
# matrix jobs and re-runs the per-scope role-assignment quota fills
837-
# with assignments whose principal SP has since been deleted (these
838-
# surface as `principalType: Unknown` / empty `principalName`).
839-
# Azure surfaces the quota exhaustion as `RoleAssignmentLimitExceeded:
840-
# No more role assignments can be created`. Before granting fresh
841-
# roles, prune orphans at both scopes for the roles we manage so we
842-
# stay under the cap.
843-
cleanup_orphans () {
844-
local scope="$1"
845-
[ -z "$scope" ] && return 0
846-
echo "Pruning orphaned role assignments at scope: $scope"
847-
# `--all` includes assignments whose principal no longer exists.
848-
# Filter to the roles we manage so we don't touch unrelated grants.
849-
# Select assignments whose principal is gone (Unknown type or
850-
# empty principalName) AND whose role is one we manage.
851-
local orphan_ids
852-
orphan_ids=$(az role assignment list --scope "$scope" --all -o json 2>/dev/null \
853-
| jq -r '.[] | select((.principalType == "Unknown") or ((.principalName // "") == "")) | select(.roleDefinitionName == "Cognitive Services OpenAI User" or .roleDefinitionName == "Cognitive Services User" or .roleDefinitionName == "Azure AI Developer" or .roleDefinitionName == "Foundry User") | .id')
854-
if [ -z "$orphan_ids" ]; then
855-
echo " no orphans to prune"
856-
return 0
857-
fi
858-
local count=0
859-
while IFS= read -r rid; do
860-
[ -z "$rid" ] && continue
861-
az role assignment delete --ids "$rid" -o none 2>/dev/null && count=$((count+1)) || true
862-
done <<< "$orphan_ids"
863-
echo " pruned $count orphan(s)"
864-
}
865-
cleanup_orphans "$AGENT_TARGET_ACCOUNT_SCOPE"
866-
cleanup_orphans "$TOOLBOX_PROJECT_ID"
867-
868-
show_out=$(azd ai agent show --no-prompt --output json 2>&1)
869-
echo "$show_out" | head -200
870-
ids=$(echo "$show_out" | grep -oE '"principal_id"[[:space:]]*:[[:space:]]*"[0-9a-f-]{36}"' | grep -oE '[0-9a-f-]{36}')
871-
if [ -z "$ids" ]; then
872-
echo "::warning::No agent principal_id found in azd output, skipping role grant"
873-
exit 0
874-
fi
875-
for app_id in $ids; do
876-
sp_id=$(az ad sp list --filter "appId eq '$app_id'" --query "[0].id" -o tsv 2>/dev/null)
877-
[ -z "$sp_id" ] && sp_id="$app_id"
878-
echo "Granting model access to agent SP: $sp_id (appId: $app_id)"
879-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Cognitive Services OpenAI User" --scope "$AGENT_TARGET_ACCOUNT_SCOPE" -o none 2>&1 | grep -v "already exists" || true
880-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Azure AI Developer" --scope "$AGENT_TARGET_ACCOUNT_SCOPE" 2>&1 | grep -v "already exists" || true
881-
# Toolbox MCP authorization is enforced at the project sub-resource.
882-
# Without a project-scoped Azure AI Developer assignment the MCP
883-
# endpoint returns 401 Unauthorized for the agent MI.
884-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Azure AI Developer" --scope "$TOOLBOX_PROJECT_ID" 2>&1 | grep -v "already exists" || true
885-
# Toolbox MCP endpoint requires Microsoft.CognitiveServices/* data
886-
# actions which Azure AI Developer does NOT include. Grant
887-
# Cognitive Services User at the account scope as well.
888-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Cognitive Services User" --scope "$AGENT_TARGET_ACCOUNT_SCOPE" 2>&1 | grep -v "already exists" || true
889-
# Also grant at the project scope — some MCP endpoints validate the
890-
# caller's role at the project sub-resource and don't honor
891-
# inherited assignments from the parent account.
892-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Cognitive Services User" --scope "$TOOLBOX_PROJECT_ID" 2>&1 | grep -v "already exists" || true
893-
# The Foundry toolbox MCP gateway authorizes by the **specific role
894-
# name** assigned at project scope (not just by effective data
895-
# actions). The Azure AI Developer description even calls this out:
896-
# "For Foundry project access, use the Foundry User or Foundry
897-
# Owner roles instead." Grant Foundry User at both account and
898-
# project scope so direct in-container MCP calls pass auth.
899-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Foundry User" --scope "$AGENT_TARGET_ACCOUNT_SCOPE" 2>&1 | grep -v "already exists" || true
900-
az role assignment create --assignee-object-id "$sp_id" --assignee-principal-type ServicePrincipal --role "Foundry User" --scope "$TOOLBOX_PROJECT_ID" 2>&1 | grep -v "already exists" || true
901-
done
902-
# Allow time for AAD role propagation (storage/history calls have been
903-
# observed to 401 for ~2 minutes after the grant). Invoke also retries
904-
# PermissionDenied responses (see invoke step).
905-
sleep 120
906-
907790
# ── Validate: invoke (unified, multi-turn) ──────────────────────
908791
# Both protocols use -p <protocol> with a payload file.
909792
# Payload file (test-payload.txt) lives under
@@ -1272,96 +1155,6 @@ jobs:
12721155
/tmp/voicelive-smoke-test.log
12731156
if-no-files-found: ignore
12741157

1275-
# ── Cleanup deployed agent + its identity SPs ────────────────────
1276-
# Each `azd deploy` mints a fresh AgentIdentityBlueprint SP and grants
1277-
# it up to 7 role assignments at the account scope and up to 3 at the
1278-
# project scope. Without per-job teardown those SPs and assignments
1279-
# accumulate forever and eventually trip the per-scope role-assignment
1280-
# cap (`RoleAssignmentLimitExceeded: No more role assignments can be
1281-
# created`). Run after the invoke step (always, so failures still
1282-
# clean up): capture the agent's SP IDs, DELETE the agent record so
1283-
# Foundry releases its claim on the identity, then delete the SP's
1284-
# role assignments at both managed scopes and finally delete the SP.
1285-
- name: Cleanup deployed agent
1286-
if: always()
1287-
working-directory: /tmp/ci-${{ matrix.combo_id }}
1288-
env:
1289-
# For toolbox samples the agent is deployed against the toolbox
1290-
# project (overridden in env-prep); for plain samples it uses the
1291-
# default project. Compute the right endpoint + scopes here.
1292-
TOOLBOX_PROJECT_ID: ${{ matrix.is_toolbox == 'true' && (matrix.use_westus2 == 'true' && vars.TOOLBOX_PROJECT_ID_WESTUS2 || vars.TOOLBOX_PROJECT_ID) || '' }}
1293-
AZURE_SUBSCRIPTION_ID: ${{ matrix.is_toolbox == 'true' && (matrix.use_westus2 == 'true' && vars.TOOLBOX_SUBSCRIPTION_ID_WESTUS2 || vars.AZURE_SUBSCRIPTION_ID) || vars.AZURE_SUBSCRIPTION_ID }}
1294-
run: |
1295-
set +e
1296-
DEPLOYED_NAME="${AGENT_NAME:-${{ matrix.name }}}"
1297-
if [ -z "$DEPLOYED_NAME" ]; then
1298-
echo "::warning::No agent name resolved — skipping cleanup"
1299-
exit 0
1300-
fi
1301-
echo "Cleaning up agent: $DEPLOYED_NAME"
1302-
1303-
# Resolve project endpoint + scopes.
1304-
if [ -n "$TOOLBOX_PROJECT_ID" ]; then
1305-
PROJECT_ID="$TOOLBOX_PROJECT_ID"
1306-
else
1307-
PROJECT_ID="${{ vars.AZURE_AI_PROJECT_ID }}"
1308-
fi
1309-
ACCOUNT_SCOPE=$(echo "$PROJECT_ID" | sed -E 's|/projects/[^/]+$||')
1310-
# Derive the project endpoint (https://<account>.services.ai.azure.com/api/projects/<name>)
1311-
# from the resource ID for the agent DELETE call.
1312-
ACCOUNT_NAME=$(echo "$PROJECT_ID" | sed -nE 's|.*/accounts/([^/]+)/.*|\1|p')
1313-
PROJECT_NAME=$(echo "$PROJECT_ID" | sed -nE 's|.*/projects/([^/]+)$|\1|p')
1314-
if [ -n "$ACCOUNT_NAME" ] && [ -n "$PROJECT_NAME" ]; then
1315-
PROJECT_ENDPOINT="https://${ACCOUNT_NAME}.services.ai.azure.com/api/projects/${PROJECT_NAME}"
1316-
else
1317-
PROJECT_ENDPOINT="${{ vars.AZURE_AI_PROJECT_ENDPOINT }}"
1318-
fi
1319-
echo "Project endpoint: $PROJECT_ENDPOINT"
1320-
echo "Account scope: $ACCOUNT_SCOPE"
1321-
echo "Project scope: $PROJECT_ID"
1322-
1323-
az account set --subscription "$AZURE_SUBSCRIPTION_ID" 2>&1 | head -5 || true
1324-
1325-
# Capture agent SPs BEFORE deleting the agent —
1326-
# `azd ai agent show` won't return them after delete.
1327-
show_out=$(azd ai agent show --no-prompt --output json 2>&1)
1328-
app_ids=$(echo "$show_out" | grep -oE '"principal_id"[[:space:]]*:[[:space:]]*"[0-9a-f-]{36}"' | grep -oE '[0-9a-f-]{36}' | sort -u)
1329-
echo "Agent identity appIds to clean up: ${app_ids:-<none>}"
1330-
1331-
# Delete the agent record so Foundry releases its claim on the SP.
1332-
API_VERSION="2025-05-15-preview"
1333-
TOKEN=$(az account get-access-token --resource https://ai.azure.com --query accessToken -o tsv 2>/dev/null)
1334-
if [ -n "$TOKEN" ] && [ -n "$PROJECT_ENDPOINT" ]; then
1335-
HTTP=$(curl -s -o /tmp/agent-delete.txt -w "%{http_code}" \
1336-
-X DELETE \
1337-
"${PROJECT_ENDPOINT}/agents/${DEPLOYED_NAME}?api-version=${API_VERSION}" \
1338-
-H "Authorization: Bearer $TOKEN")
1339-
echo "Agent DELETE: HTTP $HTTP"
1340-
if [ "$HTTP" != "204" ] && [ "$HTTP" != "200" ] && [ "$HTTP" != "404" ]; then
1341-
echo "::warning::Agent cleanup returned HTTP $HTTP"
1342-
cat /tmp/agent-delete.txt 2>/dev/null | head -50
1343-
fi
1344-
fi
1345-
1346-
# For each agent SP: delete its role assignments at our managed
1347-
# scopes, then delete the SP itself. Errors are tolerated.
1348-
for app_id in $app_ids; do
1349-
sp_id=$(az ad sp list --filter "appId eq '$app_id'" --query "[0].id" -o tsv 2>/dev/null)
1350-
[ -z "$sp_id" ] && sp_id="$app_id"
1351-
echo "Cleaning role assignments for SP appId=$app_id (objectId=$sp_id)"
1352-
for scope in "$ACCOUNT_SCOPE" "$PROJECT_ID"; do
1353-
[ -z "$scope" ] && continue
1354-
ra_ids=$(az role assignment list --assignee "$sp_id" --scope "$scope" --query "[].id" -o tsv 2>/dev/null)
1355-
ra_count=0
1356-
for ra in $ra_ids; do
1357-
az role assignment delete --ids "$ra" -o none 2>/dev/null && ra_count=$((ra_count+1)) || true
1358-
done
1359-
echo " scope=$scope: deleted $ra_count assignment(s)"
1360-
done
1361-
az ad sp delete --id "$app_id" 2>/dev/null && echo " SP $app_id deleted" || echo " SP $app_id delete failed (may need Graph permission)"
1362-
done
1363-
timeout-minutes: 5
1364-
13651158
# ── Teardown (only when provision created resources) ─────────────
13661159
- name: Teardown Azure resources
13671160
if: always() && vars.SKIP_PROVISION != 'true'

0 commit comments

Comments
 (0)