|
| 1 | +# Reusable workflow: Deploy supported Fabric items via the Bulk Import Item Definitions API (Preview). |
| 2 | +# |
| 3 | +# Alternative deployment path to reusable-deploy-supported.yml. Uses the Fabric |
| 4 | +# REST API's bulk import endpoint instead of the fabric-cicd Python library. |
| 5 | +# Selected at orchestrator level via the DEPLOY_METHOD repository variable. |
| 6 | +# |
| 7 | +# Called by: deploy-test-bulk.yml, deploy-prod-bulk.yml |
| 8 | +# |
| 9 | +# Prerequisites: |
| 10 | +# - GitHub Environment secrets: AZURE_TENANT_ID, AZURE_CLIENT_ID, |
| 11 | +# AZURE_CLIENT_SECRET, FABRIC_WORKSPACE_ID |
| 12 | +# - Service principal must have Contributor (or higher) role on the target workspace |
| 13 | +# - Fabric Admin must enable "Service principals can use Fabric APIs" |
| 14 | +# - Every item type in the request payload must support service principals |
| 15 | +# (the bulk API requires SPN support for ALL items in the request, not just some) |
| 16 | +# |
| 17 | +# Known gaps vs. reusable-deploy-supported.yml (fabric-cicd): |
| 18 | +# - No parameter.yml find_replace / key_value_replace substitution |
| 19 | +# - No orphan cleanup (Bulk Import API only supports Create/Update, not Delete) |
| 20 | +# - No item_type_in_scope filter (deploys everything in repository_directory) |
| 21 | +# |
| 22 | +# API references: |
| 23 | +# - Bulk import: https://learn.microsoft.com/en-us/rest/api/fabric/core/items/bulk-import-item-definitions(beta) |
| 24 | +# - Long running ops: https://learn.microsoft.com/en-us/rest/api/fabric/articles/long-running-operation |
| 25 | +# |
| 26 | +# TODO: When the Bulk Import API graduates from Preview, drop the ?beta=true |
| 27 | +# query parameter and re-verify the endpoint URL. |
| 28 | + |
| 29 | +name: "Reusable: Deploy via Bulk Import API" |
| 30 | + |
| 31 | +on: |
| 32 | + workflow_call: |
| 33 | + inputs: |
| 34 | + environment: |
| 35 | + description: "Target environment (Test, Prod)" |
| 36 | + required: true |
| 37 | + type: string |
| 38 | + repository_directory: |
| 39 | + description: "Path to the Fabric item definitions" |
| 40 | + required: false |
| 41 | + type: string |
| 42 | + default: "data/fabric" |
| 43 | + |
| 44 | +permissions: |
| 45 | + contents: read |
| 46 | + |
| 47 | +jobs: |
| 48 | + deploy: |
| 49 | + name: Deploy via Bulk Import API (${{ inputs.environment }}) |
| 50 | + runs-on: ubuntu-latest |
| 51 | + timeout-minutes: 30 |
| 52 | + environment: ${{ inputs.environment }} |
| 53 | + steps: |
| 54 | + - name: Checkout repository |
| 55 | + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 |
| 56 | + |
| 57 | + - name: Set up Python |
| 58 | + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 |
| 59 | + with: |
| 60 | + python-version: "3.12" |
| 61 | + |
| 62 | + - name: Install dependencies |
| 63 | + run: pip install requests |
| 64 | + |
| 65 | + - name: Bulk import item definitions |
| 66 | + env: |
| 67 | + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} |
| 68 | + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} |
| 69 | + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} |
| 70 | + FABRIC_WORKSPACE_ID: ${{ secrets.FABRIC_WORKSPACE_ID }} |
| 71 | + REPOSITORY_DIRECTORY: ${{ inputs.repository_directory }} |
| 72 | + run: | |
| 73 | + python -c " |
| 74 | + import base64 |
| 75 | + import json |
| 76 | + import os |
| 77 | + import pathlib |
| 78 | + import sys |
| 79 | + import time |
| 80 | + import requests |
| 81 | +
|
| 82 | + # Polling configuration (decisions #15, #16) |
| 83 | + POLL_FALLBACK_SECONDS = 30 |
| 84 | + POLL_FLOOR_SECONDS = 5 |
| 85 | + POLL_TIMEOUT_SECONDS = 20 * 60 |
| 86 | + TOKEN_REFRESH_EVERY_N_POLLS = 20 |
| 87 | +
|
| 88 | + # Files to skip when building definitionParts[]. Two layers of exclusion: |
| 89 | + # 1. Named files: known files that should never be sent (parameter.yml is |
| 90 | + # fabric-cicd config; .gitkeep is a Git placeholder). |
| 91 | + # 2. Structural rule: item definitions always live inside *.<Type>/ folders, |
| 92 | + # so any file directly under repository_directory is excluded by |
| 93 | + # construction (handled in build_definition_parts). |
| 94 | + EXCLUDED_FILES = {'parameter.yml', '.gitkeep'} |
| 95 | +
|
| 96 | + tenant_id = os.environ['AZURE_TENANT_ID'] |
| 97 | + client_id = os.environ['AZURE_CLIENT_ID'] |
| 98 | + client_secret = os.environ['AZURE_CLIENT_SECRET'] |
| 99 | + workspace_id = os.environ['FABRIC_WORKSPACE_ID'] |
| 100 | + repo_dir = pathlib.Path(os.environ['REPOSITORY_DIRECTORY']).resolve() |
| 101 | +
|
| 102 | +
|
| 103 | + def acquire_token() -> str: |
| 104 | + token_url = f'https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token' |
| 105 | + resp = requests.post( |
| 106 | + token_url, |
| 107 | + data={ |
| 108 | + 'grant_type': 'client_credentials', |
| 109 | + 'client_id': client_id, |
| 110 | + 'client_secret': client_secret, |
| 111 | + 'scope': 'https://api.fabric.microsoft.com/.default', |
| 112 | + }, |
| 113 | + timeout=30, |
| 114 | + ) |
| 115 | + if resp.status_code != 200: |
| 116 | + sys.exit(f'::error::Token acquisition failed: HTTP {resp.status_code} {resp.text}') |
| 117 | + token = resp.json()['access_token'] |
| 118 | + # Mask the token in workflow logs (decision #10) |
| 119 | + print(f'::add-mask::{token}') |
| 120 | + return token |
| 121 | +
|
| 122 | +
|
| 123 | + def build_definition_parts() -> list: |
| 124 | + if not repo_dir.is_dir(): |
| 125 | + sys.exit(f'::error::Repository directory not found: {repo_dir}') |
| 126 | + parts = [] |
| 127 | + for f in sorted(repo_dir.rglob('*')): |
| 128 | + if not f.is_file(): |
| 129 | + continue |
| 130 | + if f.name in EXCLUDED_FILES: |
| 131 | + continue |
| 132 | + # Item definitions live inside *.<Type>/ subfolders; anything at |
| 133 | + # the root of repository_directory cannot belong to an item. |
| 134 | + if f.parent == repo_dir: |
| 135 | + continue |
| 136 | + rel = '/' + f.relative_to(repo_dir).as_posix() |
| 137 | + parts.append({ |
| 138 | + 'path': rel, |
| 139 | + 'payload': base64.b64encode(f.read_bytes()).decode('ascii'), |
| 140 | + 'payloadType': 'InlineBase64', |
| 141 | + }) |
| 142 | + if not parts: |
| 143 | + sys.exit(f'::error::No item definition files found under {repo_dir}') |
| 144 | + return parts |
| 145 | +
|
| 146 | +
|
| 147 | + def poll_lro(operation_id: str, headers: dict, initial_retry_after: int) -> None: |
| 148 | + base = 'https://api.fabric.microsoft.com/v1/operations' |
| 149 | + retry_after = max(initial_retry_after or POLL_FALLBACK_SECONDS, POLL_FLOOR_SECONDS) |
| 150 | + started = time.monotonic() |
| 151 | + poll_count = 0 |
| 152 | +
|
| 153 | + while True: |
| 154 | + elapsed = time.monotonic() - started |
| 155 | + if elapsed > POLL_TIMEOUT_SECONDS: |
| 156 | + sys.exit( |
| 157 | + f'::error::LRO polling timed out after {POLL_TIMEOUT_SECONDS}s ' |
| 158 | + f'(operation {operation_id})' |
| 159 | + ) |
| 160 | +
|
| 161 | + time.sleep(retry_after) |
| 162 | + poll_count += 1 |
| 163 | +
|
| 164 | + # Refresh token periodically for long-running operations |
| 165 | + # (mirrors the pattern in reusable-fabric-etl.yml). |
| 166 | + if poll_count > 0 and poll_count % TOKEN_REFRESH_EVERY_N_POLLS == 0: |
| 167 | + headers['Authorization'] = f'Bearer {acquire_token()}' |
| 168 | +
|
| 169 | + resp = requests.get(f'{base}/{operation_id}', headers=headers, timeout=30) |
| 170 | + if resp.status_code != 200: |
| 171 | + sys.exit(f'::error::Poll request failed: HTTP {resp.status_code} {resp.text}') |
| 172 | +
|
| 173 | + body = resp.json() |
| 174 | + status = body.get('status', 'Unknown') |
| 175 | + print(f'Poll {poll_count} (t+{int(elapsed)}s): status={status}') |
| 176 | +
|
| 177 | + if status == 'Succeeded': |
| 178 | + return |
| 179 | + if status in ('Failed', 'Undefined'): |
| 180 | + print(json.dumps(body, indent=2)) |
| 181 | + sys.exit(f'::error::LRO ended with status: {status}') |
| 182 | +
|
| 183 | + # NotStarted or Running — keep polling. Honor Retry-After if present. |
| 184 | + retry_after = max( |
| 185 | + int(resp.headers.get('Retry-After', POLL_FALLBACK_SECONDS)), |
| 186 | + POLL_FLOOR_SECONDS, |
| 187 | + ) |
| 188 | +
|
| 189 | +
|
| 190 | + def check_per_item_status(result: dict) -> None: |
| 191 | + details = result.get('importItemDefinitionsDetails', []) |
| 192 | + print(json.dumps(result, indent=2)) |
| 193 | + if not details: |
| 194 | + sys.exit('::error::Result body has no importItemDefinitionsDetails') |
| 195 | +
|
| 196 | + failures = [ |
| 197 | + d for d in details |
| 198 | + if d.get('operationStatus') in ('Failed', 'SucceededDespiteFailures') |
| 199 | + ] |
| 200 | + if failures: |
| 201 | + summary = '\n'.join( |
| 202 | + f\" - {d.get('itemDisplayName')} ({d.get('itemType')}): \" |
| 203 | + f\"{d.get('operationStatus')}\" |
| 204 | + for d in failures |
| 205 | + ) |
| 206 | + sys.exit(f'::error::{len(failures)} item(s) failed:\n{summary}') |
| 207 | +
|
| 208 | + print(f'All {len(details)} items deployed successfully.') |
| 209 | +
|
| 210 | +
|
| 211 | + # ---------- main flow ---------- |
| 212 | + token = acquire_token() |
| 213 | + headers = {'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} |
| 214 | +
|
| 215 | + parts = build_definition_parts() |
| 216 | + print(f'Built request body with {len(parts)} definition parts from {repo_dir}') |
| 217 | +
|
| 218 | + request_body = { |
| 219 | + 'definitionParts': parts, |
| 220 | + 'options': {'allowPairingByName': False}, |
| 221 | + } |
| 222 | +
|
| 223 | + # Endpoint URL per the API reference page (the tutorial's URL is wrong). |
| 224 | + # https://learn.microsoft.com/en-us/rest/api/fabric/core/items/bulk-import-item-definitions(beta) |
| 225 | + api_url = ( |
| 226 | + f'https://api.fabric.microsoft.com/v1/workspaces/{workspace_id}' |
| 227 | + f'/items/bulkImportDefinitions?beta=true' |
| 228 | + ) |
| 229 | + print(f'POST {api_url}') |
| 230 | +
|
| 231 | + post_resp = requests.post(api_url, headers=headers, json=request_body, timeout=120) |
| 232 | +
|
| 233 | + if post_resp.status_code == 200: |
| 234 | + # Sync path — result is in the response body directly. |
| 235 | + check_per_item_status(post_resp.json()) |
| 236 | + sys.exit(0) |
| 237 | +
|
| 238 | + if post_resp.status_code == 202: |
| 239 | + # Async path — poll the LRO, then fetch the result. |
| 240 | + operation_id = post_resp.headers.get('x-ms-operation-id') |
| 241 | + if not operation_id: |
| 242 | + sys.exit('::error::202 response missing x-ms-operation-id header') |
| 243 | +
|
| 244 | + initial_retry = int(post_resp.headers.get('Retry-After', POLL_FALLBACK_SECONDS)) |
| 245 | + print(f'202 Accepted, operation_id={operation_id}, initial Retry-After={initial_retry}s') |
| 246 | +
|
| 247 | + poll_lro(operation_id, headers, initial_retry) |
| 248 | +
|
| 249 | + result_resp = requests.get( |
| 250 | + f'https://api.fabric.microsoft.com/v1/operations/{operation_id}/result', |
| 251 | + headers=headers, |
| 252 | + timeout=30, |
| 253 | + ) |
| 254 | + if result_resp.status_code != 200: |
| 255 | + sys.exit( |
| 256 | + f'::error::Failed to fetch operation result: ' |
| 257 | + f'HTTP {result_resp.status_code} {result_resp.text}' |
| 258 | + ) |
| 259 | + check_per_item_status(result_resp.json()) |
| 260 | + sys.exit(0) |
| 261 | +
|
| 262 | + sys.exit( |
| 263 | + f'::error::Bulk import POST failed: HTTP {post_resp.status_code} {post_resp.text}' |
| 264 | + ) |
| 265 | + " |
0 commit comments