Add daily extension builds to storage

rajeshkamal5050 · Copilot · rajeshkamal5050 · commit a2db89a58100 · 2026-04-03T14:48:49.000-07:00
Mirrors the core CLI daily build pattern. On CI builds (push to main), sign and upload extension binaries to Azure Storage. - publish-extension-daily.yml — new stage, depends on Sign, uploads to {ext-id}/daily (latest) and {ext-id}/daily/archive/{version} (history) - publish-extension.yml — add CreateGitHubRelease param (default true) so daily can skip GitHub Release and just upload to storage - release-azd-extension.yml — wire daily publish for CI builds - update-daily-registry.yml — download existing registry-daily.json from storage, merge in the current extension entry with checksums and storage URLs, upload back. Self-maintaining across extension builds. Closes #7317 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
diff --git a/eng/pipelines/templates/stages/publish-extension-daily.yml b/eng/pipelines/templates/stages/publish-extension-daily.yml
@@ -0,0 +1,60 @@
+parameters:
+  - name: SanitizedExtensionId
+    type: string
+  - name: AzdExtensionId
+    type: string
+
+stages:
+  - stage: PublishDaily
+    dependsOn: Sign
+    condition: >-
+      and(
+        succeeded(),
+        ne(variables['Skip.Release'], 'true'),
+        or(
+          in(variables['BuildReasonOverride'], 'IndividualCI', 'BatchedCI'),
+          and(
+            eq('', variables['BuildReasonOverride']),
+            in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')
+          )
+        )
+      )
+
+    variables:
+      - template: /eng/pipelines/templates/variables/image.yml
+      - template: /eng/pipelines/templates/variables/globals.yml
+
+    jobs:
+      - deployment: Publish_Daily
+        environment: none
+
+        pool:
+          name: azsdk-pool
+          image: ubuntu-22.04
+          os: linux
+
+        templateContext:
+          type: releaseJob
+          isProduction: false
+          inputs:
+            - input: pipelineArtifact
+              artifactName: release
+              targetPath: release
+
+        strategy:
+          runOnce:
+            deploy:
+              steps:
+                - template: /eng/pipelines/templates/steps/extension-set-metadata-variables.yml
+                  parameters:
+                    Use1ESArtifactTask: true
+
+                - template: /eng/pipelines/templates/steps/publish-extension.yml
+                  parameters:
+                    PublishUploadLocations: ${{ parameters.SanitizedExtensionId }}/daily;${{ parameters.SanitizedExtensionId }}/daily/archive/$(EXT_VERSION)
+                    CreateGitHubRelease: false
+
+                - template: /eng/pipelines/templates/steps/update-daily-registry.yml
+                  parameters:
+                    SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }}
+                    AzdExtensionId: ${{ parameters.AzdExtensionId }}
diff --git a/eng/pipelines/templates/stages/release-azd-extension.yml b/eng/pipelines/templates/stages/release-azd-extension.yml
@@ -84,3 +84,14 @@ stages:
     - template: /eng/pipelines/templates/stages/publish-extension.yml
       parameters:
         SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }}
+
+  # Publish daily builds to storage on CI (push to main)
+  - ${{ if and(eq(variables['System.TeamProject'], 'internal'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
+    - template: /eng/pipelines/templates/stages/sign-extension.yml
+      parameters:
+        SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }}
+
+    - template: /eng/pipelines/templates/stages/publish-extension-daily.yml
+      parameters:
+        SanitizedExtensionId: ${{ parameters.SanitizedExtensionId }}
+        AzdExtensionId: ${{ parameters.AzdExtensionId }}
diff --git a/eng/pipelines/templates/steps/publish-extension.yml b/eng/pipelines/templates/steps/publish-extension.yml
@@ -6,63 +6,78 @@ parameters:
     default: '`$web'
   - name: TagPrefix
     type: string
+    default: ''
   - name: TagVersion
     type: string
+    default: ''
+  - name: CreateGitHubRelease
+    type: boolean
+    default: true
 
 steps:
-  # This step must run first because a duplicated tag means we don't need to
-  # continue with any of the subsequent steps.
-  - pwsh: |
-      $tag = "${{ parameters.TagPrefix }}_${{ parameters.TagVersion}}"
-      Write-Host "Release tag: $tag"
+  - ${{ if eq(parameters.CreateGitHubRelease, true) }}:
+    # This step must run first because a duplicated tag means we don't need to
+    # continue with any of the subsequent steps.
+    - pwsh: |
+        $tag = "${{ parameters.TagPrefix }}_${{ parameters.TagVersion}}"
+        Write-Host "Release tag: $tag"
 
-      # Check for tag using gh API
-      $existingTag = gh api /repos/$(Build.Repository.Name)/tags | ConvertFrom-Json | Where-Object { $_.name -eq $tag }
-      if ($existingTag) {
-          Write-Host "Tag $tag already exists. Exiting."
+        # Check for tag using gh API
+        $existingTag = gh api /repos/$(Build.Repository.Name)/tags | ConvertFrom-Json | Where-Object { $_.name -eq $tag }
+        if ($existingTag) {
+            Write-Host "Tag $tag already exists. Exiting."
+            exit 1
+        }
+
+        gh release view $tag --repo $(Build.Repository.Name)
+        if ($LASTEXITCODE -eq 0) {
+          Write-Host "Release ($tag) already exists. Exiting."
           exit 1
-      }
+        }
 
-      gh release view $tag --repo $(Build.Repository.Name)
-      if ($LASTEXITCODE -eq 0) {
-        Write-Host "Release ($tag) already exists. Exiting."
-        exit 1
-      }
+        Write-Host "##vso[task.setvariable variable=GH_RELEASE_TAG;]$tag"
 
-      Write-Host "##vso[task.setvariable variable=GH_RELEASE_TAG;]$tag"
+        # Exit with 0 (otherwise $LASTEXITCODE will not be 0 and the pipeline
+        # will fail)
+        exit 0
+      displayName: Check for existing GitHub release
+      env:
+        GH_TOKEN: $(azuresdk-github-pat)
 
-      # Exit with 0 (otherwise $LASTEXITCODE will not be 0 and the pipeline
-      # will fail)
-      exit 0
-    displayName: Check for existing GitHub release
-    env:
-      GH_TOKEN: $(azuresdk-github-pat)
+    - pwsh: |
+        Remove-Item -Path release/_manifest -Recurse -Force
+        Write-Host "Release:"
+        Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName 
+      displayName: Remove _manifest folder
 
-  - pwsh: |
-      Remove-Item -Path release/_manifest -Recurse -Force
-      Write-Host "Release:"
-      Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName 
-    displayName: Remove _manifest folder
+    - pwsh: |
+        $version = "${{ parameters.TagVersion }}"
+        $createArgs = @(
+          "$(GH_RELEASE_TAG)",
+          "--title", "$(GH_RELEASE_TAG)",
+          "--notes-file", "changelog/CHANGELOG.md",
+          "--repo", "$(Build.Repository.Name)"
+        )
 
-  - pwsh: |
-      $version = "${{ parameters.TagVersion }}"
-      $createArgs = @(
-        "$(GH_RELEASE_TAG)",
-        "--title", "$(GH_RELEASE_TAG)",
-        "--notes-file", "changelog/CHANGELOG.md",
-        "--repo", "$(Build.Repository.Name)"
-      )
+        if ($version -match "^0\." -or $version -match "-(alpha|beta|preview)") {
+          $createArgs += "--prerelease"
+        }
 
-      if ($version -match "^0\." -or $version -match "-(alpha|beta|preview)") {
-        $createArgs += "--prerelease"
-      }
+        gh release create @createArgs
 
-      gh release create @createArgs
+        gh release upload $(GH_RELEASE_TAG) release/* --repo $(Build.Repository.Name)
+      displayName: Create GitHub Release and upload artifacts
+      env:
+        GH_TOKEN: $(azuresdk-github-pat)
 
-      gh release upload $(GH_RELEASE_TAG) release/* --repo $(Build.Repository.Name)
-    displayName: Create GitHub Release and upload artifacts
-    env:
-      GH_TOKEN: $(azuresdk-github-pat)
+  - ${{ if eq(parameters.CreateGitHubRelease, false) }}:
+    - pwsh: |
+        if (Test-Path release/_manifest) {
+          Remove-Item -Path release/_manifest -Recurse -Force
+        }
+        Write-Host "Release:"
+        Get-ChildItem -Recurse release/ | Select-Object -Property Length,FullName
+      displayName: Remove _manifest folder
 
   - task: AzurePowerShell@5
     displayName: Upload release to storage account
@@ -77,7 +92,7 @@ steps:
         Get-ChildItem release/
         foreach ($folder in $uploadLocations) {
             Write-Host "Upload to ${{ parameters.StorageContainerName }}/azd/extensions/$folder"
-            azcopy copy "release/*" "$(publish-storage-location)/${{ parameters.StorageContainerName }}/azd/extensions/$folder"
+            azcopy copy "release/*" "$(publish-storage-location)/${{ parameters.StorageContainerName }}/azd/extensions/$folder" --overwrite=true
             if ($LASTEXITCODE) {
               Write-Error "Upload failed"
               exit 1
diff --git a/eng/pipelines/templates/steps/update-daily-registry.yml b/eng/pipelines/templates/steps/update-daily-registry.yml
@@ -0,0 +1,129 @@
+parameters:
+  - name: SanitizedExtensionId
+    type: string
+  - name: AzdExtensionId
+    type: string
+
+steps:
+  - task: AzurePowerShell@5
+    displayName: Update daily registry
+    inputs:
+      azureSubscription: 'Azure SDK Artifacts'
+      azurePowerShellVersion: LatestVersion
+      pwsh: true
+      ScriptType: InlineScript
+      Inline: |
+        $extId = "${{ parameters.AzdExtensionId }}"
+        $sanitizedId = "${{ parameters.SanitizedExtensionId }}"
+        $version = "$(EXT_VERSION)"
+        $storageHost = "$(publish-storage-static-host)"
+        $registryBlobPath = "$(publish-storage-location)/`$web/azd/extensions/registry-daily.json"
+        $dailyBaseUrl = "$storageHost/azd/extensions/$sanitizedId/daily"
+
+        # Download existing registry or create empty one
+        $registryFile = "registry-daily.json"
+        azcopy copy $registryBlobPath $registryFile 2>$null
+        if ($LASTEXITCODE -ne 0 -or !(Test-Path $registryFile)) {
+          Write-Host "No existing registry found, creating new one"
+          @{ extensions = @() } | ConvertTo-Json -Depth 10 | Set-Content $registryFile
+        }
+
+        $registry = Get-Content $registryFile -Raw | ConvertFrom-Json -Depth 20
+
+        # Read extension.yaml from the release-metadata artifact
+        $extYaml = Get-Content "release-metadata/extension.yaml" -Raw
+        $extMeta = @{}
+        foreach ($line in $extYaml -split "`n") {
+          if ($line -match "^(\w[\w\-]*):\s*(.+)$") {
+            $extMeta[$matches[1]] = $matches[2].Trim()
+          }
+        }
+
+        # Parse list fields from extension.yaml
+        function Get-YamlList($yaml, $field) {
+          $items = @()
+          $inField = $false
+          foreach ($line in $yaml -split "`n") {
+            if ($line -match "^${field}:") { $inField = $true; continue }
+            if ($inField -and $line -match "^\s+-\s+(.+)$") {
+              $items += $matches[1].Trim()
+            } elseif ($inField -and $line -match "^\S") {
+              break
+            }
+          }
+          return $items
+        }
+
+        $capabilities = Get-YamlList $extYaml "capabilities"
+
+        # Compute checksums from the release artifacts
+        $platforms = @(
+          @{ key = "darwin/amd64";  file = "$sanitizedId-darwin-amd64.zip";  entry = "$sanitizedId-darwin-amd64" },
+          @{ key = "darwin/arm64";  file = "$sanitizedId-darwin-arm64.zip";  entry = "$sanitizedId-darwin-arm64" },
+          @{ key = "linux/amd64";   file = "$sanitizedId-linux-amd64.tar.gz"; entry = "$sanitizedId-linux-amd64" },
+          @{ key = "linux/arm64";   file = "$sanitizedId-linux-arm64.tar.gz"; entry = "$sanitizedId-linux-arm64" },
+          @{ key = "windows/amd64"; file = "$sanitizedId-windows-amd64.zip"; entry = "$sanitizedId-windows-amd64.exe" },
+          @{ key = "windows/arm64"; file = "$sanitizedId-windows-arm64.zip"; entry = "$sanitizedId-windows-arm64.exe" }
+        )
+
+        $artifacts = @{}
+        foreach ($p in $platforms) {
+          $filePath = "release/$($p.file)"
+          if (Test-Path $filePath) {
+            $hash = (Get-FileHash -Path $filePath -Algorithm SHA256).Hash.ToLower()
+            $artifacts[$p.key] = @{
+              checksum   = @{ algorithm = "sha256"; value = $hash }
+              entryPoint = $p.entry
+              url        = "$dailyBaseUrl/$($p.file)"
+            }
+          }
+        }
+
+        # Build the version entry
+        $versionEntry = @{
+          version      = $version
+          capabilities = $capabilities
+          artifacts    = $artifacts
+        }
+
+        if ($extMeta["usage"]) { $versionEntry.usage = $extMeta["usage"] }
+        if ($extMeta["requiredAzdVersion"]) {
+          $versionEntry.requiredAzdVersion = $extMeta["requiredAzdVersion"]
+        }
+
+        # Build the extension entry
+        $extEntry = @{
+          id          = $extId
+          namespace   = $extMeta["namespace"]
+          displayName = $extMeta["displayName"]
+          description = $extMeta["description"]
+          versions    = @($versionEntry)
+        }
+
+        # Merge into registry: replace if extension exists, add if new
+        $found = $false
+        for ($i = 0; $i -lt $registry.extensions.Count; $i++) {
+          if ($registry.extensions[$i].id -eq $extId) {
+            $registry.extensions[$i] = $extEntry
+            $found = $true
+            break
+          }
+        }
+        if (-not $found) {
+          $registry.extensions += $extEntry
+        }
+
+        # Write and upload
+        $registry | ConvertTo-Json -Depth 20 | Set-Content $registryFile -Encoding utf8
+        Write-Host "Updated registry for $extId ($version):"
+        Get-Content $registryFile
+
+        azcopy copy $registryFile $registryBlobPath --overwrite=true
+        if ($LASTEXITCODE) {
+          Write-Error "Failed to upload registry"
+          exit 1
+        }
+
+        Write-Host "Registry uploaded to $registryBlobPath"
+    env:
+      AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED'
