Release Verification and Retry

eng/pipelines/templates/jobs/ci.yml

@@ -271,6 +271,21 @@ jobs:
           ArtifactPath: $(Build.ArtifactStagingDirectory)
           ArtifactName: 'packages'
 
+      - task: PowerShell@2
+        displayName: 'Stage Maven Settings for publishing'
+        inputs:
+          pwsh: true
+          targetType: inline
+          script: |
+            $settingsArtifactDirectory = Join-Path '$(Agent.TempDirectory)' 'maven-settings'
+            New-Item -Path $settingsArtifactDirectory -ItemType Directory -Force | Out-Null
+            Copy-Item -Path '$(Build.SourcesDirectory)/eng/settings.xml' -Destination (Join-Path $settingsArtifactDirectory 'settings.xml')
+
+      - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
+        parameters:
+          ArtifactPath: $(Agent.TempDirectory)/maven-settings
+          ArtifactName: 'maven-settings'
+
       # Troubleshooting artifacts are creating in the staging directory under the folder 'troubleshooting'.
       # This will contain things such as heap dumps hprofs if testing hit OutOfMemory errors, log files captured
       # during testing if tests failed, and linting reports.

eng/pipelines/templates/stages/archetype-java-release-batch.yml

@@ -20,6 +20,9 @@ parameters:
   - name: PublicFeedUrl
     type: string
     default: 'maven.org'
+  - name: VerifyJarPublish
+    type: boolean
+    default: true
 
 stages:
   # The signing stage is responsible for submitting binaries to ESRP for our official signing
@@ -226,6 +229,9 @@ stages:
               - input: pipelineArtifact
                 artifactName: 'packages-esrp-flattened'
                 targetPath: '$(Pipeline.Workspace)/packages-esrp-flattened'
+              - input: pipelineArtifact
+                artifactName: 'maven-settings'
+                targetPath: '$(Pipeline.Workspace)/maven-settings'
             pool:
               name: azsdk-pool
               image: windows-2022
@@ -234,9 +240,15 @@ stages:
               runOnce:
                 deploy:
                   steps:
+                    - pwsh: |
+                        $m2Dir = if ($env:USERPROFILE) { "$env:USERPROFILE\.m2" } else { "$HOME/.m2" }
+                        New-Item -ItemType Directory -Force -Path $m2Dir | Out-Null
+                        Copy-Item -Path '$(Pipeline.Workspace)/maven-settings/settings.xml' -Destination "$m2Dir/settings.xml"
+                      displayName: 'Setup Maven mirror settings'
                     - template: /eng/pipelines/templates/steps/java-esrp-publishing.yml
                       parameters:
                         FlattenedDirectory: $(Pipeline.Workspace)/packages-esrp-flattened
+                        VerifyJarPublish: true
 
         - job: PublishDevFeedPackage
           displayName: "Publish to Java Dev feed"

eng/pipelines/templates/steps/java-esrp-publishing.yml

@@ -2,9 +2,37 @@ parameters:
   # This is the flattened
   FlattenedDirectory: not-specified
   ShouldPublish: true
+  VerifyJarPublish: false
 
 steps:
   - ${{if eq(parameters.ShouldPublish, 'true')}}:
+    # TODO: REMOVE BEFORE MERGE
+    - task: PowerShell@2
+      displayName: 'Move one package out of flattened ESRP directory (test)'
+      condition: and(succeeded(), eq(variables['System.JobAttempt'], '1'))
+      inputs:
+        pwsh: true
+        targetType: inline
+        script: |
+          $flattenedDirectory = (Resolve-Path '${{ parameters.FlattenedDirectory }}').Path
+          $stagedDirectory = Join-Path '$(Agent.TempDirectory)' 'java-esrp-publishing-test-package'
+          $packagePom = Get-ChildItem -Path $flattenedDirectory -Filter *.pom -File | Sort-Object Name | Select-Object -First 1
+          $packagePrefix = $packagePom.BaseName
+          $packageFiles = Get-ChildItem -Path $flattenedDirectory -File | Where-Object { $_.Name -like "$packagePrefix*" }
+
+          if (Test-Path $stagedDirectory) {
+            Remove-Item -Path $stagedDirectory -Recurse -Force
+          }
+
+          New-Item -Path $stagedDirectory -ItemType Directory -Force | Out-Null
+
+          Write-Host "Moving package prefix out of flattened directory: $packagePrefix"
+          foreach ($packageFile in $packageFiles) {
+            Write-Host "  Moving $($packageFile.Name)"
+          }
+
+          Move-Item -Path $packageFiles.FullName -Destination $stagedDirectory -Force
+
     - task: EsrpRelease@9
       displayName: 'Publish to ESRP'
       inputs:
@@ -21,3 +49,155 @@ steps:
         Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }}
         ServiceEndpointUrl: 'https://api.esrp.microsoft.com'
         MainPublisher: 'ESRPRELPACMANTEST'
+
+    # TODO: REMOVE BEFORE MERGE
+    - task: PowerShell@2
+      displayName: 'Move one package back into flattened ESRP directory (test)'
+      condition: and(always(), eq(variables['System.JobAttempt'], '1'))
+      inputs:
+        pwsh: true
+        targetType: inline
+        script: |
+          $flattenedDirectory = (Resolve-Path '${{ parameters.FlattenedDirectory }}').Path
+          $stagedDirectory = Join-Path '$(Agent.TempDirectory)' 'java-esrp-publishing-test-package'
+
+          if (Test-Path $stagedDirectory) {
+            $packageFiles = Get-ChildItem -Path $stagedDirectory -File | Sort-Object Name
+
+            Write-Host "Moving package back into flattened directory"
+            foreach ($packageFile in $packageFiles) {
+              Write-Host "  Restoring $($packageFile.Name)"
+            }
+
+            Move-Item -Path $packageFiles.FullName -Destination $flattenedDirectory -Force
+            Remove-Item -Path $stagedDirectory -Recurse -Force
+          }
+
+    - ${{ if eq('true', parameters.VerifyJarPublish) }}:
+      - task: MavenAuthenticate@0
+        displayName: 'Maven Authenticate'
+        inputs:
+          artifactsFeeds: 'azure-sdk-for-java'
+
+      - task: PowerShell@2
+        displayName: 'Verify ESRP published package jars are downloadable'
+        inputs:
+          pwsh: true
+          targetType: inline
+          workingDirectory: $(Agent.BuildDirectory)
+          script: |
+            $ErrorActionPreference = 'Stop'
+            $PSNativeCommandUseErrorActionPreference = $false
+
+            $flattenedDirectory = (Resolve-Path '${{ parameters.FlattenedDirectory }}').Path
+            $workRoot = Join-Path (Get-Location) 'work'
+            $mavenWork = Join-Path $workRoot 'maven-work'
+            $metadataRepo = Join-Path $workRoot 'maven-metadata-repo'
+            $downloadRoot = Join-Path $workRoot 'jar-downloads'
+            $retrySeconds = 60
+            $maxAttempts = 20
+
+            function Get-MavenValue([string]$PomPath, [string]$Expression) {
+              Set-Location $mavenWork
+              $mvnArguments = @(
+                '--batch-mode',
+                '-q',
+                '--file',
+                $PomPath,
+                'help:evaluate',
+                "-Dmaven.repo.local=$metadataRepo",
+                "-Dexpression=$Expression",
+                '-DforceStdout'
+              )
+
+              [string[]]$output = & mvn @mvnArguments
+              if ($LASTEXITCODE -ne 0) {
+                throw "Failed to read $Expression from $PomPath."
+              }
+
+              return ($output | Where-Object { $_ -and $_.Trim() } | Select-Object -Last 1).Trim()
+            }
+
+            New-Item -Path $workRoot -ItemType Directory -Force | Out-Null
+            New-Item -Path $mavenWork -ItemType Directory -Force | Out-Null
+            if (Test-Path $metadataRepo) {
+              Remove-Item -Path $metadataRepo -Recurse -Force
+            }
+            New-Item -Path $metadataRepo -ItemType Directory -Force | Out-Null
+            if (Test-Path $downloadRoot) {
+              Remove-Item -Path $downloadRoot -Recurse -Force
+            }
+            New-Item -Path $downloadRoot -ItemType Directory -Force | Out-Null
+
+            $packages = @()
+            $pomFiles = Get-ChildItem -Path $flattenedDirectory -Filter *.pom -File | Sort-Object Name
+
+            foreach ($pomFile in $pomFiles) {
+              $groupId = Get-MavenValue $pomFile.FullName 'project.groupId'
+              $artifactId = Get-MavenValue $pomFile.FullName 'project.artifactId'
+              $version = Get-MavenValue $pomFile.FullName 'project.version'
+              $expectedFile = "$artifactId-$version.jar"
+
+              if (-not (Test-Path (Join-Path $flattenedDirectory $expectedFile))) {
+                Write-Host "  Skipping $artifactId-$version (JAR file not found)"
+                continue
+              }
+
+              $packages += [PSCustomObject]@{
+                Coordinate = "${groupId}:${artifactId}:${version}:jar"
+                ExpectedFile = $expectedFile
+              }
+            }
+
+            Write-Host "Found $($packages.Count) package jar(s) to verify."
+            foreach ($package in $packages) {
+              Write-Host "  $($package.Coordinate)"
+            }
+
+            $pending = @($packages)
+
+            for ($attempt = 1; $attempt -le $maxAttempts -and $pending.Count -gt 0; $attempt++) {
+              $failed = @()
+              $repoPath = Join-Path $workRoot "maven-repo-$attempt"
+              New-Item -Path $repoPath -ItemType Directory -Force | Out-Null
+
+              Write-Host "Attempt: $attempt"
+
+              foreach ($package in $pending) {
+                Write-Host "  Downloading $($package.Coordinate)..."
+                Set-Location $mavenWork
+                & mvn `
+                  --batch-mode `
+                  -q `
+                  org.apache.maven.plugins:maven-dependency-plugin:copy `
+                  "-Dmaven.repo.local=$repoPath" `
+                  "-Dartifact=$($package.Coordinate)" `
+                  "-DoutputDirectory=$downloadRoot" `
+                  -Dtransitive=false
+
+                $expectedPath = Join-Path $downloadRoot $package.ExpectedFile
+                if ($LASTEXITCODE -eq 0 -and (Test-Path $expectedPath)) {
+                  Write-Host "    Verified $($package.ExpectedFile)"
+                }
+                else {
+                  Write-Host "    Failed $($package.ExpectedFile)"
+                  $failed += $package
+                }
+              }
+
+              $pending = @($failed)
+              if ($pending.Count -gt 0 -and $attempt -lt $maxAttempts) {
+                Write-Host "Retrying $($pending.Count) package(s) in $retrySeconds seconds."
+                Start-Sleep -Seconds $retrySeconds
+              }
+            }
+
+            if ($pending.Count -gt 0) {
+              Write-Host "These package jars still failed:"
+              foreach ($package in $pending) {
+                Write-Host "  $($package.Coordinate)"
+              }
+              exit 1
+            }
+
+            Write-Host 'All package jars were verified published.'