diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index fdc34a2396c0..64eadf93bc35 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -271,6 +271,21 @@ jobs: ArtifactPath: $(Build.ArtifactStagingDirectory) ArtifactName: 'packages' + - task: PowerShell@2 + displayName: 'Stage Maven Settings for publishing' + inputs: + pwsh: true + targetType: inline + script: | + $settingsArtifactDirectory = Join-Path '$(Agent.TempDirectory)' 'maven-settings' + New-Item -Path $settingsArtifactDirectory -ItemType Directory -Force | Out-Null + Copy-Item -Path '$(Build.SourcesDirectory)/eng/settings.xml' -Destination (Join-Path $settingsArtifactDirectory 'settings.xml') + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: $(Agent.TempDirectory)/maven-settings + ArtifactName: 'maven-settings' + # Troubleshooting artifacts are creating in the staging directory under the folder 'troubleshooting'. # This will contain things such as heap dumps hprofs if testing hit OutOfMemory errors, log files captured # during testing if tests failed, and linting reports. diff --git a/eng/pipelines/templates/stages/archetype-java-release-batch.yml b/eng/pipelines/templates/stages/archetype-java-release-batch.yml index 4815f1afb6ac..8b59fb273f39 100644 --- a/eng/pipelines/templates/stages/archetype-java-release-batch.yml +++ b/eng/pipelines/templates/stages/archetype-java-release-batch.yml @@ -20,6 +20,9 @@ parameters: - name: PublicFeedUrl type: string default: 'maven.org' + - name: VerifyJarPublish + type: boolean + default: true stages: # The signing stage is responsible for submitting binaries to ESRP for our official signing @@ -226,6 +229,9 @@ stages: - input: pipelineArtifact artifactName: 'packages-esrp-flattened' targetPath: '$(Pipeline.Workspace)/packages-esrp-flattened' + - input: pipelineArtifact + artifactName: 'maven-settings' + targetPath: '$(Pipeline.Workspace)/maven-settings' pool: name: azsdk-pool image: windows-2022 @@ -234,9 +240,15 @@ stages: runOnce: deploy: steps: + - pwsh: | + $m2Dir = if ($env:USERPROFILE) { "$env:USERPROFILE\.m2" } else { "$HOME/.m2" } + New-Item -ItemType Directory -Force -Path $m2Dir | Out-Null + Copy-Item -Path '$(Pipeline.Workspace)/maven-settings/settings.xml' -Destination "$m2Dir/settings.xml" + displayName: 'Setup Maven mirror settings' - template: /eng/pipelines/templates/steps/java-esrp-publishing.yml parameters: FlattenedDirectory: $(Pipeline.Workspace)/packages-esrp-flattened + VerifyJarPublish: true - job: PublishDevFeedPackage displayName: "Publish to Java Dev feed" diff --git a/eng/pipelines/templates/steps/java-esrp-publishing.yml b/eng/pipelines/templates/steps/java-esrp-publishing.yml index f27f0a896487..659c714b09cc 100644 --- a/eng/pipelines/templates/steps/java-esrp-publishing.yml +++ b/eng/pipelines/templates/steps/java-esrp-publishing.yml @@ -2,9 +2,37 @@ parameters: # This is the flattened FlattenedDirectory: not-specified ShouldPublish: true + VerifyJarPublish: false steps: - ${{if eq(parameters.ShouldPublish, 'true')}}: + # TODO: REMOVE BEFORE MERGE + - task: PowerShell@2 + displayName: 'Move one package out of flattened ESRP directory (test)' + condition: and(succeeded(), eq(variables['System.JobAttempt'], '1')) + inputs: + pwsh: true + targetType: inline + script: | + $flattenedDirectory = (Resolve-Path '${{ parameters.FlattenedDirectory }}').Path + $stagedDirectory = Join-Path '$(Agent.TempDirectory)' 'java-esrp-publishing-test-package' + $packagePom = Get-ChildItem -Path $flattenedDirectory -Filter *.pom -File | Sort-Object Name | Select-Object -First 1 + $packagePrefix = $packagePom.BaseName + $packageFiles = Get-ChildItem -Path $flattenedDirectory -File | Where-Object { $_.Name -like "$packagePrefix*" } + + if (Test-Path $stagedDirectory) { + Remove-Item -Path $stagedDirectory -Recurse -Force + } + + New-Item -Path $stagedDirectory -ItemType Directory -Force | Out-Null + + Write-Host "Moving package prefix out of flattened directory: $packagePrefix" + foreach ($packageFile in $packageFiles) { + Write-Host " Moving $($packageFile.Name)" + } + + Move-Item -Path $packageFiles.FullName -Destination $stagedDirectory -Force + - task: EsrpRelease@9 displayName: 'Publish to ESRP' inputs: @@ -21,3 +49,155 @@ steps: Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} ServiceEndpointUrl: 'https://api.esrp.microsoft.com' MainPublisher: 'ESRPRELPACMANTEST' + + # TODO: REMOVE BEFORE MERGE + - task: PowerShell@2 + displayName: 'Move one package back into flattened ESRP directory (test)' + condition: and(always(), eq(variables['System.JobAttempt'], '1')) + inputs: + pwsh: true + targetType: inline + script: | + $flattenedDirectory = (Resolve-Path '${{ parameters.FlattenedDirectory }}').Path + $stagedDirectory = Join-Path '$(Agent.TempDirectory)' 'java-esrp-publishing-test-package' + + if (Test-Path $stagedDirectory) { + $packageFiles = Get-ChildItem -Path $stagedDirectory -File | Sort-Object Name + + Write-Host "Moving package back into flattened directory" + foreach ($packageFile in $packageFiles) { + Write-Host " Restoring $($packageFile.Name)" + } + + Move-Item -Path $packageFiles.FullName -Destination $flattenedDirectory -Force + Remove-Item -Path $stagedDirectory -Recurse -Force + } + + - ${{ if eq('true', parameters.VerifyJarPublish) }}: + - task: MavenAuthenticate@0 + displayName: 'Maven Authenticate' + inputs: + artifactsFeeds: 'azure-sdk-for-java' + + - task: PowerShell@2 + displayName: 'Verify ESRP published package jars are downloadable' + inputs: + pwsh: true + targetType: inline + workingDirectory: $(Agent.BuildDirectory) + script: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $false + + $flattenedDirectory = (Resolve-Path '${{ parameters.FlattenedDirectory }}').Path + $workRoot = Join-Path (Get-Location) 'work' + $mavenWork = Join-Path $workRoot 'maven-work' + $metadataRepo = Join-Path $workRoot 'maven-metadata-repo' + $downloadRoot = Join-Path $workRoot 'jar-downloads' + $retrySeconds = 60 + $maxAttempts = 20 + + function Get-MavenValue([string]$PomPath, [string]$Expression) { + Set-Location $mavenWork + $mvnArguments = @( + '--batch-mode', + '-q', + '--file', + $PomPath, + 'help:evaluate', + "-Dmaven.repo.local=$metadataRepo", + "-Dexpression=$Expression", + '-DforceStdout' + ) + + [string[]]$output = & mvn @mvnArguments + if ($LASTEXITCODE -ne 0) { + throw "Failed to read $Expression from $PomPath." + } + + return ($output | Where-Object { $_ -and $_.Trim() } | Select-Object -Last 1).Trim() + } + + New-Item -Path $workRoot -ItemType Directory -Force | Out-Null + New-Item -Path $mavenWork -ItemType Directory -Force | Out-Null + if (Test-Path $metadataRepo) { + Remove-Item -Path $metadataRepo -Recurse -Force + } + New-Item -Path $metadataRepo -ItemType Directory -Force | Out-Null + if (Test-Path $downloadRoot) { + Remove-Item -Path $downloadRoot -Recurse -Force + } + New-Item -Path $downloadRoot -ItemType Directory -Force | Out-Null + + $packages = @() + $pomFiles = Get-ChildItem -Path $flattenedDirectory -Filter *.pom -File | Sort-Object Name + + foreach ($pomFile in $pomFiles) { + $groupId = Get-MavenValue $pomFile.FullName 'project.groupId' + $artifactId = Get-MavenValue $pomFile.FullName 'project.artifactId' + $version = Get-MavenValue $pomFile.FullName 'project.version' + $expectedFile = "$artifactId-$version.jar" + + if (-not (Test-Path (Join-Path $flattenedDirectory $expectedFile))) { + Write-Host " Skipping $artifactId-$version (JAR file not found)" + continue + } + + $packages += [PSCustomObject]@{ + Coordinate = "${groupId}:${artifactId}:${version}:jar" + ExpectedFile = $expectedFile + } + } + + Write-Host "Found $($packages.Count) package jar(s) to verify." + foreach ($package in $packages) { + Write-Host " $($package.Coordinate)" + } + + $pending = @($packages) + + for ($attempt = 1; $attempt -le $maxAttempts -and $pending.Count -gt 0; $attempt++) { + $failed = @() + $repoPath = Join-Path $workRoot "maven-repo-$attempt" + New-Item -Path $repoPath -ItemType Directory -Force | Out-Null + + Write-Host "Attempt: $attempt" + + foreach ($package in $pending) { + Write-Host " Downloading $($package.Coordinate)..." + Set-Location $mavenWork + & mvn ` + --batch-mode ` + -q ` + org.apache.maven.plugins:maven-dependency-plugin:copy ` + "-Dmaven.repo.local=$repoPath" ` + "-Dartifact=$($package.Coordinate)" ` + "-DoutputDirectory=$downloadRoot" ` + -Dtransitive=false + + $expectedPath = Join-Path $downloadRoot $package.ExpectedFile + if ($LASTEXITCODE -eq 0 -and (Test-Path $expectedPath)) { + Write-Host " Verified $($package.ExpectedFile)" + } + else { + Write-Host " Failed $($package.ExpectedFile)" + $failed += $package + } + } + + $pending = @($failed) + if ($pending.Count -gt 0 -and $attempt -lt $maxAttempts) { + Write-Host "Retrying $($pending.Count) package(s) in $retrySeconds seconds." + Start-Sleep -Seconds $retrySeconds + } + } + + if ($pending.Count -gt 0) { + Write-Host "These package jars still failed:" + foreach ($package in $pending) { + Write-Host " $($package.Coordinate)" + } + exit 1 + } + + Write-Host 'All package jars were verified published.'