|
| 1 | +#!/usr/bin/env pwsh |
| 2 | +<# |
| 3 | +.SYNOPSIS |
| 4 | + Mirrors a gradle project's dependencies into the dnceng dotnet-public-maven |
| 5 | + Azure Artifacts feed so CI can resolve them anonymously. |
| 6 | +
|
| 7 | +.DESCRIPTION |
| 8 | + When Dependabot bumps a gradle dependency (or its transitive graph changes), |
| 9 | + CI fails with 401 errors because the new package(s) haven't been pulled |
| 10 | + from upstream into the dnceng feed yet. CI agents only do anonymous reads, |
| 11 | + so a developer has to authenticate locally once to seed the feed. |
| 12 | +
|
| 13 | + This script does that by running the requested gradle build in a loop: |
| 14 | + 1. Run gradle with RunningOnCI=true so it points at the dnceng feed. |
| 15 | + 2. Parse any 'Could not GET' URLs out of the build log. |
| 16 | + 3. Re-fetch each failing URL with an Azure DevOps OAuth bearer token |
| 17 | + (obtained via `az account get-access-token`). The feed's upstream |
| 18 | + connector then pulls the package and caches it for anonymous reads. |
| 19 | + 4. Repeat until the build succeeds or no more 401s appear. |
| 20 | +
|
| 21 | + After the loop converges, no PR edits are needed — just re-run the failing |
| 22 | + CI job, since the packages are now anonymous-readable. |
| 23 | +
|
| 24 | +.PARAMETER ProjectDir |
| 25 | + Path to the gradle project (the one containing the failing dependency). |
| 26 | + Mirroring must run in the project that actually requires the package; |
| 27 | + a sibling project's build won't trigger a mirror for someone else's deps. |
| 28 | +
|
| 29 | +.PARAMETER Task |
| 30 | + Gradle task(s) to run. Should be one that resolves the new dependency |
| 31 | + graph (e.g. 'assembleDebug', 'build', 'extractProguardFiles'). |
| 32 | +
|
| 33 | +.PARAMETER AndroidHome |
| 34 | + Optional path to the Android SDK. Required when the gradle build needs it |
| 35 | + (any project using the com.android.* plugins). Defaults to the value of |
| 36 | + `$env:ANDROID_HOME` if set. |
| 37 | +
|
| 38 | +.PARAMETER MaxIterations |
| 39 | + Cap on build/mirror cycles. Default 15. Typical convergence is 2-5 |
| 40 | + iterations as the resolver walks the dep graph breadth-first. |
| 41 | +
|
| 42 | +.EXAMPLE |
| 43 | + pwsh ./eng/gradle/mirror-dependencies.ps1 ` |
| 44 | + -ProjectDir tests/CodeGen-Binding/Xamarin.Android.LibraryProjectZip-LibBinding/java/JavaLib ` |
| 45 | + -Task assembleDebug ` |
| 46 | + -AndroidHome D:\android-toolchain\sdk |
| 47 | +
|
| 48 | +.EXAMPLE |
| 49 | + pwsh ./eng/gradle/mirror-dependencies.ps1 -ProjectDir src/proguard-android -Task extractProguardFiles |
| 50 | +#> |
| 51 | +[CmdletBinding()] |
| 52 | +param( |
| 53 | + [Parameter(Mandatory=$true)] |
| 54 | + [string] $ProjectDir, |
| 55 | + |
| 56 | + [Parameter(Mandatory=$true)] |
| 57 | + [string] $Task, |
| 58 | + |
| 59 | + [string] $AndroidHome = $env:ANDROID_HOME, |
| 60 | + |
| 61 | + [int] $MaxIterations = 15 |
| 62 | +) |
| 63 | + |
| 64 | +$ErrorActionPreference = 'Stop' |
| 65 | +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '../..') | Select-Object -ExpandProperty Path |
| 66 | +$projectDirAbs = Resolve-Path (Join-Path $repoRoot $ProjectDir) -ErrorAction Stop | Select-Object -ExpandProperty Path |
| 67 | +$gradlew = if ($IsWindows -or $env:OS -eq 'Windows_NT') { |
| 68 | + Join-Path $repoRoot 'build-tools/gradle/gradlew.bat' |
| 69 | +} else { |
| 70 | + Join-Path $repoRoot 'build-tools/gradle/gradlew' |
| 71 | +} |
| 72 | +if (-not (Test-Path $gradlew)) { throw "gradlew not found at $gradlew" } |
| 73 | + |
| 74 | +# Azure DevOps resource id — same for every AzDO tenant. |
| 75 | +$azDevOpsResource = '499b84ac-1321-427f-aa17-267ca6975798' |
| 76 | + |
| 77 | +function Get-AzDevOpsToken { |
| 78 | + $token = az account get-access-token --resource $azDevOpsResource --query accessToken -o tsv 2>$null |
| 79 | + if ([string]::IsNullOrEmpty($token)) { |
| 80 | + throw "Could not get an Azure DevOps access token. Run 'az login' first." |
| 81 | + } |
| 82 | + return $token |
| 83 | +} |
| 84 | + |
| 85 | +function Invoke-Mirror($logPath) { |
| 86 | + $urls = Select-String -Path $logPath -Pattern "Could not GET 'https://pkgs\.dev\.azure\.com/dnceng/[^']+'" -AllMatches | |
| 87 | + ForEach-Object { $_.Matches } | |
| 88 | + ForEach-Object { $_.Value -replace "^Could not GET '", "" -replace "'$", "" } | |
| 89 | + Sort-Object -Unique |
| 90 | + if ($urls.Count -eq 0) { return 0 } |
| 91 | + $token = Get-AzDevOpsToken |
| 92 | + $headers = @{ Authorization = "Bearer $token" } |
| 93 | + $ok = 0; $fail = 0 |
| 94 | + foreach ($u in $urls) { |
| 95 | + try { |
| 96 | + $r = Invoke-WebRequest -Uri $u -Headers $headers -SkipHttpErrorCheck -ErrorAction Stop |
| 97 | + if ($r.StatusCode -eq 200) { $ok++ } else { $fail++; Write-Host " $($r.StatusCode) $u" -ForegroundColor Yellow } |
| 98 | + } catch { |
| 99 | + $fail++ |
| 100 | + Write-Host " ERR $u : $_" -ForegroundColor Yellow |
| 101 | + } |
| 102 | + } |
| 103 | + Write-Host " -> mirrored OK=$ok, not-found=$fail (of $($urls.Count))" -ForegroundColor Cyan |
| 104 | + return $urls.Count |
| 105 | +} |
| 106 | + |
| 107 | +Write-Host "Repo root: $repoRoot" |
| 108 | +Write-Host "Project: $projectDirAbs" |
| 109 | +Write-Host "Task: $Task" |
| 110 | +if ($AndroidHome) { Write-Host "ANDROID_HOME: $AndroidHome" } |
| 111 | + |
| 112 | +# Verify az is available and authenticated up front so we fail fast. |
| 113 | +Get-AzDevOpsToken | Out-Null |
| 114 | + |
| 115 | +if ($AndroidHome) { $env:ANDROID_HOME = $AndroidHome } |
| 116 | +$env:RunningOnCI = 'true' |
| 117 | + |
| 118 | +Push-Location $projectDirAbs |
| 119 | +try { |
| 120 | + for ($i = 1; $i -le $MaxIterations; $i++) { |
| 121 | + Write-Host "`n=== iteration $i ===" -ForegroundColor Green |
| 122 | + $log = Join-Path ([IO.Path]::GetTempPath()) "gradle-mirror-iter-$i.log" |
| 123 | + & $gradlew $Task --no-daemon --refresh-dependencies *>&1 | Tee-Object -FilePath $log | Out-Null |
| 124 | + if (Select-String -Path $log -Pattern 'BUILD SUCCESSFUL' -SimpleMatch -Quiet) { |
| 125 | + Write-Host "`nBUILD SUCCESSFUL after $i iteration(s). The feed now has the packages CI needs." -ForegroundColor Green |
| 126 | + return |
| 127 | + } |
| 128 | + $count = Invoke-Mirror $log |
| 129 | + if ($count -eq 0) { |
| 130 | + Write-Host "`nGradle failed but no 401s to mirror — see $log" -ForegroundColor Red |
| 131 | + Get-Content $log -Tail 30 |
| 132 | + exit 1 |
| 133 | + } |
| 134 | + } |
| 135 | + Write-Host "`nExhausted $MaxIterations iterations without success. Last log:" -ForegroundColor Red |
| 136 | + Get-Content $log -Tail 30 |
| 137 | + exit 1 |
| 138 | +} |
| 139 | +finally { |
| 140 | + Pop-Location |
| 141 | +} |
![dependabot[bot]](https://avatars.githubusercontent.com/in/29110?v=4&size=40){kind=avatar}
0 commit comments