|
| 1 | +# Microsoft Corporation |
| 2 | +# |
| 3 | +# Raw stages template for the sources-upload PR check pipeline. |
| 4 | +# |
| 5 | +# This template is OneBranch-agnostic: it declares the stages/jobs/steps that |
| 6 | +# do the actual work and exposes the OneBranch-coupled knobs as parameters. |
| 7 | +# The wrapper at .github/workflows/ado/sources-upload.yml is responsible for |
| 8 | +# choosing the OneBranch governed template variant (Official vs NonOfficial), |
| 9 | +# configuring its featureFlags, and supplying concrete values for the |
| 10 | +# parameters declared below. |
| 11 | + |
| 12 | +parameters: |
| 13 | + - name: outputDirectory |
| 14 | + type: string |
| 15 | + - name: artifactBaseName |
| 16 | + type: string |
| 17 | + - name: containerImage |
| 18 | + type: string |
| 19 | + - name: poolType |
| 20 | + type: string |
| 21 | + default: linux |
| 22 | + - name: serviceConnection |
| 23 | + type: string |
| 24 | + - name: variableGroup |
| 25 | + type: string |
| 26 | + - name: timeoutInMinutes |
| 27 | + type: number |
| 28 | + |
| 29 | +stages: |
| 30 | + - stage: PRCheck |
| 31 | + jobs: |
| 32 | + - job: CallControlTowerAPI |
| 33 | + # Non-blocking PR check: failing steps still render red error annotations |
| 34 | + # and surface in the build-issues view, but the job resolves to |
| 35 | + # SucceededWithIssues and the run to PartiallySucceeded, which the Azure |
| 36 | + # Pipelines GitHub integration reports as success to GitHub PR checks and |
| 37 | + # the merge queue. Errors and warnings remain visible to the user. |
| 38 | + # TODO: Remove `continueOnError` once the pipeline is stable so failures |
| 39 | + # block PRs and the merge queue. |
| 40 | + # ADO task: 19179 |
| 41 | + continueOnError: true |
| 42 | + # Must exceed the script's --poll-timeout-seconds (default 7200s = 120m) |
| 43 | + # with enough headroom for setup steps and the final API call. |
| 44 | + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} |
| 45 | + pool: |
| 46 | + type: ${{ parameters.poolType }} |
| 47 | + variables: |
| 48 | + - group: ${{ parameters.variableGroup }} |
| 49 | + - name: ob_outputDirectory |
| 50 | + value: ${{ parameters.outputDirectory }} |
| 51 | + - name: ob_artifactBaseName |
| 52 | + value: ${{ parameters.artifactBaseName }} |
| 53 | + - name: LinuxContainerImage |
| 54 | + value: ${{ parameters.containerImage }} |
| 55 | + steps: |
| 56 | + - script: | |
| 57 | + set -euo pipefail |
| 58 | +
|
| 59 | + # For both triggers, Build.SourceVersion is the relevant source commit: |
| 60 | + # - PR trigger: ADO sets it to the GitHub-created merge commit (refs/pull/{n}/merge). |
| 61 | + # - CI/merge-queue trigger: it is the commit that landed on the base branch. |
| 62 | + source_hash="$SOURCE_COMMIT" |
| 63 | +
|
| 64 | + if [[ "$BUILD_REASON" == "PullRequest" ]]; then |
| 65 | + # PR trigger: ADO provides the target branch directly. |
| 66 | + target_hash=$(git ls-remote "$UPSTREAM_REPO_URL" "$PR_TARGET_BRANCH" | cut -f1) |
| 67 | + else |
| 68 | + # CI/merge-queue trigger: extract the base branch from the merge-queue branch name. |
| 69 | + # Branch format: refs/heads/gh-readonly-queue/{base_branch}/pr-{pr_number}-{head_sha} |
| 70 | + if ! base_branch=$(grep -oP '(?<=^refs/heads/gh-readonly-queue/).+(?=/pr-[^/]+$)' <<< "$SOURCE_BRANCH"); then |
| 71 | + echo "##[error]Unsupported SOURCE_BRANCH '$SOURCE_BRANCH' for non-PullRequest build. Expected 'refs/heads/gh-readonly-queue/<base>/pr-<n>-<sha>'." |
| 72 | + exit 1 |
| 73 | + fi |
| 74 | + target_hash=$(git ls-remote "$UPSTREAM_REPO_URL" "refs/heads/$base_branch" | cut -f1) |
| 75 | + fi |
| 76 | +
|
| 77 | + echo "Source commit: $source_hash" |
| 78 | + echo "Target commit: $target_hash" |
| 79 | +
|
| 80 | + echo "##vso[task.setvariable variable=sourceCommit]$source_hash" |
| 81 | + echo "##vso[task.setvariable variable=targetCommit]$target_hash" |
| 82 | + env: |
| 83 | + BUILD_REASON: $(Build.Reason) |
| 84 | + PR_TARGET_BRANCH: $(System.PullRequest.TargetBranch) |
| 85 | + SOURCE_BRANCH: $(Build.SourceBranch) |
| 86 | + SOURCE_COMMIT: $(Build.SourceVersion) |
| 87 | + UPSTREAM_REPO_URL: $(Build.Repository.Uri) |
| 88 | + displayName: "Determine source and target commit hashes" |
| 89 | +
|
| 90 | + - task: PipAuthenticate@1 |
| 91 | + displayName: "Authenticate pip" |
| 92 | + inputs: |
| 93 | + artifactFeeds: "azl/ControlTowerFeed" |
| 94 | + |
| 95 | + - script: | |
| 96 | + set -euo pipefail |
| 97 | +
|
| 98 | + echo "##[group]Mock" |
| 99 | + tdnf install -y mock mock-rpmautospec python3-chardet |
| 100 | + sudo usermod -aG mock "$(whoami)" |
| 101 | + echo "##[endgroup]" |
| 102 | +
|
| 103 | + echo "##[group]Azldev" |
| 104 | + echo "Installing azldev@${AZLDEV_COMMIT}..." |
| 105 | + go install "github.com/microsoft/azure-linux-dev-tools/cmd/azldev@${AZLDEV_COMMIT}" |
| 106 | +
|
| 107 | + go_bin_path="$(go env GOPATH)/bin" |
| 108 | + echo "##vso[task.prependpath]$go_bin_path" |
| 109 | +
|
| 110 | + "$go_bin_path/azldev" --version |
| 111 | + echo "##[endgroup]" |
| 112 | +
|
| 113 | + echo "##[group]Python dependencies" |
| 114 | + pip install -r .github/workflows/scripts/control-tower-prcheck/requirements.txt |
| 115 | + echo "##[endgroup]" |
| 116 | + displayName: "Install dependencies" |
| 117 | + env: |
| 118 | + AZLDEV_COMMIT: $(AzldevCommit) |
| 119 | +
|
| 120 | + - script: | |
| 121 | + set -euo pipefail |
| 122 | +
|
| 123 | + # Workaround for an ADO git config error during spec rendering. |
| 124 | + # The config key may not be present on every agent image, so tolerate its absence. |
| 125 | + git config --unset extensions.worktreeConfig || true |
| 126 | +
|
| 127 | + # Full history is needed for spec rendering to work. |
| 128 | + if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then |
| 129 | + echo "##[group]Fetching full git history" |
| 130 | + git fetch --unshallow |
| 131 | + echo "##[endgroup]" |
| 132 | + fi |
| 133 | +
|
| 134 | + # TODO: Replace with using the TOML config to read the specs dir once available. |
| 135 | + # The '-o "$specs_dir"' should be removed as part of the update. Future command: |
| 136 | + # specs_dir="$(azldev config dump -q -f json | jq -r .project.renderedSpecsDir)" |
| 137 | + # Similarly, the component rendering below should change to: |
| 138 | + # azldev component render -q -a |
| 139 | + # ADO task: 19149 |
| 140 | + specs_dir="specs" |
| 141 | +
|
| 142 | + echo "##[group]Specs rendering" |
| 143 | + azldev component render -q -a -o "$specs_dir" --force --clean-stale |
| 144 | + echo "##[endgroup]" |
| 145 | +
|
| 146 | + # Check for any new or modified files under the specs directory. |
| 147 | + # TODO: Remove and replace this script with the one coming from PR #16674. |
| 148 | + # ADO task: 19186 |
| 149 | + if ! python3 .github/workflows/scripts/check_rendered_specs.py --specs-dir "$specs_dir"; then |
| 150 | + echo "##[group]Git diff" |
| 151 | + git --no-pager diff -- "$specs_dir" |
| 152 | + echo "##[endgroup]" |
| 153 | + echo "##[error]Specs are not up to date. Run 'azldev component render -q -a' and try again." |
| 154 | + exit 1 |
| 155 | + fi |
| 156 | + displayName: "Verify rendered specs are up to date" |
| 157 | + # TODO: Re-enabled failing once specs in the main branch are updated and stable. |
| 158 | + # ADO task: 19179 |
| 159 | + continueOnError: true |
| 160 | +
|
| 161 | + - script: | |
| 162 | + set -euo pipefail |
| 163 | +
|
| 164 | + cat << EOF |
| 165 | + NOTE: It's possible more components changed between the source and target commits than displayed below. |
| 166 | + This workflow only checks for updates in the 'sources' files and ignores all other changes. |
| 167 | + Only components present on the source commit are reported; components that exist only on the |
| 168 | + target branch are intentionally excluded. Pure renames are reported under their source-branch |
| 169 | + name, since the component name is part of the cache blob name we upload. |
| 170 | + EOF |
| 171 | +
|
| 172 | + # Diff direction is TARGET -> SOURCE so additions/modifications represent the source-side state: |
| 173 | + # * --no-renames decomposes a rename into an add (new path, source side) + delete (old path, |
| 174 | + # target side); we want the source-side name because the component name is part of the cache |
| 175 | + # blob name uploaded for the PR. It also silences git's diff.renameLimit warning on large diffs. |
| 176 | + # * --diff-filter=d (lowercase) excludes deletions, which here correspond to paths present only |
| 177 | + # on the target branch -- those components are not on the PR and must not be uploaded. |
| 178 | + # Net effect: the listed '*/sources' files are exactly those present on the source commit that |
| 179 | + # differ from target, including pure renames surfaced under the source-branch name. |
| 180 | + sources_files="$( |
| 181 | + git diff --no-renames --diff-filter=d "$TARGET_COMMIT" "$SOURCE_COMMIT" --name-only \ |
| 182 | + | { grep '/sources$' || true; } \ |
| 183 | + | sort -u |
| 184 | + )" |
| 185 | +
|
| 186 | + components=() |
| 187 | + while IFS= read -r line; do |
| 188 | + # Skip the spurious empty iteration when no '*/sources' files changed. |
| 189 | + if [[ -n "$line" ]]; then |
| 190 | + components+=("$(basename "$(dirname "$line")")") |
| 191 | + fi |
| 192 | + done <<< "$sources_files" |
| 193 | +
|
| 194 | + components="$(IFS=, ; echo "${components[*]}")" |
| 195 | + echo "Affected components: $components" |
| 196 | + echo "##vso[task.setvariable variable=components]$components" |
| 197 | + env: |
| 198 | + SOURCE_COMMIT: $(sourceCommit) |
| 199 | + TARGET_COMMIT: $(targetCommit) |
| 200 | + displayName: "Determine affected components" |
| 201 | +
|
| 202 | + - task: AzureCLI@2 |
| 203 | + displayName: "Call Control Tower 'prcheck' API" |
| 204 | + inputs: |
| 205 | + azureSubscription: ${{ parameters.serviceConnection }} |
| 206 | + scriptType: bash |
| 207 | + scriptLocation: inlineScript |
| 208 | + inlineScript: | |
| 209 | + set -euo pipefail |
| 210 | +
|
| 211 | + python3 .github/workflows/scripts/control-tower-prcheck/run_control_tower_prcheck.py \ |
| 212 | + --api-audience "$API_AUDIENCE" \ |
| 213 | + --api-base-url "$API_BASE_URL" \ |
| 214 | + --build-reason "$BUILD_REASON" \ |
| 215 | + --components "$COMPONENTS" \ |
| 216 | + --source-commit "$SOURCE_COMMIT" \ |
| 217 | + --repo-uri "$UPSTREAM_REPO_URL" |
| 218 | + env: |
| 219 | + API_AUDIENCE: $(ApiAudience) |
| 220 | + API_BASE_URL: $(ApiBaseUrl) |
| 221 | + BUILD_REASON: $(Build.Reason) |
| 222 | + COMPONENTS: $(components) |
| 223 | + SOURCE_COMMIT: $(sourceCommit) |
| 224 | + # TODO: Target commit is not used. Will be needed once we move detection of affected components to CT. |
| 225 | + # ADO task: 18816 |
| 226 | + TARGET_COMMIT: $(targetCommit) |
| 227 | + UPSTREAM_REPO_URL: $(Build.Repository.Uri) |
0 commit comments