Skip to content

Commit b463476

Browse files
dmcilvaneydmcilvaney
committed
fixup! ci(prcheck): use 'azldev component changed' for affected-component detection
1 parent de01ebc commit b463476

5 files changed

Lines changed: 233 additions & 147 deletions

File tree

‎.github/workflows/ado/templates/sources-upload-stages.yml‎

Lines changed: 20 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -117,43 +117,11 @@ stages:
117117
env:
118118
AZLDEV_COMMIT: $(AzldevCommit)
119119
120+
# Re-generate lock files and fail if any drifted from what's committed.
120121
- script: |
121-
set -euo pipefail
122-
123-
# Workaround for an ADO git config error.
124-
# The config key may not be present on every agent image, so tolerate its absence.
125-
git config --unset extensions.worktreeConfig || true
126-
127-
# Full history is needed for lock resolution and spec rendering.
128-
if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then
129-
echo "##[group]Fetching full git history"
130-
git fetch --unshallow
131-
echo "##[endgroup]"
132-
fi
133-
134-
echo "##[group]Updating lock files"
135-
lock_update_file="$ARTIFACT_STAGING_DIR/lock-update.json"
136-
azldev component update -a -q -O json > "$lock_update_file"
137-
echo "##[endgroup]"
138-
139-
# Also copy into ob_outputDirectory so OneBranch auto-publishes the
140-
# lock-update artifact for post-mortem triage.
141-
ob_lock_dir="$OB_OUTPUT_DIR/lock-update"
142-
mkdir -p "$ob_lock_dir"
143-
cp "$lock_update_file" "$ob_lock_dir/"
144-
145-
# Check if any locks drifted -- azldev tells us directly.
146-
# If locks are not valid, we can't guarantee that any further steps are valid, so fail the PR check immediately.
147-
# azldev emits a JSON 'null' when nothing changed; '. // []' coerces that to an
148-
# empty array so the downstream '[.[] | ...]' pipeline is well-typed.
149-
drifted=$(jq '(. // []) | [.[] | select(.changed == true)] | length' "$lock_update_file")
150-
if [[ "$drifted" -gt 0 ]]; then
151-
echo "##[error]$drifted lock file(s) are not up to date. Run 'azldev component update -a' and commit the result."
152-
echo "##[group]Drifted components"
153-
jq -r '(. // []) | .[] | select(.changed == true) | .component' "$lock_update_file"
154-
echo "##[endgroup]"
155-
exit 1
156-
fi
122+
.github/workflows/scripts/control-tower/verify_locks.sh \
123+
--artifact-dir "$ARTIFACT_STAGING_DIR" \
124+
--ob-output-dir "$OB_OUTPUT_DIR"
157125
displayName: "Verify lock files are up to date"
158126
env:
159127
ARTIFACT_STAGING_DIR: $(Build.ArtifactStagingDirectory)
@@ -162,40 +130,14 @@ stages:
162130
AZLDEV_ALLOW_ROOT: "1"
163131
OB_OUTPUT_DIR: $(ob_outputDirectory)
164132
133+
# Diff source vs target to find added/changed/deleted components and
134+
# emit the changed-components JSON consumed by later steps.
165135
- script: |
166-
set -euo pipefail
167-
168-
artifact_dir="$ARTIFACT_STAGING_DIR/changed-components"
169-
json_file="$artifact_dir/changed-components.json"
170-
mkdir -p "$artifact_dir"
171-
172-
# 'azldev component changed' compares the source and target commits and emits one
173-
# entry per component with its 'changeType' ('added', 'changed', 'unchanged', or
174-
# 'deleted') plus a 'sourcesChange' flag indicating whether the rendered 'sources'
175-
# file differs between commits. Only components with sourcesChange == true AND
176-
# changeType != 'deleted' are forwarded to Control Tower for upload.
177-
178-
echo "##[group]Computing changed components"
179-
azldev component changed --from "$TARGET_COMMIT" --to "$SOURCE_COMMIT" -a --include-unchanged -O json > "$json_file"
180-
echo "##[endgroup]"
181-
182-
echo "##[group]Changed components (non-unchanged only)"
183-
jq '(. // []) | [.[] | select(.changeType != "unchanged")]' "$json_file"
184-
echo "##[endgroup]"
185-
186-
echo "##[group]Upload set (sourcesChange == true, changeType in {added, changed})"
187-
upload_count=$(jq -r '(. // []) | [.[] | select(.sourcesChange == true and (.changeType | IN("added", "changed")))] | length' "$json_file")
188-
jq -r '(. // []) | .[] | select(.sourcesChange == true and (.changeType | IN("added", "changed"))) | .component' "$json_file" | sort
189-
echo "Total: $upload_count component(s) to upload."
190-
echo "##[endgroup]"
191-
192-
# Also write into ob_outputDirectory so OneBranch auto-publishes it
193-
# as a build artifact (explicit PublishPipelineArtifact is forbidden).
194-
ob_dir="$OB_OUTPUT_DIR/changed-components"
195-
mkdir -p "$ob_dir"
196-
cp "$json_file" "$ob_dir/"
197-
198-
echo "##vso[task.setvariable variable=changedComponentsFile;isreadonly=true]$json_file"
136+
.github/workflows/scripts/control-tower/compute_changed.sh \
137+
--artifact-dir "$ARTIFACT_STAGING_DIR" \
138+
--ob-output-dir "$OB_OUTPUT_DIR" \
139+
--source-commit "$SOURCE_COMMIT" \
140+
--target-commit "$TARGET_COMMIT"
199141
env:
200142
ARTIFACT_STAGING_DIR: $(Build.ArtifactStagingDirectory)
201143
# OneBranch containers run as root. azldev refuses to run as root
@@ -206,28 +148,10 @@ stages:
206148
TARGET_COMMIT: $(targetCommit)
207149
displayName: "Compute changed components"
208150
151+
# Fail if any component's sources file changed without an identity change.
209152
- script: |
210-
set -euo pipefail
211-
212-
# Source/identity consistency tripwire. The rendered 'sources' file
213-
# describes the lookaside tarballs Control Tower will fetch and serve to
214-
# builds. If a PR mutates that file but the component's input fingerprint
215-
# is unchanged, we cannot tell from this side whether that's a hostile
216-
# supply-chain swap or an accidental hand-edit / renderer non-determinism
217-
# bug. We treat any such record as hostile-by-default and hard-fail, so
218-
# the new bytes never reach the lookaside under an existing identity.
219-
#
220-
# The legitimate cases are explicitly enumerated as an allow-list (added,
221-
# changed, deleted) so any future 'changeType' value -- e.g. 'renamed'
222-
# that doesn't pull the sources blob into the input fingerprint -- fails
223-
# closed until the policy here is reviewed.
224-
bogus_count=$(jq -r '(. // []) | [.[] | select(.sourcesChange == true and (.changeType | IN("added", "changed", "deleted") | not))] | length' "$CHANGED_COMPONENTS_FILE")
225-
if [[ "$bogus_count" -gt 0 ]]; then
226-
echo "##[error]$bogus_count component(s) report a sources change without an accompanying identity change. Refusing to forward to Control Tower."
227-
jq -r '(. // []) | .[] | select(.sourcesChange == true and (.changeType | IN("added", "changed", "deleted") | not)) | " - " + .component + " (changeType=" + (.changeType // "null") + ")"' "$CHANGED_COMPONENTS_FILE"
228-
exit 1
229-
fi
230-
echo "OK: every sources change is accompanied by an identity change."
153+
.github/workflows/scripts/control-tower/consistency_check.sh \
154+
--changed-components-file "$CHANGED_COMPONENTS_FILE"
231155
# When this step fails, subsequent steps in the job are skipped -- the
232156
# bad data never reaches Control Tower. However, the job-level
233157
# continueOnError: true means the PR check still reports green to
@@ -237,65 +161,14 @@ stages:
237161
CHANGED_COMPONENTS_FILE: $(changedComponentsFile)
238162
displayName: "Source/identity consistency check"
239163
164+
# Render specs for changed components and verify the tree has no
165+
# uncommitted modifications or untracked files.
240166
- script: |
241-
set -euo pipefail
242-
243-
# azldev's renderedSpecsDir is absolute. Translate to repo-relative
244-
# so it matches git's output ('git diff --name-only' always emits
245-
# repo-relative paths regardless of the path arg form).
246-
specs_dir_abs="$(azldev config dump -q -f json | jq -r '.project.renderedSpecsDir')"
247-
specs_dir="$(realpath --relative-to="$(pwd)" "$specs_dir_abs")"
248-
249-
# Capture git diff under the specs tree so the render set can
250-
# include components whose specs were edited directly (which
251-
# azldev's input-fingerprint view of "changed" would miss).
252-
# --no-renames prevents collapse of delete+add into a rename
253-
# entry which would lose the old path. The Python script
254-
# filters out deleted/unknown components using the full
255-
# changed-components JSON.
256-
specs_diff_file="$ARTIFACT_STAGING_DIR/specs-diff.txt"
257-
git diff --no-renames --name-only "$TARGET_COMMIT" "$SOURCE_COMMIT" -- "$specs_dir" > "$specs_diff_file"
258-
259-
# Render set is the union of:
260-
# - components flagged by 'azldev component changed' (inputs differ)
261-
# - components whose spec tree was touched directly in the PR
262-
changed=$(python3 .github/workflows/scripts/control-tower/compute_render_set.py \
167+
.github/workflows/scripts/control-tower/render_and_verify.sh \
168+
--artifact-dir "$ARTIFACT_STAGING_DIR" \
263169
--changed-components-file "$CHANGED_COMPONENTS_FILE" \
264-
--specs-diff-file "$specs_diff_file" \
265-
--specs-dir "$specs_dir")
266-
267-
if [[ -z "$changed" ]]; then
268-
echo "No changed components -- skipping render."
269-
else
270-
changed_count=$(echo "$changed" | wc -l)
271-
echo "Rendering $changed_count component(s) (azldev dedupes internally)..."
272-
echo "##[group]Render set"
273-
echo "$changed" | sed 's/^/ - /'
274-
echo "##[endgroup]"
275-
echo "##[group]Specs rendering"
276-
printf '%s\n' "$changed" | xargs -d '\n' azldev component render -q --
277-
echo "##[endgroup]"
278-
fi
279-
280-
# Quick sanity check: no modified or untracked files under specs/.
281-
drift=0
282-
if ! git diff --quiet -- "$specs_dir"; then
283-
echo "##[group]Git diff"
284-
git --no-pager diff -- "$specs_dir"
285-
echo "##[endgroup]"
286-
drift=1
287-
fi
288-
untracked=$(git ls-files --others -- "$specs_dir")
289-
if [[ -n "$untracked" ]]; then
290-
echo "##[group]Untracked files"
291-
echo "$untracked"
292-
echo "##[endgroup]"
293-
drift=1
294-
fi
295-
if [[ "$drift" -ne 0 ]]; then
296-
echo "##[error]Specs are not up to date. Run 'azldev component render -q -a --clean-stale' and try again."
297-
exit 1
298-
fi
170+
--source-commit "$SOURCE_COMMIT" \
171+
--target-commit "$TARGET_COMMIT"
299172
env:
300173
ARTIFACT_STAGING_DIR: $(Build.ArtifactStagingDirectory)
301174
# OneBranch containers run as root. azldev refuses to run as root

‎.github/workflows/scripts/control-tower/compute_changed.sh‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/usr/bin/env bash
2+
# Compute the set of changed components between source and target commits.
3+
#
4+
# Outputs (ADO variables):
5+
# changedComponentsFile — path to the changed-components JSON file
6+
7+
set -euo pipefail
8+
9+
usage() { echo "Usage: $0 --artifact-dir DIR --ob-output-dir DIR --source-commit SHA --target-commit SHA" >&2; exit 1; }
10+
11+
while [[ $# -gt 0 ]]; do
12+
case "$1" in
13+
--artifact-dir) artifact_staging_dir="$2"; shift 2 ;;
14+
--ob-output-dir) ob_output_dir="$2"; shift 2 ;;
15+
--source-commit) source_commit="$2"; shift 2 ;;
16+
--target-commit) target_commit="$2"; shift 2 ;;
17+
*) usage ;;
18+
esac
19+
done
20+
[[ -z "${artifact_staging_dir:-}" || -z "${ob_output_dir:-}" || -z "${source_commit:-}" || -z "${target_commit:-}" ]] && usage
21+
22+
artifact_dir="$artifact_staging_dir/changed-components"
23+
json_file="$artifact_dir/changed-components.json"
24+
mkdir -p "$artifact_dir"
25+
26+
# 'azldev component changed' compares the source and target commits and emits one
27+
# entry per component with its 'changeType' ('added', 'changed', 'unchanged', or
28+
# 'deleted') plus a 'sourcesChange' flag indicating whether the rendered 'sources'
29+
# file differs between commits. Only components with sourcesChange == true AND
30+
# changeType != 'deleted' are forwarded to Control Tower for upload.
31+
32+
echo "##[group]Computing changed components"
33+
azldev component changed --from "$target_commit" --to "$source_commit" -a --include-unchanged -O json > "$json_file"
34+
echo "##[endgroup]"
35+
36+
echo "##[group]Changed components (non-unchanged only)"
37+
jq '(. // []) | [.[] | select(.changeType != "unchanged")]' "$json_file"
38+
echo "##[endgroup]"
39+
40+
echo "##[group]Upload set (sourcesChange == true, changeType in {added, changed})"
41+
upload_count=$(jq -r '(. // []) | [.[] | select(.sourcesChange == true and (.changeType | IN("added", "changed")))] | length' "$json_file")
42+
jq -r '(. // []) | .[] | select(.sourcesChange == true and (.changeType | IN("added", "changed"))) | .component' "$json_file" | sort
43+
echo "Total: $upload_count component(s) to upload."
44+
echo "##[endgroup]"
45+
46+
# Also write into ob_outputDirectory so OneBranch auto-publishes it
47+
# as a build artifact (explicit PublishPipelineArtifact is forbidden).
48+
ob_dir="$ob_output_dir/changed-components"
49+
mkdir -p "$ob_dir"
50+
cp "$json_file" "$ob_dir/"
51+
52+
echo "##vso[task.setvariable variable=changedComponentsFile;isreadonly=true]$json_file"

‎.github/workflows/scripts/control-tower/consistency_check.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/usr/bin/env bash
2+
# Source/identity consistency check.
3+
#
4+
# Ensures every component with a sources change also has an identity
5+
# (input fingerprint) change. Fails closed on unknown changeTypes.
6+
7+
set -euo pipefail
8+
9+
usage() { echo "Usage: $0 --changed-components-file FILE" >&2; exit 1; }
10+
11+
while [[ $# -gt 0 ]]; do
12+
case "$1" in
13+
--changed-components-file) changed_components_file="$2"; shift 2 ;;
14+
*) usage ;;
15+
esac
16+
done
17+
[[ -z "${changed_components_file:-}" ]] && usage
18+
19+
# Source/identity consistency tripwire. The rendered 'sources' file
20+
# describes the lookaside tarballs Control Tower will fetch and serve to
21+
# builds. If a PR mutates that file but the component's input fingerprint
22+
# is unchanged, we cannot tell from this side whether that's a hostile
23+
# supply-chain swap or an accidental hand-edit / renderer non-determinism
24+
# bug. We treat any such record as hostile-by-default and hard-fail, so
25+
# the new bytes never reach the lookaside under an existing identity.
26+
#
27+
# The legitimate cases are explicitly enumerated as an allow-list (added,
28+
# changed, deleted) so any future 'changeType' value -- e.g. 'renamed'
29+
# that doesn't pull the sources blob into the input fingerprint -- fails
30+
# closed until the policy here is reviewed.
31+
bogus_count=$(jq -r '(. // []) | [.[] | select(.sourcesChange == true and (.changeType | IN("added", "changed", "deleted") | not))] | length' "$changed_components_file")
32+
if [[ "$bogus_count" -gt 0 ]]; then
33+
echo "##[error]$bogus_count component(s) report a sources change without an accompanying identity change. Refusing to forward to Control Tower."
34+
jq -r '(. // []) | .[] | select(.sourcesChange == true and (.changeType | IN("added", "changed", "deleted") | not)) | " - " + .component + " (changeType=" + (.changeType // "null") + ")"' "$changed_components_file"
35+
exit 1
36+
fi
37+
echo "OK: every sources change is accompanied by an identity change."

‎.github/workflows/scripts/control-tower/render_and_verify.sh‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/usr/bin/env bash
2+
# Render changed specs and verify the rendered tree is clean.
3+
4+
set -euo pipefail
5+
6+
usage() { echo "Usage: $0 --artifact-dir DIR --changed-components-file FILE --source-commit SHA --target-commit SHA" >&2; exit 1; }
7+
8+
while [[ $# -gt 0 ]]; do
9+
case "$1" in
10+
--artifact-dir) artifact_dir="$2"; shift 2 ;;
11+
--changed-components-file) changed_components_file="$2"; shift 2 ;;
12+
--source-commit) source_commit="$2"; shift 2 ;;
13+
--target-commit) target_commit="$2"; shift 2 ;;
14+
*) usage ;;
15+
esac
16+
done
17+
[[ -z "${artifact_dir:-}" || -z "${changed_components_file:-}" || -z "${source_commit:-}" || -z "${target_commit:-}" ]] && usage
18+
19+
# azldev's renderedSpecsDir is absolute. Translate to repo-relative
20+
# so it matches git's output ('git diff --name-only' always emits
21+
# repo-relative paths regardless of the path arg form).
22+
specs_dir_abs="$(azldev config dump -q -f json | jq -r '.project.renderedSpecsDir')"
23+
specs_dir="$(realpath --relative-to="$(pwd)" "$specs_dir_abs")"
24+
25+
# Capture git diff under the specs tree so the render set can
26+
# include components whose specs were edited directly (which
27+
# azldev's input-fingerprint view of "changed" would miss).
28+
# --no-renames prevents collapse of delete+add into a rename
29+
# entry which would lose the old path. The Python script
30+
# filters out deleted/unknown components using the full
31+
# changed-components JSON.
32+
specs_diff_file="$artifact_dir/specs-diff.txt"
33+
git diff --no-renames --name-only "$target_commit" "$source_commit" -- "$specs_dir" > "$specs_diff_file"
34+
35+
# Render set is the union of:
36+
# - components flagged by 'azldev component changed' (inputs differ)
37+
# - components whose spec tree was touched directly in the PR
38+
changed=$(python3 .github/workflows/scripts/control-tower/compute_render_set.py \
39+
--changed-components-file "$changed_components_file" \
40+
--specs-diff-file "$specs_diff_file" \
41+
--specs-dir "$specs_dir")
42+
43+
if [[ -z "$changed" ]]; then
44+
echo "No changed components -- skipping render."
45+
else
46+
changed_count=$(echo "$changed" | wc -l)
47+
echo "Rendering $changed_count component(s) (azldev dedupes internally)..."
48+
echo "##[group]Render set"
49+
echo "$changed" | sed 's/^/ - /'
50+
echo "##[endgroup]"
51+
echo "##[group]Specs rendering"
52+
printf '%s\n' "$changed" | xargs -d '\n' azldev component render -q --
53+
echo "##[endgroup]"
54+
fi
55+
56+
# Quick sanity check: no modified or untracked files under specs/.
57+
drift=0
58+
if ! git diff --quiet -- "$specs_dir"; then
59+
echo "##[group]Git diff"
60+
git --no-pager diff -- "$specs_dir"
61+
echo "##[endgroup]"
62+
drift=1
63+
fi
64+
untracked=$(git ls-files --others -- "$specs_dir")
65+
if [[ -n "$untracked" ]]; then
66+
echo "##[group]Untracked files"
67+
echo "$untracked"
68+
echo "##[endgroup]"
69+
drift=1
70+
fi
71+
if [[ "$drift" -ne 0 ]]; then
72+
echo "##[error]Specs are not up to date. Run 'azldev component render -q -a --clean-stale' and try again."
73+
exit 1
74+
fi

0 commit comments

Comments
 (0)