Skip to content

Commit 9966c75

Browse files
m-houstonm-houston
authored
CCM-19694: Config validation (#64)
* Extract config-store validation changes Cherry-pick the shared validation and validation-reporting work onto CCM-19694 without the supplier reports package. Includes config-store integrity validation, excel-parser mapping validation/reporting, and publisher error-detail propagation. * Enhance action.yml to resolve commit SHA to tag for release asset downloads * Add make target to run validation against local specifications.xlsx file * Clean up CLI tool output
1 parent e7faff3 commit 9966c75

24 files changed

Lines changed: 2068 additions & 120 deletions

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ internal-config:
3030
npm run parse --workspace=@supplier-config/excel-parser -- \
3131
"$(PWD)/specifications.xlsx" --output-dir "$(PWD)/artifacts/config-store" --pretty
3232

33+
validate-config:
34+
npm run parse --workspace=@supplier-config/excel-parser -- \
35+
"$(PWD)/specifications.xlsx" --check-mappings
3336
# ==============================================================================
3437

3538
${VERBOSE}.SILENT: \

‎actions/ddb-publish/action.yml‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,18 +59,46 @@ runs:
5959
exit 1
6060
fi
6161
62-
# Prefer release assets when the action ref is a tag with a matching release in the action repo.
63-
if gh release view "${ACTION_REF}" --repo "${ACTION_REPO}" >/dev/null 2>&1; then
64-
echo "[ddb-publish] Found release for ref '${ACTION_REF}' in '${ACTION_REPO}'. Downloading '${RELEASE_ASSET_NAME}'."
65-
gh release download "${ACTION_REF}" --repo "${ACTION_REPO}" --pattern "${RELEASE_ASSET_NAME}" --dir "${download_dir}"
62+
resolved_release_ref="${ACTION_REF}"
63+
run_list_scope_description=""
64+
run_list_scope_args=()
65+
66+
if [[ "${ACTION_REF}" =~ ^[0-9a-fA-F]{40}$ ]]; then
67+
echo "[ddb-publish] action_ref looks like a commit SHA. Attempting to resolve it to a tag before checking releases."
68+
69+
resolved_tag="$({
70+
gh api \
71+
--paginate \
72+
"repos/${ACTION_REPO}/tags?per_page=100" \
73+
--jq ".[] | select(.commit.sha == \"${ACTION_REF}\") | .name"
74+
} | head -n 1)"
75+
76+
if [[ -n "${resolved_tag}" ]]; then
77+
resolved_release_ref="${resolved_tag}"
78+
echo "[ddb-publish] Resolved commit '${ACTION_REF}' to tag '${resolved_release_ref}'."
79+
else
80+
echo "[ddb-publish] No tag matched commit '${ACTION_REF}'."
81+
fi
82+
83+
run_list_scope_description="commit '${ACTION_REF}'"
84+
run_list_scope_args=(--commit "${ACTION_REF}")
85+
else
86+
branch="${ACTION_REF#refs/heads/}"
87+
run_list_scope_description="branch '${branch}'"
88+
run_list_scope_args=(--branch "${branch}")
89+
fi
90+
91+
# Prefer release assets when the action ref is, or resolves to, a tag with a matching release in the action repo.
92+
if gh release view "${resolved_release_ref}" --repo "${ACTION_REPO}" >/dev/null 2>&1; then
93+
echo "[ddb-publish] Found release for ref '${resolved_release_ref}' in '${ACTION_REPO}'. Downloading '${RELEASE_ASSET_NAME}'."
94+
gh release download "${resolved_release_ref}" --repo "${ACTION_REPO}" --pattern "${RELEASE_ASSET_NAME}" --dir "${download_dir}"
6695
tar -xzf "${download_dir}/${RELEASE_ASSET_NAME}" -C "${unpack_dir}"
6796
echo "[ddb-publish] Bundle extracted from release asset."
6897
exit 0
6998
fi
7099
71-
# Otherwise treat the ref as a branch-like ref and fetch the latest successful CI artifact from the action repo.
72-
branch="${ACTION_REF#refs/heads/}"
73-
echo "[ddb-publish] No release found for ref '${ACTION_REF}'. Falling back to latest workflow artifact on branch '${branch}' from '${ACTION_REPO}'."
100+
# Otherwise fetch the latest successful CI artifact using the original branch or commit from the action repo.
101+
echo "[ddb-publish] No release found for ref '${resolved_release_ref}'. Falling back to latest workflow artifact on ${run_list_scope_description} from '${ACTION_REPO}'."
74102
75103
run_id=""
76104
workflow_used=""
@@ -82,7 +110,7 @@ runs:
82110
run_id_candidate="$(gh run list \
83111
--repo "${ACTION_REPO}" \
84112
--workflow "${workflow_file}" \
85-
--branch "${branch}" \
113+
"${run_list_scope_args[@]}" \
86114
--status success \
87115
--json databaseId \
88116
--jq '.[0].databaseId' 2>/tmp/ddb_publish_run_list_err.log)"
@@ -109,7 +137,7 @@ runs:
109137
done
110138
111139
if [[ -z "${run_id}" || "${run_id}" == "null" ]]; then
112-
echo "ERROR: Could not find a successful run on branch '${branch}' in '${ACTION_REPO}' containing artifact '${WORKFLOW_ARTIFACT_NAME}'. Checked workflows: stage-3-build.yaml, cicd-1-pull-request.yaml." >&2
140+
echo "ERROR: Could not find a successful run for ${run_list_scope_description} in '${ACTION_REPO}' containing artifact '${WORKFLOW_ARTIFACT_NAME}'. Checked workflows: stage-3-build.yaml, cicd-1-pull-request.yaml." >&2
113141
exit 1
114142
fi
115143

‎actions/eventbridge-publish/action.yml‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,18 +52,46 @@ runs:
5252
exit 1
5353
fi
5454
55-
# Prefer release assets when the action ref is a tag with a matching release in the action repo.
56-
if gh release view "${ACTION_REF}" --repo "${ACTION_REPO}" >/dev/null 2>&1; then
57-
echo "[eventbridge-publish] Found release for ref '${ACTION_REF}' in '${ACTION_REPO}'. Downloading '${RELEASE_ASSET_NAME}'."
58-
gh release download "${ACTION_REF}" --repo "${ACTION_REPO}" --pattern "${RELEASE_ASSET_NAME}" --dir "${download_dir}"
55+
resolved_release_ref="${ACTION_REF}"
56+
run_list_scope_description=""
57+
run_list_scope_args=()
58+
59+
if [[ "${ACTION_REF}" =~ ^[0-9a-fA-F]{40}$ ]]; then
60+
echo "[eventbridge-publish] action_ref looks like a commit SHA. Attempting to resolve it to a tag before checking releases."
61+
62+
resolved_tag="$({
63+
gh api \
64+
--paginate \
65+
"repos/${ACTION_REPO}/tags?per_page=100" \
66+
--jq ".[] | select(.commit.sha == \"${ACTION_REF}\") | .name"
67+
} | head -n 1)"
68+
69+
if [[ -n "${resolved_tag}" ]]; then
70+
resolved_release_ref="${resolved_tag}"
71+
echo "[eventbridge-publish] Resolved commit '${ACTION_REF}' to tag '${resolved_release_ref}'."
72+
else
73+
echo "[eventbridge-publish] No tag matched commit '${ACTION_REF}'."
74+
fi
75+
76+
run_list_scope_description="commit '${ACTION_REF}'"
77+
run_list_scope_args=(--commit "${ACTION_REF}")
78+
else
79+
branch="${ACTION_REF#refs/heads/}"
80+
run_list_scope_description="branch '${branch}'"
81+
run_list_scope_args=(--branch "${branch}")
82+
fi
83+
84+
# Prefer release assets when the action ref is, or resolves to, a tag with a matching release in the action repo.
85+
if gh release view "${resolved_release_ref}" --repo "${ACTION_REPO}" >/dev/null 2>&1; then
86+
echo "[eventbridge-publish] Found release for ref '${resolved_release_ref}' in '${ACTION_REPO}'. Downloading '${RELEASE_ASSET_NAME}'."
87+
gh release download "${resolved_release_ref}" --repo "${ACTION_REPO}" --pattern "${RELEASE_ASSET_NAME}" --dir "${download_dir}"
5988
tar -xzf "${download_dir}/${RELEASE_ASSET_NAME}" -C "${unpack_dir}"
6089
echo "[eventbridge-publish] Bundle extracted from release asset."
6190
exit 0
6291
fi
6392
64-
# Otherwise treat the ref as a branch-like ref and fetch the latest successful CI artifact from the action repo.
65-
branch="${ACTION_REF#refs/heads/}"
66-
echo "[eventbridge-publish] No release found for ref '${ACTION_REF}'. Falling back to latest workflow artifact on branch '${branch}' from '${ACTION_REPO}'."
93+
# Otherwise fetch the latest successful CI artifact using the original branch or commit from the action repo.
94+
echo "[eventbridge-publish] No release found for ref '${resolved_release_ref}'. Falling back to latest workflow artifact on ${run_list_scope_description} from '${ACTION_REPO}'."
6795
6896
run_id=""
6997
workflow_used=""
@@ -75,7 +103,7 @@ runs:
75103
run_id_candidate="$(gh run list \
76104
--repo "${ACTION_REPO}" \
77105
--workflow "${workflow_file}" \
78-
--branch "${branch}" \
106+
"${run_list_scope_args[@]}" \
79107
--status success \
80108
--json databaseId \
81109
--jq '.[0].databaseId' 2>/tmp/eventbridge_publish_run_list_err.log)"
@@ -102,7 +130,7 @@ runs:
102130
done
103131
104132
if [[ -z "${run_id}" || "${run_id}" == "null" ]]; then
105-
echo "ERROR: Could not find a successful run on branch '${branch}' in '${ACTION_REPO}' containing artifact '${WORKFLOW_ARTIFACT_NAME}'. Checked workflows: stage-3-build.yaml, cicd-1-pull-request.yaml." >&2
133+
echo "ERROR: Could not find a successful run for ${run_list_scope_description} in '${ACTION_REPO}' containing artifact '${WORKFLOW_ARTIFACT_NAME}'. Checked workflows: stage-3-build.yaml, cicd-1-pull-request.yaml." >&2
106134
exit 1
107135
fi
108136

‎packages/ddb-publisher/src/__tests__/run.test.ts‎

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ jest.mock("@supplier-config/file-store", () => {
99
return {
1010
loadConfigStore: jest.fn(),
1111
validateConfigStore: jest.fn(),
12+
validateConfigStoreIntegrity: jest.fn(),
1213
};
1314
});
1415

@@ -43,6 +44,7 @@ const fileStore = jest.requireMock(
4344
) as unknown as {
4445
loadConfigStore: jest.Mock;
4546
validateConfigStore: jest.Mock;
47+
validateConfigStoreIntegrity: jest.Mock;
4648
};
4749

4850
const audit = jest.requireMock("../ddb/audit") as unknown as {
@@ -58,13 +60,17 @@ const awsClient = jest.requireMock("@aws-sdk/client-dynamodb") as unknown as {
5860
};
5961

6062
describe("runPublisher", () => {
63+
beforeEach(() => {
64+
const validResult: ValidationResult = { ok: true, issues: [] };
65+
66+
fileStore.validateConfigStore.mockReturnValue(validResult);
67+
fileStore.validateConfigStoreIntegrity.mockReturnValue(validResult);
68+
});
69+
6170
it("should stop after validation when dryRun=true", async () => {
6271
const store: LoadedConfigStore = { rootPath: "/tmp", records: [] };
6372
fileStore.loadConfigStore.mockResolvedValue(store);
6473

65-
const validation: ValidationResult = { ok: true, issues: [] };
66-
fileStore.validateConfigStore.mockReturnValue(validation);
67-
6874
await runPublisher({
6975
sourcePath: "/tmp",
7076
env: "draft",
@@ -107,6 +113,72 @@ describe("runPublisher", () => {
107113
).rejects.toThrow("Config store validation failed");
108114
});
109115

116+
it("should throw with a helpful message when integrity validation fails", async () => {
117+
const store: LoadedConfigStore = {
118+
rootPath: "/tmp",
119+
records: [],
120+
};
121+
fileStore.loadConfigStore.mockResolvedValue(store);
122+
123+
fileStore.validateConfigStoreIntegrity.mockReturnValue({
124+
ok: false,
125+
issues: [
126+
{
127+
entity: "letter-variant",
128+
sourceFilePath: "/tmp/letter-variant/variant-1.json",
129+
message: "missing promoted supplier pack",
130+
path: ["packSpecificationIds"],
131+
},
132+
],
133+
});
134+
135+
await expect(
136+
runPublisher({
137+
sourcePath: "/tmp",
138+
env: "draft",
139+
tableName: "tbl",
140+
dryRun: true,
141+
force: false,
142+
}),
143+
).rejects.toThrow("Config store integrity validation failed");
144+
});
145+
146+
it("should include issue detail lines in integrity validation errors", async () => {
147+
const store: LoadedConfigStore = {
148+
rootPath: "/tmp",
149+
records: [],
150+
};
151+
fileStore.loadConfigStore.mockResolvedValue(store);
152+
153+
fileStore.validateConfigStoreIntegrity.mockReturnValue({
154+
ok: false,
155+
issues: [
156+
{
157+
entity: "letter-variant",
158+
sourceFilePath: "/tmp/letter-variant/variant-1.json",
159+
message: "missing promoted supplier pack",
160+
path: ["packSpecificationIds"],
161+
details: [
162+
"packSpecification=pack-1 status=PROD => valid",
163+
"supplierPack=supplier-pack-1 supplier=supplier-1 status=INT approval=APPROVED => invalid",
164+
],
165+
},
166+
],
167+
});
168+
169+
await expect(
170+
runPublisher({
171+
sourcePath: "/tmp",
172+
env: "draft",
173+
tableName: "tbl",
174+
dryRun: true,
175+
force: false,
176+
}),
177+
).rejects.toThrow(
178+
"supplierPack=supplier-pack-1 supplier=supplier-1 status=INT approval=APPROVED => invalid",
179+
);
180+
});
181+
110182
it("should block upload when audit reports blocking items and force=false", async () => {
111183
const store: LoadedConfigStore = {
112184
rootPath: "/tmp",

‎packages/ddb-publisher/src/run.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb";
44
import {
55
loadConfigStore,
66
validateConfigStore,
7+
validateConfigStoreIntegrity,
78
} from "@supplier-config/file-store";
89
import type {
910
ConfigRecord,
@@ -19,9 +20,13 @@ function issueLabel(i: {
1920
sourceFilePath: string;
2021
path?: (string | number)[];
2122
message: string;
23+
details?: string[];
2224
}): string {
2325
const pathPart = i.path?.length ? `:${i.path.join(".")}` : "";
24-
return `${i.entity} ${i.sourceFilePath}${pathPart} - ${i.message}`;
26+
const formattedDetails = i.details?.map((detail) => ` ${detail}`).join("\n");
27+
const detailsPart = i.details?.length ? `\n${formattedDetails}` : "";
28+
29+
return `${i.entity} ${i.sourceFilePath}${pathPart} - ${i.message}${detailsPart}`;
2530
}
2631

2732
function logStep(message: string): void {
@@ -81,6 +86,26 @@ async function runPublisher(plan: LoadPlan): Promise<void> {
8186

8287
logStep("Validation passed.");
8388

89+
logStep("Running config-store integrity checks...");
90+
const integrityValidation = validateConfigStoreIntegrity(store);
91+
92+
if (!integrityValidation.ok) {
93+
logStep(
94+
`Integrity validation failed with ${integrityValidation.issues.length} issue(s).`,
95+
);
96+
97+
const summary = integrityValidation.issues
98+
.slice(0, 20)
99+
.map((i: ValidationIssue) => issueLabel(i))
100+
.join("\n");
101+
102+
throw new Error(
103+
`Config store integrity validation failed with ${integrityValidation.issues.length} issue(s).\n${summary}`,
104+
);
105+
}
106+
107+
logStep("Integrity validation passed.");
108+
84109
if (plan.dryRun) {
85110
logStep("Dry-run enabled; skipping DynamoDB audit and publish.");
86111
return;

0 commit comments

Comments
 (0)