Skip to content

Commit b73183a

Browse files
authored
fix(tests): prepare AgentPlayground CI cutover (#1560)
Add fail-closed trigger mirror synchronization, align smoke failure reporting with deployed ADO names and the staging issue repository, and update the operational handoff. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0fe13dc9-ae18-4e54-b587-a53f5afb28f2
1 parent a59eda0 commit b73183a

13 files changed

Lines changed: 586 additions & 125 deletions

File tree

.gitattributes

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,32 +16,9 @@ tests/fixtures/runtime_imports_author_marker_stage.lock.yml linguist-generated=t
1616
tests/fixtures/runtime_imports_job.lock.yml linguist-generated=true merge=ours text eol=lf
1717
tests/fixtures/runtime_imports_stage.lock.yml linguist-generated=true merge=ours text eol=lf
1818
tests/fixtures/stage-agent.lock.yml linguist-generated=true merge=ours text eol=lf
19-
tests/safe-outputs/add-build-tag.lock.yml linguist-generated=true merge=ours text eol=lf
20-
tests/safe-outputs/add-pr-comment.lock.yml linguist-generated=true merge=ours text eol=lf
2119
tests/safe-outputs/azure-cli.lock.yml linguist-generated=true merge=ours text eol=lf
2220
tests/safe-outputs/canary.lock.yml linguist-generated=true merge=ours text eol=lf
23-
tests/safe-outputs/comment-on-work-item.lock.yml linguist-generated=true merge=ours text eol=lf
24-
tests/safe-outputs/create-branch.lock.yml linguist-generated=true merge=ours text eol=lf
25-
tests/safe-outputs/create-git-tag.lock.yml linguist-generated=true merge=ours text eol=lf
26-
tests/safe-outputs/create-pull-request.lock.yml linguist-generated=true merge=ours text eol=lf
27-
tests/safe-outputs/create-wiki-page.lock.yml linguist-generated=true merge=ours text eol=lf
28-
tests/safe-outputs/create-work-item.lock.yml linguist-generated=true merge=ours text eol=lf
2921
tests/safe-outputs/janitor.lock.yml linguist-generated=true merge=ours text eol=lf
30-
tests/safe-outputs/link-work-items.lock.yml linguist-generated=true merge=ours text eol=lf
31-
tests/safe-outputs/missing-data.lock.yml linguist-generated=true merge=ours text eol=lf
32-
tests/safe-outputs/missing-tool.lock.yml linguist-generated=true merge=ours text eol=lf
3322
tests/safe-outputs/noop-target.lock.yml linguist-generated=true merge=ours text eol=lf
34-
tests/safe-outputs/noop.lock.yml linguist-generated=true merge=ours text eol=lf
35-
tests/safe-outputs/queue-build.lock.yml linguist-generated=true merge=ours text eol=lf
36-
tests/safe-outputs/reply-to-pr-comment.lock.yml linguist-generated=true merge=ours text eol=lf
37-
tests/safe-outputs/report-incomplete.lock.yml linguist-generated=true merge=ours text eol=lf
38-
tests/safe-outputs/resolve-pr-thread.lock.yml linguist-generated=true merge=ours text eol=lf
3923
tests/safe-outputs/smoke-failure-reporter.lock.yml linguist-generated=true merge=ours text eol=lf
40-
tests/safe-outputs/submit-pr-review.lock.yml linguist-generated=true merge=ours text eol=lf
41-
tests/safe-outputs/update-pr.lock.yml linguist-generated=true merge=ours text eol=lf
42-
tests/safe-outputs/update-wiki-page.lock.yml linguist-generated=true merge=ours text eol=lf
43-
tests/safe-outputs/update-work-item.lock.yml linguist-generated=true merge=ours text eol=lf
44-
tests/safe-outputs/upload-build-attachment.lock.yml linguist-generated=true merge=ours text eol=lf
45-
tests/safe-outputs/upload-pipeline-artifact.lock.yml linguist-generated=true merge=ours text eol=lf
46-
tests/safe-outputs/upload-workitem-attachment.lock.yml linguist-generated=true merge=ours text eol=lf
4724
# END ado-aw managed
Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
import { describe, expect, it, vi } from "vitest";
2+
3+
import {
4+
loadMirrorSyncConfig,
5+
mirrorRepoUrl,
6+
runMirrorSyncPreflight,
7+
type MirrorGitRequest,
8+
type MirrorGitResult,
9+
type MirrorGitRunner,
10+
} from "../mirror.js";
11+
12+
const HEAD = "1111111111111111111111111111111111111111";
13+
14+
function baseEnv(): NodeJS.ProcessEnv {
15+
return {
16+
TRIGGER_E2E_SYNC_MIRROR: "true",
17+
TRIGGER_E2E_VICTIM_REPO: "ado-aw-mirror",
18+
SYSTEM_COLLECTIONURI: "https://dev.azure.com/msazuresphere/",
19+
SYSTEM_TEAMPROJECT: "Agent Playground",
20+
SYSTEM_ACCESSTOKEN: "secret-token",
21+
BUILD_SOURCESDIRECTORY: "/src",
22+
BUILD_SOURCEBRANCH: "refs/heads/main",
23+
BUILD_SOURCEVERSION: HEAD,
24+
};
25+
}
26+
27+
function successRunner(calls: MirrorGitRequest[]): MirrorGitRunner {
28+
return async (request): Promise<MirrorGitResult> => {
29+
calls.push(request);
30+
if (request.args[0] === "rev-parse" && request.args[1] === "--is-shallow-repository") {
31+
return { status: 0, stdout: "false\n", stderr: "" };
32+
}
33+
if (request.args[0] === "rev-parse") {
34+
return { status: 0, stdout: `${HEAD}\n`, stderr: "" };
35+
}
36+
if (request.args[0] === "ls-remote") {
37+
return { status: 0, stdout: `${HEAD}\trefs/heads/main\n`, stderr: "" };
38+
}
39+
return { status: 0, stdout: "", stderr: "" };
40+
};
41+
}
42+
43+
describe("trigger mirror sync", () => {
44+
it("is disabled unless explicitly opted in", async () => {
45+
const runner = vi.fn<MirrorGitRunner>();
46+
const result = await runMirrorSyncPreflight({}, () => {}, runner);
47+
expect(result).toBeUndefined();
48+
expect(runner).not.toHaveBeenCalled();
49+
});
50+
51+
it("builds an encoded ADO Git URL", () => {
52+
expect(
53+
mirrorRepoUrl(
54+
"https://dev.azure.com/msazuresphere/",
55+
"Agent Playground",
56+
"ado aw/mirror",
57+
),
58+
).toBe(
59+
"https://dev.azure.com/msazuresphere/Agent%20Playground/_git/ado%20aw%2Fmirror",
60+
);
61+
});
62+
63+
it("pushes main fast-forward-only and verifies the remote SHA", async () => {
64+
const calls: MirrorGitRequest[] = [];
65+
const result = await runMirrorSyncPreflight(baseEnv(), () => {}, successRunner(calls));
66+
67+
expect(result?.ok).toBe(true);
68+
expect(calls.map((call) => call.args[0])).toEqual([
69+
"rev-parse",
70+
"rev-parse",
71+
"push",
72+
"ls-remote",
73+
]);
74+
const push = calls[2];
75+
expect(push?.args).toContain("HEAD:refs/heads/main");
76+
expect(push?.args).not.toContain("--force");
77+
expect(push?.args.join(" ")).not.toContain("secret-token");
78+
expect(push?.env.GIT_CONFIG_VALUE_0).toBe("Authorization: bearer secret-token");
79+
expect(push?.cwd).toBe("/src");
80+
});
81+
82+
it("rejects a non-main orchestrator checkout before invoking git", async () => {
83+
const runner = vi.fn<MirrorGitRunner>();
84+
const result = await runMirrorSyncPreflight(
85+
{ ...baseEnv(), BUILD_SOURCEBRANCH: "refs/heads/feature" },
86+
() => {},
87+
runner,
88+
);
89+
90+
expect(result?.ok).toBe(false);
91+
expect(result?.message).toContain("refs/heads/main");
92+
expect(runner).not.toHaveBeenCalled();
93+
});
94+
95+
it("rejects a shallow checkout", async () => {
96+
const runner: MirrorGitRunner = async () => ({
97+
status: 0,
98+
stdout: "true\n",
99+
stderr: "",
100+
});
101+
const result = await runMirrorSyncPreflight(baseEnv(), () => {}, runner);
102+
103+
expect(result?.ok).toBe(false);
104+
expect(result?.message).toContain("fetchDepth: 0");
105+
});
106+
107+
it("fails closed on a non-fast-forward push", async () => {
108+
const calls: MirrorGitRequest[] = [];
109+
const runner: MirrorGitRunner = async (request) => {
110+
calls.push(request);
111+
if (request.args[0] === "rev-parse" && request.args[1] === "--is-shallow-repository") {
112+
return { status: 0, stdout: "false\n", stderr: "" };
113+
}
114+
if (request.args[0] === "rev-parse") {
115+
return { status: 0, stdout: `${HEAD}\n`, stderr: "" };
116+
}
117+
return { status: 1, stdout: "", stderr: "! [rejected] non-fast-forward" };
118+
};
119+
120+
const result = await runMirrorSyncPreflight(baseEnv(), () => {}, runner);
121+
expect(result?.ok).toBe(false);
122+
expect(result?.message).toContain("non-fast-forward");
123+
expect(calls.some((call) => call.args[0] === "ls-remote")).toBe(false);
124+
});
125+
126+
it("fails when the verified remote SHA differs", async () => {
127+
const runner: MirrorGitRunner = async (request) => {
128+
if (request.args[0] === "rev-parse" && request.args[1] === "--is-shallow-repository") {
129+
return { status: 0, stdout: "false\n", stderr: "" };
130+
}
131+
if (request.args[0] === "rev-parse") {
132+
return { status: 0, stdout: `${HEAD}\n`, stderr: "" };
133+
}
134+
if (request.args[0] === "ls-remote") {
135+
return {
136+
status: 0,
137+
stdout: "2222222222222222222222222222222222222222\trefs/heads/main\n",
138+
stderr: "",
139+
};
140+
}
141+
return { status: 0, stdout: "", stderr: "" };
142+
};
143+
144+
const result = await runMirrorSyncPreflight(baseEnv(), () => {}, runner);
145+
expect(result?.ok).toBe(false);
146+
expect(result?.message).toContain("mirror verification failed");
147+
});
148+
149+
it("preserves the bypass-only baseline when the victim repo is unset", async () => {
150+
const runner = vi.fn<MirrorGitRunner>();
151+
const logs: string[] = [];
152+
const env = {
153+
...baseEnv(),
154+
TRIGGER_E2E_VICTIM_REPO: "",
155+
};
156+
157+
expect(loadMirrorSyncConfig(env)).toBeUndefined();
158+
expect(await runMirrorSyncPreflight(env, (message) => logs.push(message), runner)).toBeUndefined();
159+
expect(logs).toContain(
160+
"[mirror-sync] skipped: TRIGGER_E2E_VICTIM_REPO is not configured",
161+
);
162+
expect(runner).not.toHaveBeenCalled();
163+
});
164+
});

scripts/ado-script/src/trigger-e2e/index.ts

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
*/
2626
import { AdoRest } from "../executor-e2e/ado-rest.js";
2727
import { fileFailureIssue, loadIssueEnv } from "./github-issue.js";
28+
import { runMirrorSyncPreflight } from "./mirror.js";
2829
import { runAll } from "./runner.js";
2930
import { allScenarios } from "./scenarios/index.js";
3031
import type { ScenarioResult, TriggerContext } from "./scenario.js";
@@ -87,12 +88,18 @@ export async function main(): Promise<number> {
8788
prefix: (id) => `ado-aw-trig-${buildId}-${id}`,
8889
};
8990

90-
log(
91-
`Running ${allScenarios.length} trigger E2E scenarios against ${orgUrl}${project} ` +
92-
`(victim def #${victimDefinitionId}${adoRepo ? `, repo ${adoRepo}` : ", no PR repo — PR scenarios will skip"})`,
93-
);
94-
95-
const results = await runAll(ctx, allScenarios);
91+
const preflight = await runMirrorSyncPreflight(process.env, log);
92+
let results: ScenarioResult[];
93+
if (preflight && !preflight.ok) {
94+
results = [preflight];
95+
} else {
96+
log(
97+
`Running ${allScenarios.length} trigger E2E scenarios against ${orgUrl}${project} ` +
98+
`(victim def #${victimDefinitionId}${adoRepo ? `, repo ${adoRepo}` : ", no PR repo — PR scenarios will skip"})`,
99+
);
100+
const scenarioResults = await runAll(ctx, allScenarios);
101+
results = preflight ? [preflight, ...scenarioResults] : scenarioResults;
102+
}
96103
log(summarise(results));
97104

98105
const issueEnv = loadIssueEnv();

0 commit comments

Comments
 (0)