Skip to content

Commit df634b9

Browse files
authored
Persist visual compare DOM snapshots (#1749)
1 parent 879ff49 commit df634b9

5 files changed

Lines changed: 172 additions & 0 deletions

File tree

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@
9494
"test:browser-visual-compare-fair-ratio": "tsx tests/browser-visual-compare-fair-ratio.test.ts",
9595
"test:browser-visual-compare-layout-drift": "tsx tests/browser-visual-compare-layout-drift.test.ts",
9696
"test:browser-visual-compare-capture-reliability": "tsx tests/browser-visual-compare-capture-reliability.test.ts",
97+
"test:browser-visual-compare-dom-snapshots": "npm run build && tsx tests/browser-visual-compare-dom-snapshots.test.ts",
9798
"test:browser-session-public-dto": "tsx tests/browser-session-public-dto.test.ts",
9899
"test:browser-playground-session-run": "tsx tests/browser-playground-session-run.test.ts",
99100
"test:browser-contained-site-status": "tsx tests/browser-contained-site-status.test.ts",

packages/runtime-playground/src/browser-artifacts.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ export interface BrowserArtifactFiles {
8585
actionCorpus?: string
8686
sourceScreenshot?: string | string[]
8787
candidateScreenshot?: string | string[]
88+
sourceDomSnapshot?: string | string[]
89+
candidateDomSnapshot?: string | string[]
8890
diffScreenshot?: string | string[]
8991
visualDiff?: string | string[]
9092
visualExplanation?: string | string[]
@@ -1081,6 +1083,8 @@ const BROWSER_ARTIFACT_FILE_MANIFEST: Record<keyof BrowserArtifactFiles, Browser
10811083
actionCorpus: { kind: "browser-action-corpus", contentType: "application/json", redact: true },
10821084
sourceScreenshot: { kind: "browser-visual-source-screenshot", contentType: "image/png", redact: false },
10831085
candidateScreenshot: { kind: "browser-visual-candidate-screenshot", contentType: "image/png", redact: false },
1086+
sourceDomSnapshot: { kind: "browser-visual-source-dom-snapshot", contentType: "application/json", redact: true },
1087+
candidateDomSnapshot: { kind: "browser-visual-candidate-dom-snapshot", contentType: "application/json", redact: true },
10841088
diffScreenshot: { kind: "browser-visual-diff-screenshot", contentType: "image/png", redact: false },
10851089
visualDiff: { kind: "browser-visual-diff", contentType: "application/json", redact: true },
10861090
visualExplanation: { kind: "browser-visual-explanation", contentType: "application/json", redact: true },

packages/runtime-playground/src/browser-visual-compare.ts

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,10 @@ async function runVisualComparePairCommand({
676676
const files = {
677677
sourceScreenshot: `${artifactPathPrefix}/source.png`,
678678
candidateScreenshot: `${artifactPathPrefix}/candidate.png`,
679+
...(sourceDomSnapshot && candidateDomSnapshot ? {
680+
sourceDomSnapshot: `${artifactPathPrefix}/source-dom-snapshot.json`,
681+
candidateDomSnapshot: `${artifactPathPrefix}/candidate-dom-snapshot.json`,
682+
} : {}),
679683
diffScreenshot: `${artifactPathPrefix}/diff.png`,
680684
visualDiff: `${artifactPathPrefix}/visual-diff.json`,
681685
...(explanation ? { visualExplanation: `${artifactPathPrefix}/visual-explanation.json` } : {}),
@@ -708,6 +712,22 @@ async function runVisualComparePairCommand({
708712
}
709713
const blocksEngineVisualParity = blocksEngineVisualParityReportFromVisualCompare({ ...summary, explanation })
710714
const summaryWithBlocksEngineVisualParity = { ...summary, blocksEngineVisualParity }
715+
if (sourceDomSnapshot && candidateDomSnapshot) {
716+
await artifactSession.writeJson("sourceDomSnapshot", "source-dom-snapshot.json", visualCompareDomSnapshotArtifact({
717+
screenshot: files.sourceScreenshot,
718+
finalUrl: finalSourceUrl ?? sourceDomSnapshot.url,
719+
viewport,
720+
maxElements: maxExplanationCandidates,
721+
snapshot: sourceDomSnapshot,
722+
}))
723+
await artifactSession.writeJson("candidateDomSnapshot", "candidate-dom-snapshot.json", visualCompareDomSnapshotArtifact({
724+
screenshot: files.candidateScreenshot,
725+
finalUrl: finalCandidateUrl ?? candidateDomSnapshot.url,
726+
viewport,
727+
maxElements: maxExplanationCandidates,
728+
snapshot: candidateDomSnapshot,
729+
}))
730+
}
711731
await artifactSession.writeJson("visualDiff", "visual-diff.json", summaryWithBlocksEngineVisualParity)
712732
if (explanation) {
713733
await artifactSession.writeJson("visualExplanation", "visual-explanation.json", explanation)
@@ -816,6 +836,8 @@ function visualCompareMatrixArtifact(
816836
const diffScreenshots = entries.map((entry) => entry.artifact.files.diffScreenshot).filter((file): file is string => typeof file === "string")
817837
const visualDiffs = entries.map((entry) => entry.artifact.files.visualDiff).filter((file): file is string => typeof file === "string")
818838
const visualExplanations = entries.map((entry) => entry.artifact.files.visualExplanation).filter((file): file is string => typeof file === "string")
839+
const sourceDomSnapshots = entries.map((entry) => entry.artifact.files.sourceDomSnapshot).filter((file): file is string => typeof file === "string")
840+
const candidateDomSnapshots = entries.map((entry) => entry.artifact.files.candidateDomSnapshot).filter((file): file is string => typeof file === "string")
819841
const firstArtifact = entries[0]?.artifact
820842
const captureDiagnostics = visualCompareMatrixCompactCaptureDiagnostics(matrixSummary)
821843
return {
@@ -829,6 +851,8 @@ function visualCompareMatrixArtifact(
829851
visualDiff: visualDiffs,
830852
sourceScreenshot: sourceScreenshots,
831853
candidateScreenshot: candidateScreenshots,
854+
...(sourceDomSnapshots.length > 0 ? { sourceDomSnapshot: sourceDomSnapshots } : {}),
855+
...(candidateDomSnapshots.length > 0 ? { candidateDomSnapshot: candidateDomSnapshots } : {}),
832856
diffScreenshot: diffScreenshots,
833857
...(visualExplanations.length > 0 ? { visualExplanation: visualExplanations } : {}),
834858
},
@@ -1590,6 +1614,30 @@ async function readVisualCompareDomSnapshotArtifact(requestedPath: string, artif
15901614
return artifact
15911615
}
15921616

1617+
function visualCompareDomSnapshotArtifact(input: {
1618+
screenshot: string
1619+
finalUrl: string
1620+
viewport: BrowserProbeViewport | null
1621+
maxElements: number
1622+
snapshot: VisualCompareDomSnapshot
1623+
}): VisualCompareDomSnapshotArtifact {
1624+
return {
1625+
schema: "wp-codebox/browser-dom-snapshot/v1",
1626+
command: "wordpress.visual-compare",
1627+
screenshot: input.screenshot,
1628+
finalUrl: input.finalUrl,
1629+
viewport: input.viewport,
1630+
capturedAt: now(),
1631+
limits: { maxElements: input.maxElements },
1632+
summary: {
1633+
elementCount: input.snapshot.elementCount,
1634+
capturedElements: input.snapshot.capturedElements.length,
1635+
truncated: input.snapshot.truncated,
1636+
},
1637+
snapshot: input.snapshot,
1638+
}
1639+
}
1640+
15931641
async function resolveVisualCompareArtifactPath(requestedPath: string, artifactRoot: string, label: string): Promise<string> {
15941642
try {
15951643
await access(requestedPath)

tests/browser-artifact-session.test.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,19 @@ assert.deepEqual(files.get("files/browser/screenshot.png")?.redaction, { policy:
5959
const visualSession = new BrowserArtifactSession(artifactRoot, "files/browser/visual-compare/mobile", { source: "wordpress.visual-compare", operation: "visual-compare" })
6060
await visualSession.writeJson("visualDiff", "visual-diff.json", { schema: "wp-codebox/visual-compare/v1", status: "different" })
6161
await visualSession.writeBuffer("sourceScreenshot", "source.png", Buffer.from([3, 4, 5]))
62+
await visualSession.writeJson("sourceDomSnapshot", "source-dom-snapshot.json", { schema: "wp-codebox/browser-dom-snapshot/v1" })
63+
await visualSession.writeJson("candidateDomSnapshot", "candidate-dom-snapshot.json", { schema: "wp-codebox/browser-dom-snapshot/v1" })
6264

6365
assert.equal(visualSession.path("/tmp/source.png"), "files/browser/visual-compare/mobile/source.png")
6466
const visualFiles = new Map(visualSession.writer.artifacts.files().map((file) => [file.path, file]))
6567
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/visual-diff.json")?.kind, "browser-visual-diff")
6668
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/visual-diff.json")?.redaction?.policy, "required")
6769
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/source.png")?.kind, "browser-visual-source-screenshot")
6870
assert.deepEqual(visualFiles.get("files/browser/visual-compare/mobile/source.png")?.redaction, { policy: "none", sensitive: false })
71+
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/source-dom-snapshot.json")?.kind, "browser-visual-source-dom-snapshot")
72+
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/source-dom-snapshot.json")?.contentType, "application/json")
73+
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/source-dom-snapshot.json")?.redaction?.policy, "required")
74+
assert.equal(visualFiles.get("files/browser/visual-compare/mobile/candidate-dom-snapshot.json")?.kind, "browser-visual-candidate-dom-snapshot")
6975

7076
const review = browserReviewSummary([{
7177
artifactType: "probe",
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import assert from "node:assert/strict"
2+
import { once } from "node:events"
3+
import { readFile, writeFile } from "node:fs/promises"
4+
import { createServer } from "node:http"
5+
import { join } from "node:path"
6+
7+
import { PNG } from "pngjs"
8+
9+
import { runVisualCompareCommand } from "../packages/runtime-playground/dist/browser-visual-compare.js"
10+
import { withTempDir } from "../scripts/test-kit.js"
11+
12+
const snapshot = (url: string, title: string) => ({
13+
url,
14+
title,
15+
elementCount: 1,
16+
capturedElements: [{ path: "main", tag: "main", text: title, attributes: {}, boundingBox: { x: 0, y: 0, width: 1, height: 1 }, styles: { display: "block" } }],
17+
truncated: false,
18+
})
19+
20+
const domSnapshotArtifact = (url: string, title: string) => ({
21+
schema: "wp-codebox/browser-dom-snapshot/v1" as const,
22+
command: "wordpress.browser-actions" as const,
23+
screenshot: "input.png",
24+
finalUrl: url,
25+
viewport: null,
26+
capturedAt: "2026-01-01T00:00:00.000Z",
27+
limits: { maxElements: 1 },
28+
summary: { elementCount: 1, capturedElements: 1, truncated: false },
29+
snapshot: snapshot(url, title),
30+
})
31+
32+
async function writePng(path: string): Promise<void> {
33+
const png = new PNG({ width: 1, height: 1 })
34+
png.data.set([255, 255, 255, 255])
35+
await writeFile(path, PNG.sync.write(png))
36+
}
37+
38+
async function visualCompare(artifactRoot: string, args: string[]) {
39+
return runVisualCompareCommand({
40+
artifactRoot,
41+
server: {
42+
serverUrl: "http://127.0.0.1:1",
43+
playground: { run: async () => ({ text: "" }) },
44+
async [Symbol.asyncDispose]() {},
45+
},
46+
spec: { command: "wordpress.visual-compare", args },
47+
})
48+
}
49+
50+
await withTempDir("wp-codebox-visual-dom-snapshots-", async (artifactRoot) => {
51+
const sourceScreenshot = join(artifactRoot, "input-source.png")
52+
const candidateScreenshot = join(artifactRoot, "input-candidate.png")
53+
const sourceSidecar = join(artifactRoot, "source-sidecar.json")
54+
const candidateSidecar = join(artifactRoot, "candidate-sidecar.json")
55+
await Promise.all([writePng(sourceScreenshot), writePng(candidateScreenshot)])
56+
await writeFile(sourceSidecar, JSON.stringify(domSnapshotArtifact("https://source.example.test/", "Source")))
57+
await writeFile(candidateSidecar, JSON.stringify(domSnapshotArtifact("https://candidate.example.test/", "Candidate")))
58+
59+
const supplied = await visualCompare(artifactRoot, [
60+
`source-screenshot=${sourceScreenshot}`,
61+
`candidate-screenshot=${candidateScreenshot}`,
62+
`source-dom-snapshot=${sourceSidecar}`,
63+
`candidate-dom-snapshot=${candidateSidecar}`,
64+
])
65+
const suppliedSummary = JSON.parse(supplied.output)
66+
assert.equal(suppliedSummary.schema, "wp-codebox/visual-compare/v1")
67+
assert.equal(suppliedSummary.files.sourceDomSnapshot, "files/browser/visual-compare/source-dom-snapshot.json")
68+
assert.equal(suppliedSummary.files.candidateDomSnapshot, "files/browser/visual-compare/candidate-dom-snapshot.json")
69+
const persistedSource = JSON.parse(await readFile(join(artifactRoot, suppliedSummary.files.sourceDomSnapshot), "utf8"))
70+
const persistedCandidate = JSON.parse(await readFile(join(artifactRoot, suppliedSummary.files.candidateDomSnapshot), "utf8"))
71+
assert.equal(persistedSource.schema, "wp-codebox/browser-dom-snapshot/v1")
72+
assert.equal(persistedSource.command, "wordpress.visual-compare")
73+
assert.equal(persistedSource.screenshot, "files/browser/visual-compare/source.png")
74+
assert.deepEqual(persistedSource.snapshot, snapshot("https://source.example.test/", "Source"))
75+
assert.equal(persistedCandidate.schema, "wp-codebox/browser-dom-snapshot/v1")
76+
assert.equal(persistedCandidate.screenshot, "files/browser/visual-compare/candidate.png")
77+
assert.equal(persistedCandidate.finalUrl, "https://candidate.example.test/")
78+
79+
await withTempDir("wp-codebox-visual-dom-snapshots-screenshot-only-", async (screenshotOnlyRoot) => {
80+
const screenshotOnly = await visualCompare(screenshotOnlyRoot, [`source-screenshot=${sourceScreenshot}`, `candidate-screenshot=${candidateScreenshot}`])
81+
const screenshotOnlySummary = JSON.parse(screenshotOnly.output)
82+
assert.equal("sourceDomSnapshot" in screenshotOnlySummary.files, false)
83+
assert.equal("candidateDomSnapshot" in screenshotOnlySummary.files, false)
84+
})
85+
})
86+
87+
const page = createServer((_request, response) => {
88+
response.writeHead(200, { "content-type": "text/html" })
89+
response.end("<!doctype html><title>URL snapshot</title><main>URL snapshot</main>")
90+
})
91+
page.listen(0, "127.0.0.1")
92+
await once(page, "listening")
93+
try {
94+
const address = page.address()
95+
if (!address || typeof address === "string") {
96+
throw new Error("test server did not expose a TCP address")
97+
}
98+
await withTempDir("wp-codebox-visual-dom-url-capture-", async (artifactRoot) => {
99+
const url = `http://127.0.0.1:${address.port}/`
100+
const result = await visualCompare(artifactRoot, [`source-url=${url}`, `candidate-url=${url}`])
101+
const summary = JSON.parse(result.output)
102+
const source = JSON.parse(await readFile(join(artifactRoot, summary.files.sourceDomSnapshot), "utf8"))
103+
const candidate = JSON.parse(await readFile(join(artifactRoot, summary.files.candidateDomSnapshot), "utf8"))
104+
assert.equal(source.schema, "wp-codebox/browser-dom-snapshot/v1")
105+
assert.equal(source.finalUrl, url)
106+
assert.equal(source.snapshot.title, "URL snapshot")
107+
assert.equal(candidate.snapshot.url, url)
108+
})
109+
} finally {
110+
page.close()
111+
}
112+
113+
console.log("browser visual compare DOM snapshots passed")

0 commit comments

Comments
 (0)