Skip to content

Commit 3906f41

Browse files
emma-sgemma-sg
committed
first pass at tests
1 parent 2259431 commit 3906f41

4 files changed

Lines changed: 378 additions & 0 deletions

File tree

‎tests/qa_compare.test.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,11 @@ test("run QA comparison without CSR detection, with write pages to redis", async
7777

7878
expect(json.comparison).toHaveProperty("screenshotMatch");
7979
expect(json.comparison).toHaveProperty("textMatch");
80+
expect(json.comparison).toHaveProperty("rawTextMatch");
8081
expect(json.comparison).toHaveProperty("resourceCounts");
8182
expect(json.comparison.screenshotMatch).toBe(1);
8283
expect(json.comparison.textMatch).toBe(1);
84+
expect(typeof json.comparison.rawTextMatch).toBe("number");
8385

8486
expect(json.comparison.resourceCounts).toHaveProperty("crawlGood");
8587
expect(json.comparison.resourceCounts).toHaveProperty("crawlBad");
@@ -156,9 +158,11 @@ test("run QA comparison with CSR detection, with write pages to redis", async ()
156158

157159
expect(json.comparison).toHaveProperty("screenshotMatch");
158160
expect(json.comparison).toHaveProperty("textMatch");
161+
expect(json.comparison).toHaveProperty("rawTextMatch");
159162
expect(json.comparison).toHaveProperty("resourceCounts");
160163
expect(json.comparison.screenshotMatch).toBe(1);
161164
expect(json.comparison.textMatch).toBe(1);
165+
expect(typeof json.comparison.rawTextMatch).toBe("number");
162166

163167
expect(json.comparison.resourceCounts).toHaveProperty("crawlGood");
164168
expect(json.comparison.resourceCounts).toHaveProperty("crawlBad");

‎tests/raw-text-extract.test.js‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
import child_process from "child_process";
2+
import fs from "fs";
3+
import { Redis } from "ioredis";
4+
5+
/**
6+
*
7+
* @param {number} ms
8+
* @returns {Promise<void>}
9+
*/
10+
const sleep = (ms) => new Promise((res) => setTimeout(res, ms));
11+
12+
/**
13+
*
14+
* @param {child_process.ChildProcess} child
15+
* @returns {Promise<string>}
16+
*/
17+
const asPromise = (child) =>
18+
new Promise(function (resolve, reject) {
19+
child.addListener("error", reject);
20+
child.addListener("exit", resolve);
21+
});
22+
23+
test("run crawl with to-warc-from-raw option and verify text records", async () => {
24+
fs.rmSync("./test-crawls/raw-text-test", { recursive: true, force: true });
25+
26+
child_process.execSync(
27+
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --scopeType page --collection raw-text-test --text to-warc,to-warc-from-raw --generateCDX",
28+
);
29+
30+
const cdxData = fs.readFileSync(
31+
"test-crawls/collections/raw-text-test/indexes/index.cdxj",
32+
{ encoding: "utf-8" },
33+
);
34+
35+
// Check that both regular text and raw text records exist
36+
expect(cdxData.indexOf("urn:text:https://old.webrecorder.net/") >= 0).toBe(true);
37+
expect(cdxData.indexOf("urn:text-from-response:https://old.webrecorder.net/") >= 0).toBe(true);
38+
});
39+
40+
test("verify raw text and rendered text are different for CSR pages", async () => {
41+
fs.rmSync("./test-crawls/csr-test", { recursive: true, force: true });
42+
43+
// Crawl a page known to have client-side rendering
44+
child_process.execSync(
45+
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --url https://old.webrecorder.net/ --scopeType page --collection csr-test --text to-warc,to-warc-from-raw --generateWACZ",
46+
);
47+
48+
expect(
49+
fs.existsSync("test-crawls/collections/csr-test/csr-test.wacz"),
50+
).toBe(true);
51+
});
52+
53+
test("run QA comparison with raw text match", async () => {
54+
fs.rmSync("./test-crawls/qa-raw-text-replay", { recursive: true, force: true });
55+
56+
const child = child_process.exec(
57+
"docker run -p 36381:6379 -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler qa --qaSource /crawls/collections/raw-text-test/raw-text-test.wacz --collection qa-raw-text-replay --crawlId raw-text-test --writePagesToRedis --debugAccessRedis",
58+
);
59+
60+
// detect crawler exit
61+
let crawler_exited = false;
62+
child.on("exit", function () {
63+
crawler_exited = true;
64+
});
65+
const crawlerExitedPromise = asPromise(child);
66+
67+
const redis = new Redis("redis://127.0.0.1:36381/0", {
68+
lazyConnect: true,
69+
retryStrategy: () => null,
70+
});
71+
72+
await sleep(3000);
73+
74+
await redis.connect({ maxRetriesPerRequest: 50 });
75+
76+
let count = 0;
77+
78+
while (count < 1) {
79+
const res = await redis.lpop("raw-text-test:pages");
80+
if (!res) {
81+
if (crawler_exited) {
82+
break;
83+
}
84+
await sleep(100);
85+
continue;
86+
}
87+
const json = JSON.parse(res);
88+
expect(json).toHaveProperty("id");
89+
expect(json).toHaveProperty("url");
90+
expect(json).toHaveProperty("comparison");
91+
92+
// Check that raw text comparison is present
93+
expect(json.comparison).toHaveProperty("rawTextMatch");
94+
expect(typeof json.comparison.rawTextMatch).toBe("number");
95+
expect(json.comparison.rawTextMatch).toBeGreaterThanOrEqual(0);
96+
expect(json.comparison.rawTextMatch).toBeLessThanOrEqual(1);
97+
98+
// Check that both regular and raw text comparisons exist
99+
expect(json.comparison).toHaveProperty("textMatch");
100+
expect(json.comparison).toHaveProperty("screenshotMatch");
101+
102+
count++;
103+
}
104+
105+
expect(count).toBe(1);
106+
107+
// wait for crawler exit
108+
await crawlerExitedPromise;
109+
});
110+
111+
test("run QA comparison with fallback to on-the-fly raw text extraction", async () => {
112+
// Use the original qa-wr-net archive which doesn't have stored raw text
113+
fs.rmSync("./test-crawls/qa-fallback-test", { recursive: true, force: true });
114+
115+
const child = child_process.exec(
116+
"docker run -p 36382:6379 -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler qa --qaSource /crawls/collections/qa-wr-net/qa-wr-net.wacz --collection qa-fallback-test --crawlId fallback-test --writePagesToRedis --debugAccessRedis --exclude contact",
117+
);
118+
119+
// detect crawler exit
120+
let crawler_exited = false;
121+
child.on("exit", function () {
122+
crawler_exited = true;
123+
});
124+
const crawlerExitedPromise = asPromise(child);
125+
126+
const redis = new Redis("redis://127.0.0.1:36382/0", {
127+
lazyConnect: true,
128+
retryStrategy: () => null,
129+
});
130+
131+
await sleep(3000);
132+
133+
await redis.connect({ maxRetriesPerRequest: 50 });
134+
135+
let count = 0;
136+
137+
while (count < 3) {
138+
const res = await redis.lpop("fallback-test:pages");
139+
if (!res) {
140+
if (crawler_exited) {
141+
break;
142+
}
143+
await sleep(100);
144+
continue;
145+
}
146+
const json = JSON.parse(res);
147+
expect(json).toHaveProperty("id");
148+
expect(json).toHaveProperty("url");
149+
expect(json).toHaveProperty("comparison");
150+
151+
// Check that raw text comparison works even without stored raw text
152+
expect(json.comparison).toHaveProperty("rawTextMatch");
153+
expect(typeof json.comparison.rawTextMatch).toBe("number");
154+
155+
count++;
156+
}
157+
158+
expect(count).toBe(3);
159+
160+
// wait for crawler exit
161+
await crawlerExitedPromise;
162+
});

‎tests/text-extract.test.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,26 @@ test("check that urn:text and urn:textfinal records are written to WARC", async
2020

2121
expect(data.indexOf("urn:textFinal:https://www.nytimes.com/") > 0).toBe(true);
2222
});
23+
24+
test("check that raw text extraction creates urn:text-from-response records", async () => {
25+
fs.rmSync("./test-crawls/raw-text-crawl", { recursive: true, force: true });
26+
27+
try {
28+
child_process.execSync(
29+
"docker run -v $PWD/test-crawls:/crawls webrecorder/browsertrix-crawler crawl --collection raw-text-crawl --url https://example.com/ --scopeType page --generateCDX --text to-warc,to-warc-from-raw",
30+
);
31+
} catch (error) {
32+
console.log(error.stderr);
33+
}
34+
35+
const data = fs.readFileSync(
36+
"test-crawls/collections/raw-text-crawl/indexes/index.cdxj",
37+
{ encoding: "utf-8" },
38+
);
39+
40+
// Regular rendered text record
41+
expect(data.indexOf("urn:text:https://example.com/") >= 0).toBe(true);
42+
43+
// Raw response text record
44+
expect(data.indexOf("urn:text-from-response:https://example.com/") >= 0).toBe(true);
45+
});

0 commit comments

Comments
 (0)