Skip to content

Commit a83b8da

Browse files
committed
fix site crawler checks
1 parent dda4a35 commit a83b8da

8 files changed

Lines changed: 136 additions & 3 deletions

File tree

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
"lint": "oxlint . --config oxlint.json",
2020
"format": "oxfmt --write .",
2121
"format:check": "oxfmt --check .",
22+
"website:smoke": "node scripts/website-smoke.mjs",
2223
"test": "vitest run",
2324
"pack:smoke": "node scripts/package-smoke.mjs"
2425
},

scripts/website-smoke.mjs

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import { readFile, stat } from "node:fs/promises";
2+
import { join } from "node:path";
3+
4+
const root = process.cwd();
5+
const website = join(root, "website");
6+
const failures = [];
7+
8+
function fail(message) {
9+
failures.push(message);
10+
}
11+
12+
async function mustRead(relativePath) {
13+
try {
14+
return await readFile(join(root, relativePath), "utf8");
15+
} catch {
16+
fail(`missing ${relativePath}`);
17+
return "";
18+
}
19+
}
20+
21+
function stripTags(value) {
22+
return value
23+
.replace(/<br\s*\/?>/giu, " ")
24+
.replace(/<[^>]+>/gu, "")
25+
.replace(/\s+/gu, " ")
26+
.trim();
27+
}
28+
29+
function extractFirst(html, pattern, label) {
30+
const match = html.match(pattern);
31+
if (!match) {
32+
fail(`missing ${label}`);
33+
return "";
34+
}
35+
return match[1] || "";
36+
}
37+
38+
const html = await mustRead("website/index.html");
39+
const robots = await mustRead("website/robots.txt");
40+
const sitemap = await mustRead("website/sitemap.xml");
41+
const headers = await mustRead("website/_headers");
42+
43+
const title = stripTags(extractFirst(html, /<title>([\s\S]*?)<\/title>/iu, "title"));
44+
if (title !== "Clawpatch — Automated Code Review") {
45+
fail(`unexpected title: ${title}`);
46+
}
47+
48+
const description = html.match(/<meta\s+name="description"\s+content="([^"]+)"/iu)?.[1] || "";
49+
if (!description.includes("Automated code review that lands fixes")) {
50+
fail("meta description does not contain the product promise");
51+
}
52+
53+
const h1 = stripTags(extractFirst(html, /<h1>([\s\S]*?)<\/h1>/iu, "h1"));
54+
if (h1 !== "Code review with explicit fixes") {
55+
fail(`unexpected h1 text: ${h1}`);
56+
}
57+
58+
const ids = new Set([...html.matchAll(/\sid="([^"]+)"/giu)].map((match) => match[1]));
59+
const anchorLinks = [...html.matchAll(/href="#([^"]+)"/giu)].map((match) => match[1]);
60+
for (const id of anchorLinks) {
61+
if (!ids.has(id)) fail(`missing anchor target: #${id}`);
62+
}
63+
64+
if (!robots.includes("Sitemap: https://clawpatch.ai/sitemap.xml")) {
65+
fail("robots.txt missing sitemap reference");
66+
}
67+
68+
if (!sitemap.includes("<loc>https://clawpatch.ai/</loc>")) {
69+
fail("sitemap.xml missing canonical homepage loc");
70+
}
71+
72+
for (const expectedHeader of [
73+
"Strict-Transport-Security",
74+
"X-Content-Type-Options",
75+
"X-Frame-Options",
76+
"Referrer-Policy",
77+
"Permissions-Policy",
78+
"Content-Security-Policy",
79+
]) {
80+
if (!headers.includes(expectedHeader)) {
81+
fail(`_headers missing ${expectedHeader}`);
82+
}
83+
}
84+
85+
const socialCard = await readFile(join(website, "social-card.png"));
86+
if (socialCard.toString("ascii", 1, 4) !== "PNG") {
87+
fail("social-card.png is not a PNG");
88+
} else {
89+
const width = socialCard.readUInt32BE(16);
90+
const height = socialCard.readUInt32BE(20);
91+
if (width !== 1200 || height !== 630) {
92+
fail(`social-card.png dimensions are ${width}x${height}, expected 1200x630`);
93+
}
94+
}
95+
96+
for (const file of ["website/favicon.svg", "website/CNAME", "website/.nojekyll"]) {
97+
try {
98+
await stat(join(root, file));
99+
} catch {
100+
fail(`missing ${file}`);
101+
}
102+
}
103+
104+
if (failures.length) {
105+
console.error(failures.join("\n"));
106+
process.exit(1);
107+
}
108+
109+
console.log("Website smoke checks passed.");

src/exec.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ describe("runCommandArgs", () => {
9292
"import { writeFileSync } from 'node:fs';",
9393
"process.on('SIGTERM', () => {});",
9494
"process.send?.('ready');",
95-
`setTimeout(() => writeFileSync(${JSON.stringify(marker)}, 'alive'), 2500);`,
95+
`setTimeout(() => writeFileSync(${JSON.stringify(marker)}, 'alive'), 4500);`,
9696
"setInterval(() => {}, 1000);",
9797
].join("\n"),
9898
"utf8",
@@ -111,7 +111,7 @@ describe("runCommandArgs", () => {
111111
);
112112

113113
const result = await runCommandArgs(process.execPath, [parentScript], dir, undefined, {
114-
timeoutMs: 1000,
114+
timeoutMs: 3000,
115115
});
116116
await new Promise((resolve) => setTimeout(resolve, 1200));
117117

website/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ Files:
88
- `favicon.svg`: browser icon
99
- `social-card.svg`: link preview card
1010
- `social-card.png`: raster link preview card for Open Graph/Twitter
11+
- `robots.txt`: crawler policy with sitemap reference
12+
- `sitemap.xml`: canonical single-page sitemap
13+
- `_headers`: static security headers for hosts that support header files
1114

1215
Preview:
1316

website/_headers

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/*
2+
Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
3+
X-Content-Type-Options: nosniff
4+
X-Frame-Options: DENY
5+
Referrer-Policy: strict-origin-when-cross-origin
6+
Permissions-Policy: camera=(), microphone=(), geolocation=(), payment=(), usb=()
7+
Content-Security-Policy: default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; font-src https://fonts.gstatic.com; img-src 'self' data:; connect-src 'self'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'

website/index.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1057,7 +1057,7 @@ <h2>Reference</h2>
10571057
<main>
10581058
<header class="home-hero">
10591059
<p class="eyebrow">Automated Code Review · Explicit Fixes</p>
1060-
<h1>Code review with<br />explicit fixes</h1>
1060+
<h1>Code review with <br />explicit fixes</h1>
10611061
<p class="lede">
10621062
Clawpatch maps codebases into semantic feature slices, reviews them for bugs and quality
10631063
issues, and records explicit fix attempts with validation.

website/robots.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
User-agent: *
2+
Allow: /
3+
4+
Sitemap: https://clawpatch.ai/sitemap.xml

website/sitemap.xml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
3+
<url>
4+
<loc>https://clawpatch.ai/</loc>
5+
<lastmod>2026-05-20</lastmod>
6+
<changefreq>weekly</changefreq>
7+
<priority>1.0</priority>
8+
</url>
9+
</urlset>

0 commit comments

Comments
 (0)