|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | +Frontmatter & canonical integrity check for docs pages. |
| 4 | +
|
| 5 | +Guards the SEO bug class where a malformed YAML frontmatter fence silently |
| 6 | +drops a page's title/description/canonical and makes Mintlify fall back to the |
| 7 | +default no-slash canonical (which 308-redirects). See the docs SEO audit |
| 8 | +(2026-07): four pages shipped with a trailing space on the opening `---` fence |
| 9 | +(`--- ` instead of `---`), so Mintlify ignored the whole block. |
| 10 | +
|
| 11 | +For every built docs page (all *.mdx except snippets/includes and mint-ignored |
| 12 | +trees) this enforces: |
| 13 | + 1. Opening fence is EXACTLY `---` (no trailing whitespace / extra dashes / BOM). |
| 14 | + 2. A closing `---` fence exists. |
| 15 | + 3. A `canonical:` key is present and equals the trailing-slash, www URL |
| 16 | + derived from the file path (self-referential canonical policy). |
| 17 | + 4. A `title:` key is present, unless the page is OpenAPI-generated |
| 18 | + (`openapi:` key -> title comes from the spec). |
| 19 | + 5. No page lives under a top-level `docs/` folder (would serve at a broken |
| 20 | + double `/docs/docs/...` URL). |
| 21 | +
|
| 22 | +Stdlib only (no PyYAML) so it runs in CI without extra install steps. |
| 23 | +Exit 1 with a list of violations if any are found. |
| 24 | +""" |
| 25 | +import os |
| 26 | +import re |
| 27 | +import sys |
| 28 | + |
| 29 | +BASE = "https://www.checklyhq.com/docs/" |
| 30 | +# Trees excluded from the Mintlify build (.mintignore) or that are include-only. |
| 31 | +EXCLUDE_PREFIXES = ("api-reference-old/", "skills/", "node_modules/") |
| 32 | +SNIPPET_PREFIX = "snippets/" # reusable includes, not standalone pages |
| 33 | + |
| 34 | + |
| 35 | +def expected_canonical(rel): |
| 36 | + slug = rel[:-4] # strip ".mdx" |
| 37 | + if slug == "index": |
| 38 | + return BASE |
| 39 | + return f"{BASE}{slug}/" |
| 40 | + |
| 41 | + |
| 42 | +def frontmatter_lines(lines): |
| 43 | + """Return (fm_lines, close_idx) for a well-formed `---`...`---` block, else (None, None).""" |
| 44 | + if not lines or lines[0] != "---": |
| 45 | + return None, None |
| 46 | + for i in range(1, len(lines)): |
| 47 | + if lines[i] == "---": |
| 48 | + return lines[1:i], i |
| 49 | + return None, None |
| 50 | + |
| 51 | + |
| 52 | +def main(): |
| 53 | + root = os.getcwd() |
| 54 | + violations = [] |
| 55 | + |
| 56 | + prune = {"node_modules", ".git", "api-reference-old", "skills"} |
| 57 | + mdx_files = [] |
| 58 | + for dirpath, dirs, files in os.walk(root): |
| 59 | + dirs[:] = [d for d in dirs if d not in prune] |
| 60 | + for name in files: |
| 61 | + if name.endswith(".mdx"): |
| 62 | + rel = os.path.relpath(os.path.join(dirpath, name), root) |
| 63 | + mdx_files.append(rel.replace(os.sep, "/")) |
| 64 | + mdx_files.sort() |
| 65 | + |
| 66 | + for rel in mdx_files: |
| 67 | + if any(rel.startswith(p) for p in EXCLUDE_PREFIXES): |
| 68 | + continue |
| 69 | + if rel.startswith(SNIPPET_PREFIX): |
| 70 | + continue # includes: no frontmatter expected |
| 71 | + |
| 72 | + if rel.startswith("docs/"): |
| 73 | + violations.append( |
| 74 | + f"{rel}: page lives under a top-level 'docs/' folder -> would serve " |
| 75 | + f"at a broken double '/docs/docs/...' URL. Move it up one level." |
| 76 | + ) |
| 77 | + |
| 78 | + raw = open(rel, "rb").read() |
| 79 | + if raw.startswith(b"\xef\xbb\xbf"): |
| 80 | + violations.append(f"{rel}: file starts with a UTF-8 BOM (breaks frontmatter parsing).") |
| 81 | + raw = raw[3:] |
| 82 | + # Normalize CRLF/CR so a Windows-authored file isn't misreported as a |
| 83 | + # malformed fence (Mintlify/gray-matter parse CRLF fine). .gitattributes |
| 84 | + # also pins *.mdx to LF as defense-in-depth. |
| 85 | + text = raw.decode("utf-8", errors="replace").replace("\r\n", "\n").replace("\r", "\n") |
| 86 | + lines = text.split("\n") |
| 87 | + |
| 88 | + first = lines[0] if lines else "" |
| 89 | + if first != "---": |
| 90 | + if first.strip().startswith("---") or first.lstrip("").startswith("---"): |
| 91 | + violations.append( |
| 92 | + f"{rel}: malformed opening frontmatter fence {first!r} (must be exactly '---'). " |
| 93 | + f"Trailing whitespace makes Mintlify ignore title/description/canonical." |
| 94 | + ) |
| 95 | + else: |
| 96 | + violations.append(f"{rel}: missing opening frontmatter fence (first line {first!r}).") |
| 97 | + continue # can't trust the rest of the block |
| 98 | + |
| 99 | + fm, close_idx = frontmatter_lines(lines) |
| 100 | + if fm is None: |
| 101 | + violations.append(f"{rel}: no closing '---' frontmatter fence found.") |
| 102 | + continue |
| 103 | + |
| 104 | + keys = {} |
| 105 | + for line in fm: |
| 106 | + m = re.match(r"^([A-Za-z0-9_]+):\s*(.*)$", line) |
| 107 | + if m: |
| 108 | + keys[m.group(1)] = m.group(2).strip().strip("'\"") |
| 109 | + |
| 110 | + is_openapi = "openapi" in keys |
| 111 | + |
| 112 | + canonical = keys.get("canonical") |
| 113 | + if canonical is None: |
| 114 | + violations.append(f"{rel}: missing 'canonical' in frontmatter.") |
| 115 | + else: |
| 116 | + exp = expected_canonical(rel) |
| 117 | + if canonical != exp: |
| 118 | + violations.append(f"{rel}: canonical {canonical!r} != expected {exp!r}.") |
| 119 | + |
| 120 | + if not is_openapi and not keys.get("title"): |
| 121 | + violations.append(f"{rel}: missing 'title' in frontmatter.") |
| 122 | + |
| 123 | + if violations: |
| 124 | + print("Frontmatter check FAILED with the following issues:\n", file=sys.stderr) |
| 125 | + for v in violations: |
| 126 | + print(f" - {v}", file=sys.stderr) |
| 127 | + print(f"\n{len(violations)} issue(s). See .github/scripts/check_frontmatter.py.", file=sys.stderr) |
| 128 | + return 1 |
| 129 | + |
| 130 | + print(f"Frontmatter check passed ({len(mdx_files)} .mdx files scanned).") |
| 131 | + return 0 |
| 132 | + |
| 133 | + |
| 134 | +if __name__ == "__main__": |
| 135 | + sys.exit(main()) |
0 commit comments