Update scripts and CI

Author: johnsamuelwrites

.github/workflows/ci.yml

@@ -35,11 +35,23 @@ jobs:
       - name: Check internal broken links across the whole site
         run: python src/main/ci_internal_links.py --all
 
-      - name: Check generated abstract pages
+      - name: Check Q315 links
         run: python src/main/ci_internal_links.py --root Q315
 
       - name: Check canonical abstract CSS references
         run: python src/main/abstract/css_assets.py check
 
-      - name: Check abstract HTML contract pilot
-        run: python src/main/abstract/validate_abstract_html.py Q315/Q3062/index.html
+      - name: Check every canonical Q315 document
+        run: python src/main/abstract/validate_abstract_html.py Q315
+
+      - name: Check rendered Q315 labels are current
+        run: python src/main/abstract/render_page.py --check
+
+      - name: Check Q315 structural repairs are current
+        run: python src/main/abstract/repair_structure.py --check
+
+      - name: Check Q315 round-trip has no regressions
+        run: |
+          python src/main/abstract/verify_content_roundtrip.py \
+            --max-mismatches 214 \
+            --report "${RUNNER_TEMP}/q315-content-roundtrip.json"

.gitignore

@@ -17,22 +17,17 @@ test.py
 # Q315 abstract workflow generated artifacts
 src/main/abstract/pilots/
 src/main/abstract/snapshots/
-src/main/abstract/travel-content*.csv
-src/main/abstract/travel-content*.quickstatements
+# CSV reports, translation scratch files, and QuickStatements are disposable
+# migration output.  Durable inputs belong in data/ or have an explicit
+# exception below.
+src/main/abstract/*.csv
+src/main/abstract/*.quickstatements
+!src/main/abstract/abstract-functions.quickstatements
 src/main/abstract/content-migration-registry.csv
 src/main/abstract/content-migration-report.json
 src/main/abstract/content-roundtrip.json
-src/main/abstract/missing-content-review.csv
-src/main/abstract/missing-content*.quickstatements
-src/main/abstract/content-corrections-review.csv
-src/main/abstract/content-corrections.quickstatements
-src/main/abstract/abstract-composition.quickstatements
 *.quickstatements.state.json
-src/main/abstract/abstract-composition-partial.quickstatements
-src/main/abstract/abstract-composition-review.csv
-src/main/abstract/abstract-composition-structure.quickstatements
 src/main/abstract/abstract-composition-markup.txt
-src/main/abstract/abstract-composition.quickstatements
 
 .vscode
 .obsidian

src/main/abstract/README.md

@@ -88,6 +88,10 @@ python3 src/main/abstract/verify_content_roundtrip.py
 Both renderers take `--page QID` to scope to a single page. `verify_content_roundtrip.py`
 is the completion gate; residual mismatches are structural and must be placed
 inside their corresponding canonical containers, not appended elsewhere.
+CI runs both renderers in `--check` mode and rejects any increase over the
+documented 214-pair structural round-trip baseline. It also validates every
+HTML document below `Q315/`; passing a directory to `validate_abstract_html.py`
+discovers its HTML files recursively.
 
 ## Direct Wikibase bot
 
@@ -96,7 +100,7 @@ the MediaWiki API. Validation is the default and does not require credentials:
 
 ```bash
 python3 src/main/wikibase_write.py \
-  src/main/abstract/missing-label-updates.quickstatements
+  /path/to/generated-batch.quickstatements
 ```
 
 Create a dedicated bot password in the Wikibase user preferences, copy
@@ -105,7 +109,7 @@ enable writes:
 
 ```bash
 python3 src/main/wikibase_write.py \
-  src/main/abstract/missing-label-updates.quickstatements --apply
+  /path/to/generated-batch.quickstatements --apply
 ```
 
 The writer supports the commands generated in this repository: `CREATE`,

src/main/abstract/gen_compose_research.py

@@ -86,23 +86,38 @@ def validate(path: Path) -> list[str]:
 
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("paths", type=Path, nargs="+")
+    parser.add_argument(
+        "paths",
+        type=Path,
+        nargs="+",
+        help="HTML files or directories to search recursively",
+    )
     return parser
 
 
-def main(argv: Sequence[str] | None = None) -> int:
-    args = build_parser().parse_args(argv)
+def html_documents(paths: Sequence[Path]) -> tuple[list[Path], list[str]]:
+    documents: list[Path] = []
     errors: list[str] = []
-    for path in args.paths:
-        if not path.is_file():
-            errors.append(f"{path}: file does not exist")
+    for path in paths:
+        if path.is_dir():
+            documents.extend(sorted(path.rglob("*.html")))
+        elif path.is_file():
+            documents.append(path)
         else:
-            errors.extend(validate(path))
+            errors.append(f"{path}: file or directory does not exist")
+    return documents, errors
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    documents, errors = html_documents(args.paths)
+    for path in documents:
+        errors.extend(validate(path))
     if errors:
         for error in errors:
             print(f"ERROR: {error}")
         return 1
-    print(f"Validated {len(args.paths)} abstract HTML document(s)")
+    print(f"Validated {len(documents)} abstract HTML document(s)")
     return 0
 
 

src/main/abstract/validate_abstract_html.py

@@ -146,6 +146,14 @@ def main() -> int:
         default="",
         help="restrict verification to a single abstract page QID",
     )
+    parser.add_argument(
+        "--max-mismatches",
+        type=int,
+        help=(
+            "fail only when mismatch count exceeds this known structural "
+            "baseline (default: require complete equivalence)"
+        ),
+    )
     parser.add_argument("--report", type=Path, default=DEFAULT_REPORT)
     args = parser.parse_args()
     repo_root = args.repo_root.resolve()
@@ -160,6 +168,8 @@ def main() -> int:
         f"Round-trip status: {report['status']}; "
         f"language-page mismatches: {report['mismatch_count']}"
     )
+    if args.max_mismatches is not None:
+        return 0 if report["mismatch_count"] <= args.max_mismatches else 1
     return 0 if report["status"] == "equivalent" else 1
 
 

src/main/abstract/verify_content_roundtrip.py

@@ -11,7 +11,7 @@
 )
 from abstract.model import FunctionCall, MonolingualText
 from abstract.prepare_travel_content import validate_quickstatements
-from abstract.validate_abstract_html import validate
+from abstract.validate_abstract_html import html_documents, validate
 
 
 class AbstractFunctionTests(unittest.TestCase):
@@ -84,6 +84,19 @@ def test_argument_outside_call_is_rejected(self):
         self.assertEqual(len(errors), 1)
         self.assertIn("contained in q-call", errors[0])
 
+    def test_directories_are_discovered_recursively(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            nested = root / "nested"
+            nested.mkdir()
+            page = nested / "page.html"
+            page.write_text("<html></html>", encoding="utf-8")
+
+            documents, errors = html_documents([root])
+
+            self.assertEqual(documents, [page])
+            self.assertEqual(errors, [])
+
 
 class TravelContentQuickStatementsTests(unittest.TestCase):
     def test_validator_accepts_complete_queryable_languages(self):