|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Audit YAML-writing scripts in CommunityMech. |
| 3 | +
|
| 4 | +For every Python module under ``scripts/`` and the central |
| 5 | +``src/communitymech/`` package that writes a YAML (looks for |
| 6 | +``yaml.dump``, ``yaml.safe_dump``, ``write_validated_community``, or a |
| 7 | +``path.write_text(yaml.safe_dump(...))`` flow), record: |
| 8 | +
|
| 9 | +- ``appends_curation_history``: does the script append a CurationEvent |
| 10 | + to ``community['curation_history']``? |
| 11 | +- ``has_write_safeguard``: a ``--dry-run`` opt-out OR ``--apply``/``--write`` |
| 12 | + opt-in flag. |
| 13 | +- ``validates_before_write``: does it route through |
| 14 | + ``write_validated_community`` or call ``validate_community`` / |
| 15 | + ``linkml-validate`` first? |
| 16 | +- ``wired_into_just``: is the script invoked from a justfile recipe? |
| 17 | +
|
| 18 | +TSV columns: path, writes_yaml, appends_curation_history, |
| 19 | +has_write_safeguard, validates_before_write, wired_into_just. |
| 20 | +
|
| 21 | +Output: TSV to stdout (and via ``--out`` to a file). |
| 22 | +
|
| 23 | +Ported from CultureMech / MediaIngredientMech / TraitMech. |
| 24 | +""" |
| 25 | + |
| 26 | +from __future__ import annotations |
| 27 | + |
| 28 | +import argparse |
| 29 | +import csv |
| 30 | +import re |
| 31 | +import sys |
| 32 | +from pathlib import Path |
| 33 | + |
| 34 | +SEARCH_DIRS = [ |
| 35 | + Path("scripts"), |
| 36 | + Path("src/communitymech"), |
| 37 | +] |
| 38 | + |
| 39 | +# Patterns |
| 40 | +_WRITE_TEXT_OF_YAML = re.compile(r"\.write_text\s*\(\s*yaml\.(?:safe_)?dump") |
| 41 | +_CURATION_APPEND = re.compile( |
| 42 | + r"curation_history.*?(append|\+=|\.insert)" |
| 43 | + r"|['\"]curator['\"]\s*:" |
| 44 | + r"|append_curation_event" |
| 45 | + r"|record_curation_event" |
| 46 | +) |
| 47 | +_WRITE_SAFEGUARD = re.compile( |
| 48 | + r"--dry[-_]run|dry_run\s*[:=]" |
| 49 | + r"|--apply\b|args\.apply\b" |
| 50 | + r"|--write\b|args\.write\b" |
| 51 | +) |
| 52 | +_VALIDATE_BEFORE_WRITE = re.compile( |
| 53 | + r"linkml[._-]?validate" |
| 54 | + r"|validate_community\(" |
| 55 | + r"|validator\.validate\(" |
| 56 | + r"|write_validated_community\(" |
| 57 | +) |
| 58 | + |
| 59 | + |
| 60 | +def script_paths() -> list[Path]: |
| 61 | + out: list[Path] = [] |
| 62 | + for d in SEARCH_DIRS: |
| 63 | + if not d.exists(): |
| 64 | + continue |
| 65 | + out.extend(sorted(p for p in d.rglob("*.py") if "__pycache__" not in str(p))) |
| 66 | + return out |
| 67 | + |
| 68 | + |
| 69 | +def looks_like_yaml_writer(text: str) -> bool: |
| 70 | + if "yaml.safe_dump(" in text or "yaml.dump(" in text: |
| 71 | + return True |
| 72 | + if _WRITE_TEXT_OF_YAML.search(text): |
| 73 | + return True |
| 74 | + # write_validated_community is the closed-schema-gated wrapper that |
| 75 | + # callers route through instead of yaml.dump directly. |
| 76 | + return "write_validated_community(" in text |
| 77 | + |
| 78 | + |
| 79 | +def audit(path: Path, justfile_text: str) -> dict | None: |
| 80 | + # Suppress self-match: this module's regex source contains |
| 81 | + # `yaml.safe_dump` etc., so it would otherwise appear in its own output. |
| 82 | + if path.resolve() == Path(__file__).resolve(): |
| 83 | + return None |
| 84 | + try: |
| 85 | + text = path.read_text() |
| 86 | + except (UnicodeDecodeError, OSError): |
| 87 | + return None |
| 88 | + if not looks_like_yaml_writer(text): |
| 89 | + return None |
| 90 | + return { |
| 91 | + "path": str(path), |
| 92 | + "writes_yaml": "yes", |
| 93 | + "appends_curation_history": "yes" if _CURATION_APPEND.search(text) else "no", |
| 94 | + "has_write_safeguard": "yes" if _WRITE_SAFEGUARD.search(text) else "no", |
| 95 | + "validates_before_write": "yes" if _VALIDATE_BEFORE_WRITE.search(text) else "no", |
| 96 | + "wired_into_just": "yes" if _is_wired_into_just(path, justfile_text) else "no", |
| 97 | + } |
| 98 | + |
| 99 | + |
| 100 | +def _is_wired_into_just(path: Path, justfile_text: str) -> bool: |
| 101 | + """Detect whether a justfile recipe actually invokes this script. |
| 102 | +
|
| 103 | + The earlier substring check (``path.stem in justfile_text``) had false |
| 104 | + positives — e.g. ``write_validated.py`` matched a justfile comment |
| 105 | + referencing ``write_validated_community``. Require the filename to |
| 106 | + appear as an explicit ``python ... <name>.py`` invocation, which is |
| 107 | + how every justfile recipe actually runs a script. |
| 108 | + """ |
| 109 | + needle = re.compile(rf"\b{re.escape(path.name)}\b") |
| 110 | + for line in justfile_text.splitlines(): |
| 111 | + stripped = line.strip() |
| 112 | + # Ignore comment-only lines so a mention in docs doesn't count. |
| 113 | + if stripped.startswith("#"): |
| 114 | + continue |
| 115 | + if needle.search(stripped): |
| 116 | + return True |
| 117 | + return False |
| 118 | + |
| 119 | + |
| 120 | +def main() -> int: |
| 121 | + ap = argparse.ArgumentParser(description=__doc__) |
| 122 | + ap.add_argument("--out", type=Path, default=None, help="TSV output path (default stdout)") |
| 123 | + args = ap.parse_args() |
| 124 | + |
| 125 | + justfile_path = Path("justfile") |
| 126 | + justfile_text = justfile_path.read_text() if justfile_path.exists() else "" |
| 127 | + |
| 128 | + rows: list[dict] = [] |
| 129 | + for p in script_paths(): |
| 130 | + row = audit(p, justfile_text) |
| 131 | + if row is not None: |
| 132 | + rows.append(row) |
| 133 | + |
| 134 | + fields = [ |
| 135 | + "path", |
| 136 | + "writes_yaml", |
| 137 | + "appends_curation_history", |
| 138 | + "has_write_safeguard", |
| 139 | + "validates_before_write", |
| 140 | + "wired_into_just", |
| 141 | + ] |
| 142 | + |
| 143 | + if args.out: |
| 144 | + args.out.parent.mkdir(parents=True, exist_ok=True) |
| 145 | + with args.out.open("w", newline="") as fh: |
| 146 | + w = csv.DictWriter(fh, fieldnames=fields, delimiter="\t", lineterminator="\n") |
| 147 | + w.writeheader() |
| 148 | + for row in rows: |
| 149 | + w.writerow(row) |
| 150 | + print(f"Wrote {len(rows)} rows to {args.out}", file=sys.stderr) |
| 151 | + else: |
| 152 | + w = csv.DictWriter(sys.stdout, fieldnames=fields, delimiter="\t") |
| 153 | + w.writeheader() |
| 154 | + for row in rows: |
| 155 | + w.writerow(row) |
| 156 | + |
| 157 | + def count(field: str, val: str) -> int: |
| 158 | + return sum(1 for r in rows if r[field] == val) |
| 159 | + |
| 160 | + print("", file=sys.stderr) |
| 161 | + print(f"=== writers audit summary ({len(rows)} writers) ===", file=sys.stderr) |
| 162 | + print(f" appends curation_history: {count('appends_curation_history', 'yes')} / {len(rows)}", |
| 163 | + file=sys.stderr) |
| 164 | + print(f" has write safeguard: {count('has_write_safeguard', 'yes')} / {len(rows)}", |
| 165 | + file=sys.stderr) |
| 166 | + print(f" validates before write: {count('validates_before_write', 'yes')} / {len(rows)}", |
| 167 | + file=sys.stderr) |
| 168 | + print(f" wired into justfile: {count('wired_into_just', 'yes')} / {len(rows)}", |
| 169 | + file=sys.stderr) |
| 170 | + return 0 |
| 171 | + |
| 172 | + |
| 173 | +if __name__ == "__main__": |
| 174 | + sys.exit(main()) |
0 commit comments