Skip to content

Commit effc6bb

Browse files
committed
feat(indexer): build edges extraction + CI diagnostics ingestion
- Add _build_edges_from_detection(): materializes CMake/Make/Bazel parse results into token_build_edges (target→source, target→dep, target→flag edges). Integrated into build_build_doc(). - Add scripts/ingest_ci_diagnostics.py: fetches GitHub Actions logs, parses GCC/Clang/MSVC/CMake/Ninja diagnostics, outputs structured JSONL for token_diagnostic_edges pipeline. - Add scripts/cleanup_verified_intermediates.py: purges source/extract cache for repos with verified parquet output.
1 parent ec1bf93 commit effc6bb

4 files changed

Lines changed: 1084 additions & 0 deletions

File tree

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/usr/bin/env python3
2+
"""Purge source/intermediate files for repos whose parquet output is verified.
3+
4+
Streaming principle: download -> process -> verify parquet -> delete source+intermediate.
5+
Parquet (zstd-max compressed) is the ONLY persistent artifact.
6+
7+
Usage:
8+
python scripts/cleanup_verified_intermediates.py [--dry-run] [--extract-cache] [--source-cache]
9+
"""
10+
from __future__ import annotations
11+
12+
import argparse
13+
import json
14+
import shutil
15+
from pathlib import Path
16+
17+
import pyarrow.parquet as pq
18+
19+
OUTPUT_BASE = Path("outputs")
20+
EXTRACT_CACHE = OUTPUT_BASE / "extract_cache_case5_shared"
21+
SOURCE_CACHE = OUTPUT_BASE / "source_cache" / "code"
22+
CODE_PARQUET = OUTPUT_BASE / "reindexed_case5_v7_20260715_130725_code"
23+
DONE_MANIFESTS = [
24+
OUTPUT_BASE / "conveyor_case5_v11_resume_20260719_120500" / "_done.json",
25+
OUTPUT_BASE / "conveyor_case5_v10_resume_20260718_150406" / "_done.json",
26+
]
27+
28+
29+
def load_done_repos() -> set[str]:
30+
done = set()
31+
for mf in DONE_MANIFESTS:
32+
if not mf.exists():
33+
continue
34+
data = json.loads(mf.read_text())
35+
for key, val in data.get("done", {}).items():
36+
if isinstance(val, dict) and val.get("source") == "code":
37+
done.add(key.split("::")[0])
38+
return done
39+
40+
41+
def verify_parquet_exists(repo: str) -> bool:
42+
for length_dir in CODE_PARQUET.iterdir():
43+
if not length_dir.is_dir():
44+
continue
45+
if (length_dir / f"{repo}.parquet").exists():
46+
schema = pq.read_schema(length_dir / f"{repo}.parquet")
47+
if "trained_token_count" in schema.names:
48+
return True
49+
return False
50+
51+
52+
def main():
53+
parser = argparse.ArgumentParser()
54+
parser.add_argument("--dry-run", action="store_true")
55+
parser.add_argument("--extract-cache", action="store_true", help="Clean extract cache")
56+
parser.add_argument("--source-cache", action="store_true", help="Clean source cache")
57+
args = parser.parse_args()
58+
59+
done_repos = load_done_repos()
60+
print(f"Repos with verified parquet: {len(done_repos)}")
61+
62+
freed = 0
63+
64+
if args.extract_cache and EXTRACT_CACHE.exists():
65+
for entry in EXTRACT_CACHE.iterdir():
66+
repo_name = entry.name
67+
if repo_name in done_repos and verify_parquet_exists(repo_name):
68+
size = sum(f.stat().st_size for f in entry.rglob("*") if f.is_file())
69+
if args.dry_run:
70+
print(f" [dry-run] would remove extract_cache/{repo_name} ({size/1e9:.2f} GB)")
71+
else:
72+
shutil.rmtree(entry, ignore_errors=True)
73+
print(f" removed extract_cache/{repo_name} ({size/1e9:.2f} GB)")
74+
freed += size
75+
76+
if args.source_cache and SOURCE_CACHE.exists():
77+
for entry in SOURCE_CACHE.iterdir():
78+
repo_name = entry.name
79+
if repo_name in done_repos and verify_parquet_exists(repo_name):
80+
size = sum(f.stat().st_size for f in entry.rglob("*") if f.is_file())
81+
if args.dry_run:
82+
print(f" [dry-run] would remove source_cache/{repo_name} ({size/1e9:.2f} GB)")
83+
else:
84+
shutil.rmtree(entry, ignore_errors=True)
85+
print(f" removed source_cache/{repo_name} ({size/1e9:.2f} GB)")
86+
freed += size
87+
88+
print(f"\nTotal freed: {freed/1e9:.2f} GB")
89+
90+
91+
if __name__ == "__main__":
92+
main()

0 commit comments

Comments
 (0)