diff --git a/chipfoundry_cli/__init__.py b/chipfoundry_cli/__init__.py index a2efd49..9ea8481 100644 --- a/chipfoundry_cli/__init__.py +++ b/chipfoundry_cli/__init__.py @@ -1,2 +1,2 @@ """ChipFoundry CLI package: Automate project submission to SFTP.""" -__version__ = "0.1.0" \ No newline at end of file +__version__ = "2.3.2" \ No newline at end of file diff --git a/chipfoundry_cli/main.py b/chipfoundry_cli/main.py index 3b9ac68..1e28da9 100644 --- a/chipfoundry_cli/main.py +++ b/chipfoundry_cli/main.py @@ -1,5 +1,6 @@ import click import getpass +from chipfoundry_cli.remote_precheck_git import RemotePrecheckGitError, verify_remote_precheck_repo from chipfoundry_cli.utils import ( collect_project_files, ensure_cf_directory, update_or_create_project_json, sftp_connect, upload_with_progress, sftp_ensure_dirs, sftp_download_recursive, @@ -1653,6 +1654,13 @@ def pull(project_name, output_dir, sftp_host, sftp_username, sftp_key): "CHANGES_REQUESTED": "Changes requested by the review team. See notes above.", } +REMOTE_PRECHECK_STATUS_COLORS = { + "queued": "yellow", + "running": "cyan bold", + "completed": "green", + "failed": "red", +} + def _show_platform_status(project_root: str): """Show the platform pipeline panel if the project is linked. Returns True if shown.""" @@ -1695,6 +1703,28 @@ def _show_platform_status(project_root: str): lines.append(f"[bold]Tapeout:[/bold] [{tc}]{tl}[/{tc}]") if project.get('gds_hash'): lines.append(f"[bold]GDS Hash:[/bold] {project['gds_hash'][:16]}...") + rj = project.get("latest_remote_precheck_job") + if isinstance(rj, dict) and rj.get("status"): + jst = str(rj.get("status", "")) + jc = REMOTE_PRECHECK_STATUS_COLORS.get(jst, "white") + lines.append(f"[bold]Remote precheck:[/bold] [{jc}]{jst}[/{jc}]") + ref = rj.get("git_ref") + if ref: + lines.append(f"[dim] git ref: {ref}[/dim]") + created = rj.get("created_at") + if created and isinstance(created, str): + lines.append(f"[dim] started: {created[:19]}[/dim]") + if jst in ("completed", "failed"): + done = rj.get("completed_at") + if done and isinstance(done, str): + lines.append(f"[dim] finished: {done[:19]}[/dim]") + if jst == "failed" and rj.get("error_message"): + err = str(rj["error_message"]) + if len(err) > 240: + err = err[:237] + "..." + lines.append(f"[red] {err}[/red]") + if jst == "completed" and rj.get("github_pr_url"): + lines.append(f"[green] PR:[/green] {rj['github_pr_url']}") if project.get('updated_at'): lines.append(f"[bold]Updated:[/bold] {project['updated_at'][:10]}") if project.get('admin_review_notes'): @@ -3248,7 +3278,21 @@ def _upload_precheck_results(project_json_path: Path): @click.option('--magic-drc', is_flag=True, help='Include Magic DRC check (optional, off by default)') @click.option('--checks', multiple=True, help='Specific checks to run (can be specified multiple times)') @click.option('--dry-run', is_flag=True, help='Show the command without running') -def precheck(project_root, skip_checks, magic_drc, checks, dry_run): +@click.option('--remote', is_flag=True, help='Queue precheck on the chipIgnite platform (requires cf login + linked project)') +@click.option( + '--poll', + is_flag=True, + help='With --remote: poll until the job finishes and print progress (5s interval).', +) +@click.option('--git-ref', default='main', show_default=True, help='Git branch or tag for remote precheck') +@click.option( + '--wait-timeout', + type=int, + default=7200, + show_default=True, + help='With --remote --poll: max seconds to wait (0 = no limit). Ignored without --poll.', +) +def precheck(project_root, skip_checks, magic_drc, checks, dry_run, remote, poll, git_ref, wait_timeout): """Run precheck validation on the project. This runs the cf-precheck tool to validate your design before @@ -3259,6 +3303,13 @@ def precheck(project_root, skip_checks, magic_drc, checks, dry_run): cf precheck --skip-checks lvs # Skip LVS check cf precheck --magic-drc # Include optional Magic DRC cf precheck --checks topcell_check # Run specific checks only + cf precheck --remote # Queue on platform; exit when accepted + cf precheck --remote --poll # Wait and stream progress + cf precheck --remote --poll --wait-timeout 0 # Poll until done (no time limit) + + Remote precheck requires your local HEAD to match origin for --git-ref, and precheck + inputs (wrapper GDS, verilog/rtl/user_defines.v when the GPIO check runs, and tracked + .cf/project.json) to match that commit. """ cwd_root, _ = get_project_json_from_cwd() if not project_root and cwd_root: @@ -3274,6 +3325,180 @@ def precheck(project_root, skip_checks, magic_drc, checks, dry_run): return project_json_path = project_root_path / '.cf' / 'project.json' + + if poll and not remote: + console.print("[red]✗[/red] --poll requires --remote.") + raise SystemExit(1) + + if remote: + import time + from urllib.parse import urlencode + + import httpx as httpx_remote + platform_id = _load_project_platform_id(str(project_root_path)) + if not platform_id: + console.print( + "[red]✗[/red] Link this repo to a platform project (set platform_project_id via [bold]cf link[/bold])." + ) + raise SystemExit(1) + try: + verify_remote_precheck_repo( + project_root_path, + git_ref, + checks=tuple(checks), + skip_checks=tuple(skip_checks), + ) + except RemotePrecheckGitError as e: + console.print(f"[red]✗[/red] {e}") + raise SystemExit(1) + remote_params = [("git_ref", git_ref)] + # Single checks= / skip_checks= value so proxies do not drop duplicate query keys. + if checks: + remote_params.append(("checks", ",".join(checks))) + if skip_checks: + remote_params.append(("skip_checks", ",".join(skip_checks))) + if magic_drc: + remote_params.append(("magic_drc", "true")) + if dry_run: + console.print( + f"[cyan]Would POST[/cyan] /projects/{platform_id}/precheck-jobs?" + + urlencode(remote_params) + ) + return + if poll and wait_timeout < 0: + console.print( + "[red]✗[/red] --wait-timeout must be >= 0 (0 means no limit while polling)." + ) + raise SystemExit(1) + config = load_user_config() + api_key = config.get('api_key') + if not api_key: + console.print("[yellow]Not logged in.[/yellow] Run [bold]cf login[/bold] first.") + raise SystemExit(1) + api_url = _get_api_url() + client = httpx_remote.Client( + base_url=f"{api_url}/api/v1", + headers={'Authorization': f'Bearer {api_key}'}, + timeout=120.0, + ) + try: + resp = client.post( + f"/projects/{platform_id}/precheck-jobs", + params=remote_params, + ) + if resp.status_code == 401: + console.print("[red]✗[/red] API key is invalid or expired. Run [bold]cf login[/bold].") + raise SystemExit(1) + if not resp.is_success: + try: + detail = resp.json().get("detail", resp.text) + except Exception: + detail = resp.text + console.print(f"[red]✗[/red] {detail}") + raise SystemExit(1) + job = resp.json() + jid = job["id"] + st0 = job.get("status") or "unknown" + if st0 == "failed": + console.print(f"[cyan]Remote precheck[/cyan] job_id={jid} status={st0}") + elif st0 == "running": + console.print(f"[cyan]Remote precheck started[/cyan] job_id={jid} status={st0}") + else: + console.print(f"[cyan]Queued remote precheck[/cyan] job_id={jid} status={st0}") + if job.get("status") == "failed" and job.get("error_message"): + console.print(f"[red]✗[/red] {job['error_message']}") + raise SystemExit(1) + if job.get("status") == "completed": + console.print("[green]✓[/green] Remote precheck completed") + if job.get("github_pr_url"): + console.print(f" Pull request: {job['github_pr_url']}") + return + if not poll: + console.print( + "[dim]Not waiting: use [bold]cf precheck --remote --poll[/bold] to stream progress " + "([bold]--wait-timeout 0[/bold] = no time limit while polling).[/dim]" + ) + return + deadline = None if wait_timeout == 0 else time.monotonic() + wait_timeout + if wait_timeout == 0: + console.print("[dim]Polling until the job completes (no timeout).[/dim]") + else: + console.print( + f"[dim]Polling every 5s; stops after {wait_timeout}s if still queued or running. " + f"Use [bold]--wait-timeout 0[/bold] for no limit.[/dim]" + ) + last_status_seen = st0 + terminal = None + github_pr_url = None + fail_message = None + progress_emitted = 0 + console.print("[dim]Worker log batches appear below as the platform receives them (5s poll).[/dim]") + while True: + if deadline is not None and time.monotonic() > deadline: + console.print( + "[yellow]⚠[/yellow] Timed out waiting for remote precheck (job still queued or running)." + ) + console.print( + f"[dim]job_id={jid} — open the project in the portal or run [bold]cf status[/bold].[/dim]" + ) + console.print( + "[dim]Cancel a stuck run in the portal, or retry with e.g. " + "[bold]cf precheck --remote --poll --wait-timeout 14400[/bold].[/dim]" + ) + raise SystemExit(1) + time.sleep(5) + r2 = client.get(f"/projects/{platform_id}/precheck-jobs/{jid}") + if r2.status_code == 401: + console.print("[red]✗[/red] API key is invalid or expired.") + raise SystemExit(1) + r2.raise_for_status() + j2 = r2.json() + st = j2.get("status") + prog = j2.get("progress") + if isinstance(prog, list) and len(prog) > progress_emitted: + for row in prog[progress_emitted:]: + if not isinstance(row, dict): + continue + msg = row.get("message") + if msg: + det = row.get("details") + if ( + isinstance(det, dict) + and det.get("event") == "check_done" + ): + console.print(Text(str(msg), style="bold")) + else: + console.print(Text(str(msg), style="dim")) + progress_emitted = len(prog) + if st == "completed": + terminal = "completed" + github_pr_url = j2.get("github_pr_url") + break + if st == "failed": + terminal = "failed" + fail_message = j2.get("error_message") or "unknown error" + break + if st != last_status_seen: + console.print( + f"[dim]… job status[/dim] [cyan]{st or 'unknown'}[/cyan]" + ) + last_status_seen = st + + if terminal == "completed": + console.print("[green]✓[/green] Remote precheck completed") + if github_pr_url: + console.print(f" Pull request: {github_pr_url}") + elif terminal == "failed": + console.print(f"[red]✗[/red] Remote precheck failed: {fail_message}") + raise SystemExit(1) + except SystemExit: + raise + except Exception as e: + console.print(f"[red]✗[/red] Remote precheck request failed: {e}") + raise SystemExit(1) + finally: + client.close() + return with open(project_json_path, 'r') as f: project_data = json.load(f) @@ -3319,12 +3544,14 @@ def precheck(project_root, skip_checks, magic_drc, checks, dry_run): if magic_drc: precheck_args.append('--magic-drc') - - if skip_checks: - precheck_args.extend(['--skip-checks'] + list(skip_checks)) - + + # Positional check names before --skip-checks (matches cf-precheck argparse; see + # precheck-runner _cf_precheck_shell_cmd). if checks: precheck_args.extend(list(checks)) + + if skip_checks: + precheck_args.extend(['--skip-checks'] + list(skip_checks)) inner_cmd = 'pip3 install --upgrade -q --root-user-action=ignore cf-precheck 2>/dev/null && exec cf-precheck ' + ' '.join(precheck_args) diff --git a/chipfoundry_cli/remote_precheck_git.py b/chipfoundry_cli/remote_precheck_git.py new file mode 100644 index 0000000..a4acef4 --- /dev/null +++ b/chipfoundry_cli/remote_precheck_git.py @@ -0,0 +1,228 @@ +""" +Git consistency checks before queueing remote precheck. + +Ensures local HEAD matches the commit the platform will clone for --git-ref and that +working tree / index match that commit for precheck inputs (wrapper GDS, user_defines.v +when the GPIO check runs, and .cf/project.json when it is tracked). +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import List, Optional, Set, Tuple + + +class RemotePrecheckGitError(Exception): + """Local repository state is not consistent with origin for remote precheck.""" + + +_GDS_BASES: Tuple[Tuple[str, str], ...] = ( + ("analog", "gds/user_analog_project_wrapper"), + ("digital", "gds/user_project_wrapper"), + ("openframe", "gds/openframe_project_wrapper"), + ("mini", "gds/user_project_wrapper_mini4"), +) +_GDS_SUFFIXES: Tuple[str, ...] = (".gds", ".gds.gz") + +USER_DEFINES_REL = "verilog/rtl/user_defines.v" +CF_PROJECT_JSON_REL = ".cf/project.json" + + +def _run_git(repo: Path, *args: str, timeout: float = 120.0) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=str(repo), + capture_output=True, + text=True, + timeout=timeout, + ) + + +def _resolve_origin_tip_sha(repo: Path, git_ref: str) -> str: + ref = git_ref.strip() + if not ref: + raise RemotePrecheckGitError("--git-ref must be a non-empty branch or tag name.") + r = _run_git(repo, "ls-remote", "origin", f"refs/heads/{ref}") + if r.returncode != 0: + raise RemotePrecheckGitError( + f"git ls-remote failed (is 'origin' configured and reachable?): {r.stderr.strip() or r.stdout}" + ) + lines = [ln for ln in r.stdout.strip().splitlines() if ln.strip()] + if lines: + return lines[0].split()[0] + r2 = _run_git(repo, "ls-remote", "origin", f"refs/tags/{ref}") + if r2.returncode != 0: + raise RemotePrecheckGitError( + f"git ls-remote failed for tags: {r2.stderr.strip() or r2.stdout}" + ) + lines2 = [ln for ln in r2.stdout.strip().splitlines() if ln.strip()] + if lines2: + return lines2[0].split()[0] + raise RemotePrecheckGitError( + f"No branch or tag {ref!r} found on origin. Push the ref or fix --git-ref." + ) + + +def _local_head_sha(repo: Path) -> str: + r = _run_git(repo, "rev-parse", "HEAD") + if r.returncode != 0: + raise RemotePrecheckGitError( + f"Not a valid git checkout: {r.stderr.strip() or 'git rev-parse HEAD failed'}" + ) + return r.stdout.strip() + + +def _detect_wrapper_gds(repo: Path) -> Tuple[str, str]: + """Return (project_kind, relative_path) for the single wrapper GDS, or raise.""" + hits: list[Tuple[str, str]] = [] + for kind, base in _GDS_BASES: + for suf in _GDS_SUFFIXES: + rel = base + suf + if (repo / rel).is_file(): + hits.append((kind, rel)) + break + if not hits: + raise RemotePrecheckGitError( + "No wrapper GDS found (expected exactly one of e.g. " + "gds/user_project_wrapper.gds, gds/user_analog_project_wrapper.gds, …). " + "Remote precheck requires the same layout as local cf-precheck." + ) + if len(hits) > 1: + paths = ", ".join(h[1] for h in hits) + raise RemotePrecheckGitError( + f"Multiple wrapper GDS layouts found ({paths}). Remove extras so only one project type is present." + ) + return hits[0] + + +def _load_cf_project_type(project_json: Path) -> Optional[str]: + if not project_json.is_file(): + return None + try: + data = json.loads(project_json.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + t = data.get("project", {}).get("type") + return str(t).strip().lower() if t else None + + +def _gpio_defines_will_run( + checks: Tuple[str, ...], + skip_checks: Tuple[str, ...], + project_kind: str, +) -> bool: + if project_kind not in ("analog", "digital"): + return False + cnorm = {x.strip().lower() for x in checks if x.strip()} + snorm = {x.strip().lower() for x in skip_checks if x.strip()} + if "gpio_defines" in snorm: + return False + if cnorm: + return "gpio_defines" in cnorm + return True + + +def _path_tracked_in_git(repo: Path, rel: str) -> bool: + r = _run_git(repo, "ls-files", "--error-unmatch", rel) + return r.returncode == 0 + + +def _critical_precheck_paths( + repo: Path, + project_json: Path, + checks: Tuple[str, ...], + skip_checks: Tuple[str, ...], +) -> Set[str]: + kind_gds, gds_rel = _detect_wrapper_gds(repo) + out: Set[str] = {gds_rel} + + cf_type = _load_cf_project_type(project_json) + if cf_type and cf_type != kind_gds: + raise RemotePrecheckGitError( + f".cf/project.json type is {cf_type!r} but the wrapper GDS indicates {kind_gds!r}. " + "Fix project type or GDS layout before remote precheck." + ) + + if _gpio_defines_will_run(checks, skip_checks, kind_gds): + ud = repo / USER_DEFINES_REL + if ud.is_file() or _path_tracked_in_git(repo, USER_DEFINES_REL): + out.add(USER_DEFINES_REL) + + if _path_tracked_in_git(repo, CF_PROJECT_JSON_REL): + out.add(CF_PROJECT_JSON_REL) + + return out + + +def _porcelain_paths(repo: Path) -> List[str]: + r = _run_git(repo, "status", "--porcelain=v1", "-u") + if r.returncode != 0: + raise RemotePrecheckGitError(f"git status failed: {r.stderr.strip()}") + paths: List[str] = [] + for line in r.stdout.splitlines(): + if len(line) < 4: + continue + code = line[:2] + rest = line[3:].strip() + if " -> " in rest: + rest = rest.split(" -> ", 1)[-1] + if rest.startswith('"') and rest.endswith('"'): + rest = rest[1:-1].replace('\\"', '"') + if code == "??": + paths.append(f"??{rest}") + elif code.strip(): + paths.append(rest) + return paths + + +def verify_remote_precheck_repo( + project_root: Path, + git_ref: str, + *, + checks: Tuple[str, ...], + skip_checks: Tuple[str, ...], +) -> None: + """ + Raise RemotePrecheckGitError unless origin/{git_ref} tip matches HEAD and precheck + input paths are clean and match that revision. + """ + repo = project_root.resolve() + git_marker = repo / ".git" + if not (git_marker.is_dir() or git_marker.is_file()): + raise RemotePrecheckGitError( + "Remote precheck requires a git checkout with .git (clone your GitHub repo, not a plain folder copy)." + ) + + remote_sha = _resolve_origin_tip_sha(repo, git_ref) + head_sha = _local_head_sha(repo) + if head_sha != remote_sha: + raise RemotePrecheckGitError( + f"Local HEAD ({head_sha[:7]}) must match origin {git_ref!r} ({remote_sha[:7]}). " + f"git checkout {git_ref} && git pull, or push your commits, then retry." + ) + + project_json = repo / ".cf" / "project.json" + critical = _critical_precheck_paths(repo, project_json, checks, skip_checks) + + dirty = _porcelain_paths(repo) + for entry in dirty: + if entry.startswith("??"): + path = entry[2:] + if path in critical: + raise RemotePrecheckGitError( + f"{path!r} is untracked but required for remote precheck. " + "Add and commit it (or remove it) so the remote clone matches your machine." + ) + elif entry in critical: + raise RemotePrecheckGitError( + f"{entry!r} has uncommitted changes. Commit or stash before remote precheck." + ) + + for rel in sorted(critical): + r = _run_git(repo, "diff-index", "--quiet", "HEAD", "--", rel) + if r.returncode != 0: + raise RemotePrecheckGitError( + f"{rel!r} has uncommitted changes. Commit or stash before remote precheck." + ) diff --git a/pyproject.toml b/pyproject.toml index e6b4553..f1c1cd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "chipfoundry-cli" -version = "2.3.1" +version = "2.3.12" description = "CLI tool to automate ChipFoundry project submission to SFTP server" authors = ["ChipFoundry "] readme = "README.md" diff --git a/tests/test_precheck_command.py b/tests/test_precheck_command.py index f2e903d..1dc16fd 100644 --- a/tests/test_precheck_command.py +++ b/tests/test_precheck_command.py @@ -35,6 +35,8 @@ def test_precheck_help(self): assert '--magic-drc' in result.output assert '--checks' in result.output assert '--dry-run' in result.output + assert '--poll' in result.output + assert '--wait-timeout' in result.output def test_precheck_dry_run(self, temp_project_dir): """Test precheck command with --dry-run flag.""" diff --git a/tests/test_remote_precheck_git.py b/tests/test_remote_precheck_git.py new file mode 100644 index 0000000..60a372d --- /dev/null +++ b/tests/test_remote_precheck_git.py @@ -0,0 +1,90 @@ +"""Tests for remote precheck git consistency checks.""" + +import subprocess +from pathlib import Path + +import pytest + +from chipfoundry_cli.remote_precheck_git import RemotePrecheckGitError, verify_remote_precheck_repo + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(cwd), *args], check=True, capture_output=True) + + +def _init_digital_project(work: Path) -> None: + (work / "gds").mkdir(parents=True) + (work / "gds" / "user_project_wrapper.gds").write_bytes(b"\x00") + (work / "verilog" / "rtl").mkdir(parents=True) + (work / "verilog" / "rtl" / "user_defines.v").write_text("//gpio\n") + cf = work / ".cf" + cf.mkdir() + (cf / "project.json").write_text('{"project":{"type":"digital"}}') + + +def test_verify_passes_when_head_matches_origin_main(tmp_path: Path) -> None: + bare = tmp_path / "origin.git" + work = tmp_path / "work" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + subprocess.run(["git", "clone", str(bare), str(work)], check=True, capture_output=True) + _init_digital_project(work) + _git(work, "add", ".") + _git(work, "commit", "-m", "init") + _git(work, "branch", "-M", "main") + _git(work, "push", "-u", "origin", "main") + + verify_remote_precheck_repo(work, "main", checks=(), skip_checks=()) + + +def test_verify_fails_when_local_ahead_of_origin(tmp_path: Path) -> None: + bare = tmp_path / "origin.git" + work = tmp_path / "work" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + subprocess.run(["git", "clone", str(bare), str(work)], check=True, capture_output=True) + _init_digital_project(work) + _git(work, "add", ".") + _git(work, "commit", "-m", "init") + _git(work, "branch", "-M", "main") + _git(work, "push", "-u", "origin", "main") + + (work / "README.md").write_text("x") + _git(work, "add", "README.md") + _git(work, "commit", "-m", "second") + + with pytest.raises(RemotePrecheckGitError, match="must match origin"): + verify_remote_precheck_repo(work, "main", checks=(), skip_checks=()) + + +def test_verify_fails_dirty_user_defines(tmp_path: Path) -> None: + bare = tmp_path / "origin.git" + work = tmp_path / "work" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + subprocess.run(["git", "clone", str(bare), str(work)], check=True, capture_output=True) + _init_digital_project(work) + _git(work, "add", ".") + _git(work, "commit", "-m", "init") + _git(work, "branch", "-M", "main") + _git(work, "push", "-u", "origin", "main") + + ud = work / "verilog" / "rtl" / "user_defines.v" + ud.write_text("//changed\n") + + with pytest.raises(RemotePrecheckGitError, match="user_defines.v"): + verify_remote_precheck_repo(work, "main", checks=(), skip_checks=()) + + +def test_verify_skips_user_defines_when_only_xor(tmp_path: Path) -> None: + bare = tmp_path / "origin.git" + work = tmp_path / "work" + subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True) + subprocess.run(["git", "clone", str(bare), str(work)], check=True, capture_output=True) + _init_digital_project(work) + _git(work, "add", ".") + _git(work, "commit", "-m", "init") + _git(work, "branch", "-M", "main") + _git(work, "push", "-u", "origin", "main") + + ud = work / "verilog" / "rtl" / "user_defines.v" + ud.write_text("//changed\n") + + verify_remote_precheck_repo(work, "main", checks=("xor",), skip_checks=())