diff --git a/README.md b/README.md index 804c1dc8..5f104dda 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,8 @@ lc status --json # machine-readable output lc verify # recompute hashes and walk the provenance chain lc build # pre-build container images from Containerfiles lc build --force --runtime podman # force rebuild and pin runtime +lc export wrroc # export a Workflow Run RO-Crate bundle for publication +lc export wrroc --zip -o run.zip # zip the bundle for upload to Zenodo / WorkflowHub ``` ## Capabilities @@ -110,6 +112,10 @@ Jobs always dispatch through a Dask cluster: a `LocalCluster` on a workstation, Every materialized output gets a `.lightcone-manifest.json` capturing `code_version` (sha256 of recipe + container + decisions), `data_version` (sha256 of the output directory contents), input manifest versions, git SHA, lc version, and host. `lc verify` recomputes hashes and walks the chain — failures surface as `tampered_data`, `broken_chain`, or `missing_manifest`. `lc status` reads only manifests, so it works offline. +### Publishing analyses + +`lc export wrroc` walks your manifests and emits a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) bundle — a JSON-LD package readable by WorkflowHub, Zenodo's RO-Crate plugin, and any RO-Crate-aware archive. Each materialization becomes a `CreateAction` with `object` (inputs, including upstream datasets via stable `@id` references) and `result` (the output dataset); decisions become `PropertyValue` entities; the workflow is captured as a `ComputationalWorkflow`. The lightcone manifest format on disk is unchanged — WRROC is the publication view, generated on demand. Use `--metadata-only` to ship only the provenance graph (useful when data files are huge), `--zip` to package the bundle for upload, or `-u ` to restrict to specific universes. + ### Telemetry Claude Code sessions are traced to Langfuse with full conversation structure, tool calls, and git commit linking. Disable with `TRACE_TO_LANGFUSE=false` in `.claude/settings.local.json`. diff --git a/claude/lightcone/guides/lightcone-cli-reference.md b/claude/lightcone/guides/lightcone-cli-reference.md index c9156663..7868ceff 100644 --- a/claude/lightcone/guides/lightcone-cli-reference.md +++ b/claude/lightcone/guides/lightcone-cli-reference.md @@ -10,6 +10,7 @@ lc run [OUTPUTS...] [--universe NAME] [--force] [--verbose] [--rerun-triggers TR lc build [--force] [--runtime docker] # Build container images from specs lc status [--universe NAME] [--json] # Materialization status (text or JSON) lc verify [--universe NAME] # Recompute hashes and walk the provenance chain +lc export wrroc [--output PATH] [--universe NAME] [--zip] [--metadata-only] [--author "NAME "] # Export Workflow Run RO-Crate bundle lc eval {run,report,compare} # Run/inspect eval suites (requires the 'eval' extra) ``` @@ -72,3 +73,17 @@ Outputs land at `results///`, with the per-output manifest - **`lc verify` failure** — `missing_manifest` (output dir exists with no `.lightcone-manifest.json`), `tampered_data` (bytes on disk no longer hash to the recorded `data_version`), or `broken_chain` (an upstream's `data_version` drifted from what this output's manifest recorded). Re-run the affected output with `lc run` to repair. After failure: fix, then `lc run --universe `. + +## Publishing Analyses + +`lc export wrroc` bundles the project's manifests, workflow definition, decisions, and (optionally) data files into a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) — a JSON-LD package readable by RO-Crate-aware archives (WorkflowHub, Zenodo's RO-Crate plugin, etc.). The lightcone manifest format on disk is unchanged; the bundle is generated on demand. + +```bash +lc export wrroc # ./wrroc/ directory +lc export wrroc -o run.zip --zip # zip bundle +lc export wrroc --metadata-only # provenance graph + manifests only (no data files) +lc export wrroc -u baseline -u alt_method # restrict to specific universes +lc export wrroc --author "Name " # override git config +``` + +The bundle's `@graph` contains a `ComputationalWorkflow` (the astra.yaml), one `CreateAction` per materialized output (with `object` referencing upstream datasets and external inputs, `result` referencing the produced dataset, and `instrument` referencing the recipe `SoftwareApplication`), `PropertyValue` entries for decisions and provenance metadata (`code_version`, `data_version`), and a `Person` for the author. diff --git a/docs/cli/export.md b/docs/cli/export.md new file mode 100644 index 00000000..3e3b52f4 --- /dev/null +++ b/docs/cli/export.md @@ -0,0 +1,106 @@ +# lc export + +Export project artifacts in interoperable formats. Currently the only +exporter is `wrroc` (Workflow Run RO-Crate); the group is shaped to host +future formats without breaking the CLI surface. + +## Synopsis + +``` +lc export wrroc [OPTIONS] +``` + +## lc export wrroc + +Walk the project's per-output `.lightcone-manifest.json` sidecars and +emit a [Workflow Run RO-Crate](https://www.researchobject.org/workflow-run-crate/) +bundle — a JSON-LD package readable by WorkflowHub, Zenodo's RO-Crate +plugin, and any RO-Crate-aware archive. The on-disk manifest format is +unchanged; the bundle is the *publication view*, generated on demand. + +### Options + +| Option | Default | Effect | +|--------|---------|--------| +| `--output`, `-o PATH` | `./wrroc` | Bundle directory, or `.zip` path when `--zip` is set. | +| `--universe`, `-u NAME` | every universe with materialized outputs | Restrict to specific universes. Repeatable. | +| `--author "NAME "` | git `user.name` / `user.email`, then `LIGHTCONE_AUTHOR` env | Override the author recorded in the bundle. | +| `--license URL` | `https://creativecommons.org/licenses/by/4.0/` | License URL or SPDX identifier for the bundle. Required by the WRROC profile. | +| `--zip` / `--no-zip` | `--no-zip` | Package the bundle as a single `.zip` after building. | +| `--metadata-only` | off | Skip data files; bundle manifests + `astra.yaml` + universe files only. | + +### What gets bundled + +- `astra.yaml` → `ComputationalWorkflow` (`programmingLanguage: snakemake`). +- Each materialized output → `Dataset` with `version = data_version`. +- Each recipe execution → `CreateAction` with `object` (inputs, both upstream datasets via stable `@id` and external files), `result` (the produced dataset), and `instrument` (the recipe `SoftwareApplication`). +- Each container → `SoftwareApplication` referenced via `softwareRequirements`. +- Each decision → `FormalParameter` + per-run `PropertyValue`. +- Author → `Person`. + +Provenance metadata (`code_version`, `data_version`, `git_sha`, `lc_version`, host) lands as `PropertyValue` entries on the relevant entities. + +### Examples + +```bash +lc export wrroc # ./wrroc/ directory +lc export wrroc -o run.zip --zip # zip bundle for upload +lc export wrroc --metadata-only # provenance graph only, no data files +lc export wrroc -u baseline -u alt_method # restrict to specific universes +lc export wrroc --author "Ada Lovelace " +lc export wrroc --license https://opensource.org/licenses/MIT +``` + +### Output + +``` +✓ Wrote WRROC directory: ./wrroc + Captured 7 runs across universes: baseline, alt_method +``` + +If no materialized outputs are found, the bundle still writes — but only +contains the workflow definition, and a warning is printed: + +``` +✓ Wrote WRROC directory: ./wrroc +Warning: no materialized outputs were found — the bundle contains only + the workflow definition. + This usually means recipes haven't been run yet (try lc run) or the + .lightcone-manifest.json sidecars are missing. + Workflow-only bundles will not pass strict Provenance Run Crate + validation; that profile requires at least one materialized run. +``` + +### Failure modes + +| Error | Cause | +|---|---| +| `No astra.yaml at ; cannot export.` | The cwd is not inside an ASTRA project. | +| ` is non-empty; refuse to clobber.` | The target directory already has contents. Pass a fresh path or remove the existing one. | +| ` is an existing directory; cannot overwrite with a zip.` | `--zip` was requested but the output path resolves to a directory. Use a file path like `bundle.zip`. | + +Manifests that exist but are unreadable (e.g. cross-user symlinks under +`results/` with permission denied) are warned about and skipped — they +do not abort the export. + +### Validation + +The bundle conforms to the [Provenance Run Crate 0.5](https://w3id.org/ro/wfrun/provenance/0.5) +profile. To validate locally: + +```bash +pip install git+https://github.com/crs4/rocrate-validator.git +rocrate-validator -y validate ./wrroc/ +``` + +### When to run + +- Before submitting a paper or depositing artifacts in Zenodo / WorkflowHub. +- After a clean run (`lc verify` clean) on the final commit you intend to publish. +- For external collaborators who don't have `lc` installed but need to inspect provenance. + +### Related + +- [`lc verify`](verify.md) — confirm the manifest chain is intact before exporting. +- [`lc status`](status.md) — see which outputs will be captured by the export. +- [api/manifest](../api/manifest.md) — the on-disk format the export reads from. diff --git a/docs/cli/index.md b/docs/cli/index.md index 8d3d348c..be2a4c48 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -21,6 +21,7 @@ Code skills, with the CLI as the durable, scriptable backstop. | [`lc build`](build.md) | Build container images declared in `astra.yaml`. | | [`lc status`](status.md) | Manifest-driven status report. No Snakemake import needed. | | [`lc verify`](verify.md) | Recompute hashes, walk the input chain, surface tampering. | +| [`lc export`](export.md) | Emit interoperable bundles (Workflow Run RO-Crate) for publication. | | [`lc setup`](setup.md) | Write a minimal `~/.lightcone/config.yaml`. | ## Global options diff --git a/pyproject.toml b/pyproject.toml index 32aa237c..75563a35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ dependencies = [ "dask>=2024.1", "distributed>=2024.1", "langfuse>=2.0", + "rocrate>=0.11", ] [dependency-groups] @@ -88,6 +89,10 @@ module = ["dask.*", "distributed.*"] ignore_missing_imports = true follow_untyped_imports = true +[[tool.mypy.overrides]] +module = ["rocrate.*", "astra.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] markers = [ diff --git a/src/lightcone/cli/commands.py b/src/lightcone/cli/commands.py index 0c4b8dff..03d10871 100644 --- a/src/lightcone/cli/commands.py +++ b/src/lightcone/cli/commands.py @@ -827,6 +827,113 @@ def _ensure_images(project: Path, *, runtime: str, force: bool = False) -> None: raise click.ClickException(str(e)) +# ============================================================================= +# lc export +# ============================================================================= + + +@main.group() +def export() -> None: + """Export project artifacts in interoperable formats.""" + + +@export.command("wrroc") +@click.option( + "--output", + "-o", + type=click.Path(path_type=Path), + default=Path("./wrroc"), + help="Bundle directory (or .zip path with --zip).", + show_default=True, +) +@click.option( + "--universe", + "-u", + multiple=True, + help="Restrict to specific universes (default: all).", +) +@click.option( + "--author", + default=None, + help="Author override, e.g. \"Name \". Default: git config.", +) +@click.option( + "--license", + "license_url", + default=None, + help="License URL or SPDX identifier. Default: CC-BY-4.0.", +) +@click.option( + "--zip/--no-zip", + "zip_bundle", + default=False, + help="Package the bundle as a .zip after building.", +) +@click.option( + "--metadata-only", + is_flag=True, + help="Skip data files; bundle manifests + astra.yaml + universes only.", +) +def export_wrroc_cmd( + output: Path, + universe: tuple[str, ...], + author: str | None, + license_url: str | None, + zip_bundle: bool, + metadata_only: bool, +) -> None: + """Export a Workflow Run RO-Crate (WRROC) bundle. + + The bundle is suitable for upload to WorkflowHub, Zenodo (with the + RO-Crate plugin), or any RO-Crate-aware archive. The lightcone + manifest format on disk is unchanged — this is a publication view + generated on demand. + + Examples: + + lc export wrroc # ./wrroc/ directory + lc export wrroc -o my-run.zip --zip # zip bundle + lc export wrroc --metadata-only # provenance, no data + lc export wrroc -u baseline -u alt # specific universes + """ + from lightcone.engine.wrroc import export_wrroc + + project = _project_root() + + try: + result = export_wrroc( + project_path=project, + output_path=output, + universes=list(universe) or None, + author=author, + license=license_url, + zip_bundle=zip_bundle, + include_data=not metadata_only, + ) + except FileExistsError as e: + raise click.ClickException(str(e)) + + flavor = "zip bundle" if result.is_zip else "directory" + console.print( + f"[green]✓[/green] Wrote WRROC {flavor}: [cyan]{result.bundle_path}[/cyan]" + ) + if result.runs_included == 0: + console.print( + "[yellow]Warning:[/yellow] no materialized outputs were found — " + "the bundle contains only the workflow definition.\n" + " This usually means recipes haven't been run yet (try [cyan]lc run[/cyan]) " + "or the [cyan].lightcone-manifest.json[/cyan] sidecars are missing.\n" + " Workflow-only bundles will not pass strict Provenance Run Crate " + "validation; that profile requires at least one materialized run." + ) + else: + u_list = ", ".join(result.universes_included) + console.print( + f" Captured [bold]{result.runs_included}[/bold] runs across " + f"universes: [cyan]{u_list}[/cyan]" + ) + + # Register eval subgroup (requires optional 'eval' extra) try: from lightcone.eval.cli import eval_group diff --git a/src/lightcone/engine/manifest.py b/src/lightcone/engine/manifest.py index 9ba4798a..27749fbb 100644 --- a/src/lightcone/engine/manifest.py +++ b/src/lightcone/engine/manifest.py @@ -192,6 +192,10 @@ def write_manifest( "decisions": cfg.get("decisions", {}), "input_versions": input_versions, "git_sha": cfg.get("git_sha"), + # URL of the git origin remote at the time of materialization. + # Optional/additive — older manifests without this field still + # parse. Surfaced by ``lc export wrroc`` as a CodeRepository entity. + "git_remote": cfg.get("git_remote"), "lc_version": cfg.get("lc_version"), "finished_at": time.time(), "host": socket.gethostname(), diff --git a/src/lightcone/engine/snakefile.py b/src/lightcone/engine/snakefile.py index 9a0b1346..87b20158 100644 --- a/src/lightcone/engine/snakefile.py +++ b/src/lightcone/engine/snakefile.py @@ -131,6 +131,38 @@ def _git_sha(project_path: Path) -> str | None: return None +def _git_remote(project_path: Path) -> str | None: + """URL of the ``origin`` git remote, if the project is a git clone. + + Captured into the manifest alongside ``git_sha`` so a published + bundle can identify *which repository* the commit belongs to. + SSH URLs (``git@host:owner/repo.git``) are normalised to ``https`` + form so consumers (e.g. WorkflowHub) can render them as clickable + links. + """ + try: + out = subprocess.run( + ["git", "-C", str(project_path), + "config", "--get", "remote.origin.url"], + capture_output=True, + text=True, + check=False, + ) + if out.returncode != 0: + return None + url = out.stdout.strip() + if not url: + return None + if url.startswith("git@"): + host_path = url.removeprefix("git@").replace(":", "/", 1) + url = f"https://{host_path}" + if url.endswith(".git"): + url = url.removesuffix(".git") + return url + except FileNotFoundError: + return None + + def _lc_version() -> str: try: from importlib.metadata import version @@ -317,6 +349,7 @@ def generate( cfg: dict[str, dict[str, dict[str, Any]]] = {} git_sha = _git_sha(project_path) + git_remote = _git_remote(project_path) lc_version = _lc_version() resolve_image = make_image_tag_resolver(project_path, project_name) @@ -411,6 +444,7 @@ def generate( "decisions": scoped_decisions, "code_version": cv, "git_sha": git_sha, + "git_remote": git_remote, "lc_version": lc_version, } diff --git a/src/lightcone/engine/wrroc.py b/src/lightcone/engine/wrroc.py new file mode 100644 index 00000000..af9f72f1 --- /dev/null +++ b/src/lightcone/engine/wrroc.py @@ -0,0 +1,885 @@ +"""Workflow Run RO-Crate (WRROC) exporter. + +Walks a project's per-output ``.lightcone-manifest.json`` sidecars and +emits a `Workflow Run RO-Crate `_ +bundle suitable for upload to WorkflowHub, Zenodo, or any RO-Crate-aware +archive. + +The lightcone manifest layer remains the canonical internal format. WRROC +is the **publication** view, generated on demand. We target the deepest +of the three WRROC profiles — *Provenance Run Crate* — because lightcone +already captures the per-step data it requires. + +Entity mapping +-------------- + +============================== ===================================== +lightcone concept WRROC entity +============================== ===================================== +``astra.yaml`` ``ComputationalWorkflow`` +each universe ``PropertyValue`` set on the workflow +each materialized output dir ``Dataset`` (data files inside) +each recipe execution ``CreateAction`` + ``object`` upstream Datasets / external Files + ``result`` the output Dataset + ``instrument`` the recipe ``SoftwareApplication`` + ``agent`` the human author (``Person``) +each container image ``SoftwareApplication`` +each decision value ``PropertyValue`` on the workflow +============================== ===================================== + +The exporter is one-shot: ``export_wrroc()`` produces a directory (or +``--zip`` archive). We do not maintain a live crate; the user invokes +this when they're ready to publish. +""" +from __future__ import annotations + +import logging +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from astra.helpers import get_decisions, load_yaml, resolve_analysis_tree + +from lightcone.engine.manifest import MANIFEST_FILENAME, read_manifest +from lightcone.engine.tree import ( + TreeOutput, + collect_tree_outputs, + resolve_output_path, + resolve_universe_decisions, +) + +logger = logging.getLogger(__name__) + +#: WRROC profile we target. Pinned explicitly — bump when upgrading. +PROVENANCE_RUN_CRATE_PROFILE = ( + "https://w3id.org/ro/wfrun/provenance/0.5" +) +WORKFLOW_RUN_CRATE_PROFILE = ( + "https://w3id.org/ro/wfrun/workflow/0.5" +) +PROCESS_RUN_CRATE_PROFILE = ( + "https://w3id.org/ro/wfrun/process/0.5" +) + + +@dataclass +class ExportResult: + """Returned by :func:`export_wrroc` so callers can act on the outcome.""" + + bundle_path: Path + runs_included: int + universes_included: list[str] + is_zip: bool + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +#: Default license URL when none is supplied. CC-BY-4.0 is widely accepted +#: by Zenodo and WorkflowHub for research outputs and is permissive enough +#: not to surprise users who didn't think about licensing. Can be +#: overridden via the ``license`` argument or ``--license`` CLI flag. +DEFAULT_LICENSE = "https://creativecommons.org/licenses/by/4.0/" + + +def export_wrroc( + project_path: Path, + output_path: Path, + *, + universes: list[str] | None = None, + author: str | None = None, + license: str | None = None, + zip_bundle: bool = False, + include_data: bool = True, +) -> ExportResult: + """Walk manifests in *project_path* and emit a WRROC bundle. + + Parameters + ---------- + project_path: + Project root containing ``astra.yaml``. + output_path: + Where to write the bundle. If *zip_bundle* is True, this is the + ``.zip`` file; otherwise it is the bundle directory. + universes: + Restrict to a subset of universes. ``None`` includes all + universes that have at least one materialized output. + author: + Override the author. If ``None``, falls back to ``git config + user.name``/``user.email`` then to ``LIGHTCONE_AUTHOR`` env var. + license: + License URL or SPDX-style identifier for the bundle. Required by + the Workflow RO-Crate profile; defaults to :data:`DEFAULT_LICENSE` + (CC-BY-4.0) when ``None``. + zip_bundle: + Package as a single zip after building. The zip contains the + bundle directory at its root. + include_data: + When False, only the manifests, ``astra.yaml`` and universe files + are bundled — useful for archiving provenance without re-uploading + large data files. + """ + # Lazy import to keep the module importable without rocrate installed. + from rocrate.rocrate import ROCrate + + project_path = Path(project_path).resolve() + spec_path = project_path / "astra.yaml" + if not spec_path.is_file(): + raise FileNotFoundError( + f"No astra.yaml at {project_path}; cannot export." + ) + + spec = resolve_analysis_tree(load_yaml(spec_path), project_path) + project_name = (spec.get("name") or project_path.name).lower().replace(" ", "-") + + # Resolve which universes to include. If the caller didn't restrict, + # discover from the universes/ directory. + if universes is None: + universes = _discover_all_universes(project_path) + + crate = ROCrate() + crate.name = spec.get("name") or project_name + # RO-Crate REQUIRES a description on the root. Fall back to a + # generated one when astra.yaml doesn't define one. + crate.description = spec.get("description") or ( + f"WRROC bundle exported from {crate.name} on " + f"{_format_finished_at(_now())}." + ) + crate.creativeWorkStatus = "Published" + + # License — required by Workflow RO-Crate. Caller supplies it + # explicitly or we fall back to a permissive default. + license_url = license or spec.get("license") or DEFAULT_LICENSE + crate.root_dataset["license"] = {"@id": license_url} + + # Mark the root as a workflow run crate. + crate.root_dataset["conformsTo"] = [ + {"@id": PROCESS_RUN_CRATE_PROFILE}, + {"@id": WORKFLOW_RUN_CRATE_PROFILE}, + {"@id": PROVENANCE_RUN_CRATE_PROFILE}, + ] + # The validator expects each profile URL referenced by conformsTo to + # also exist as a CreativeWork entity in the @graph — declare them. + from rocrate.model import ContextEntity + for profile_url, profile_name in [ + (PROCESS_RUN_CRATE_PROFILE, "Process Run Crate"), + (WORKFLOW_RUN_CRATE_PROFILE, "Workflow Run Crate"), + (PROVENANCE_RUN_CRATE_PROFILE, "Provenance Run Crate"), + ]: + crate.add(ContextEntity(crate, profile_url, properties={ + "@type": "CreativeWork", + "name": f"{profile_name} 0.5", + "version": "0.5", + })) + # The metadata file descriptor itself must conformsTo a specific + # RO-Crate spec version. rocrate-py 0.15 emits a 1.2 @context but + # the validator looks for a conformsTo on the descriptor — set both + # 1.1 (for backward-compat validators) and 1.2 explicitly. + crate.metadata["conformsTo"] = [ + {"@id": "https://w3id.org/ro/crate/1.1"}, + {"@id": "https://w3id.org/ro/crate/1.2"}, + ] + + builder = WRROCBuilder( + crate=crate, + project_path=project_path, + spec=spec, + author_str=author or _detect_author(project_path), + include_data=include_data, + ) + + # Add the workflow definition (astra.yaml). + builder.add_workflow() + + # Walk each universe's outputs and emit Datasets + CreateActions. + runs_added = 0 + included_universes: list[str] = [] + tree_outputs = collect_tree_outputs(spec) + + for universe_id in universes: + universe_runs = builder.add_universe_runs(universe_id, tree_outputs) + if universe_runs > 0: + runs_added += universe_runs + included_universes.append(universe_id) + + if runs_added == 0: + logger.warning( + "No materialized outputs found for universes %s — bundle will " + "contain only the workflow definition.", + universes, + ) + + # Render the bundle. + output_path = Path(output_path).resolve() + + if zip_bundle: + if output_path.exists(): + if output_path.is_dir(): + raise FileExistsError( + f"{output_path} is an existing directory; cannot overwrite with a zip. " + "Pass a file path (e.g. bundle.zip) or remove the existing directory." + ) + output_path.unlink() + crate.write_zip(output_path) + result_path = output_path + else: + if output_path.exists() and any(output_path.iterdir()): + raise FileExistsError( + f"{output_path} is non-empty; refuse to clobber. " + "Pass a fresh path or remove the existing one." + ) + crate.write(output_path) + result_path = output_path + + return ExportResult( + bundle_path=result_path, + runs_included=runs_added, + universes_included=included_universes, + is_zip=zip_bundle, + ) + + +# --------------------------------------------------------------------------- +# Builder — accumulates entities into a single ROCrate instance +# --------------------------------------------------------------------------- + + +class WRROCBuilder: + """Accumulates lightcone state into a WRROC ``ROCrate`` instance. + + The builder owns the @id minting strategy and the de-duplication of + repeated entities (e.g. the same recipe ``SoftwareApplication`` is + used by multiple ``CreateAction`` runs). + """ + + def __init__( + self, + crate: Any, # rocrate.rocrate.ROCrate + project_path: Path, + spec: dict[str, Any], + author_str: str | None, + include_data: bool, + ) -> None: + self.crate = crate + self.project_path = project_path + self.spec = spec + self.include_data = include_data + + self._workflow_id: str | None = None + self._dataset_ids: dict[tuple[str, str], str] = {} # (universe, output) → @id + self._software_ids: dict[str, str] = {} # recipe text → @id + self._container_ids: dict[str, str] = {} # image tag → @id + self._person_id: str | None = None + self._code_repo_id: str | None = None # set lazily from manifest's git_remote + + if author_str: + self._person_id = self._add_person(author_str) + + # ----- Workflow + universes ----- + + def add_workflow(self) -> str: + """Add the astra.yaml as a ``ComputationalWorkflow`` entity.""" + from rocrate.model import ContextEntity + + spec_path = self.project_path / "astra.yaml" + wf_id = "astra.yaml" + # Also bundle the file itself (always — it's small and central). + # WRROC requires a known ComputerLanguage. astra.yaml is a spec + # over Snakemake (the actual executor), so we tag the workflow + # language as "snakemake" — the truthful description of what runs. + wf = self.crate.add_workflow( + spec_path, + wf_id, + main=True, + lang="snakemake", + ) + wf["name"] = self.spec.get("name") or "ASTRA analysis" + if "description" in self.spec: + wf["description"] = self.spec["description"] + + # Decisions: declare each decision (root + sub-analysis) as a + # FormalParameter of the workflow. Per-universe values get + # attached as PropertyValue on the CreateAction (see + # add_universe_runs). ASTRA's decisions are a dict keyed by id; + # use get_decisions() so sub-analysis decisions are merged in. + for decision_id, decision in get_decisions(self.spec).items(): + param_id = f"#param-{decision_id}" + # additionalType is REQUIRED by the WRROC FormalParameter + # shape. Infer from the default value (or first option) so + # we report the right schema.org primitive. + sample_val = _decision_sample_value(decision) + param = ContextEntity( + self.crate, + param_id, + properties={ + "@type": "FormalParameter", + "name": decision_id, + "description": ( + decision.get("rationale") + or decision.get("label") + or decision.get("description", "") + ), + "additionalType": _infer_additional_type(sample_val), + }, + ) + self.crate.add(param) + wf.append_to("input", param) + + # Bundle universe files for full reproducibility. + universes_dir = self.project_path / "universes" + if universes_dir.is_dir(): + for u_file in sorted(universes_dir.glob("*.yaml")): + rel = u_file.relative_to(self.project_path) + self.crate.add_file(u_file, str(rel)) + + self._workflow_id = wf_id + return wf_id + + def add_universe_runs( + self, + universe_id: str, + tree_outputs: list[TreeOutput], + ) -> int: + """Add Datasets + CreateActions for every materialized output in the + given universe. Returns the count of CreateActions added. + """ + decisions = _safe_load_universe_decisions( + self.project_path, self.spec, universe_id + ) + runs_added = 0 + + for tree_out in tree_outputs: + recipe = tree_out.output_def.get("recipe") + if recipe is None: # alias output, no own materialization + continue + + out_dir = ( + resolve_output_path(self.project_path, tree_out, universe_id) + / tree_out.output_id + ) + # Best-effort manifest read: skip outputs whose directory + # is unreadable (permission-denied scratch entries, broken + # symlinks, mid-rsync states). For `lc export`, partial + # bundles are more useful than a total abort. + try: + manifest = read_manifest(out_dir) + except OSError as exc: + logger.warning( + "Skipping %s/%s: cannot read manifest (%s)", + universe_id, tree_out.output_id, exc, + ) + continue + if manifest is None: + continue # not yet materialized in this universe + + dataset_id = self._add_output_dataset( + tree_out, universe_id, out_dir, manifest + ) + self._add_create_action( + tree_out=tree_out, + universe_id=universe_id, + dataset_id=dataset_id, + manifest=manifest, + decisions=decisions, + tree_outputs=tree_outputs, + ) + runs_added += 1 + + return runs_added + + # ----- Datasets / runs ----- + + def _add_output_dataset( + self, + tree_out: TreeOutput, + universe_id: str, + out_dir: Path, + manifest: dict[str, Any], + ) -> str: + """Add a ``Dataset`` for an output directory and return its @id.""" + rel_dir = out_dir.relative_to(self.project_path).as_posix() + dataset_id = rel_dir + "/" + + # If this dataset was already added (which can happen if the user + # passes overlapping universes), return the existing @id. + cache_key = (universe_id, tree_out.output_id) + if cache_key in self._dataset_ids: + return self._dataset_ids[cache_key] + + if self.include_data: + self.crate.add_dataset(out_dir, dataset_id) + else: + # Metadata-only mode: we still want the dataset in the graph + # for chain integrity, but we don't copy data files. Bundle + # only the manifest itself. + self.crate.add_dataset(None, dataset_id) + manifest_src = out_dir / MANIFEST_FILENAME + if manifest_src.exists(): + self.crate.add_file( + manifest_src, + f"{dataset_id}{MANIFEST_FILENAME}", + ) + + ds = self.crate.dereference(dataset_id) + ds["name"] = f"{tree_out.output_id} (universe={universe_id})" + # schema.org's `version` is the standard place for a content hash + # / version identifier on a Dataset. dataVersion (lightcone term) + # is not in the RO-Crate context so the validator rejects it. + if data_version := manifest.get("data_version"): + ds["version"] = data_version + + self._dataset_ids[cache_key] = dataset_id + return dataset_id + + def _add_create_action( + self, + *, + tree_out: TreeOutput, + universe_id: str, + dataset_id: str, + manifest: dict[str, Any], + decisions: dict[str, Any], + tree_outputs: list[TreeOutput], + ) -> str: + """Add a ``CreateAction`` linking inputs → instrument → output.""" + from rocrate.model import ContextEntity + + action_id = f"#run-{universe_id}-{_qualified_id(tree_out)}" + recipe_cmd = (tree_out.output_def.get("recipe") or {}).get("command", "") + + instrument_id = self._add_recipe_software( + recipe_cmd, + manifest.get("container_image"), + tool_name=(tree_out.output_def.get("recipe") or {}).get("tool_name"), + output_id=tree_out.output_id, + ) + + # If the manifest carries a git_remote URL, surface it as a + # CodeRepository entity once (de-duplicated across all actions). + self._link_code_repository(manifest.get("git_remote")) + + # Resolve `object` (inputs to the action). Each upstream input + # references the producing dataset's @id (Provenance chain). + # External inputs we represent as ContextEntity File-with-fingerprint. + objects: list[dict[str, str]] = [] + for inp_id, version_str in (manifest.get("input_versions") or {}).items(): + obj_ref = self._resolve_input_reference( + inp_id=inp_id, + version_str=version_str, + consumer=tree_out, + universe_id=universe_id, + tree_outputs=tree_outputs, + ) + if obj_ref is not None: + objects.append(obj_ref) + + properties: dict[str, Any] = { + "@type": "CreateAction", + "name": f"Run of {tree_out.output_id} (universe={universe_id})", + "instrument": {"@id": instrument_id}, + "object": objects, + "result": [{"@id": dataset_id}], + "endTime": _format_finished_at(manifest.get("finished_at")), + "actionStatus": {"@id": "http://schema.org/CompletedActionStatus"}, + } + if self._person_id: + properties["agent"] = {"@id": self._person_id} + + action = ContextEntity(self.crate, action_id, properties=properties) + self.crate.add(action) + + # Per-decision PropertyValue entities, attached to the action as + # parameter values rather than to the workflow (so multiple + # universes don't trample each other). + for d_id, d_value in decisions.items(): + pv_id = f"#pv-{universe_id}-{_qualified_id(tree_out)}-{d_id}" + pv = ContextEntity( + self.crate, + pv_id, + properties={ + "@type": "PropertyValue", + "name": d_id, + "value": _coerce_value(d_value), + }, + ) + self.crate.add(pv) + action.append_to("object", pv) + + # Workflow & manifest provenance metadata + action.append_to( + "object", + self._add_property_value( + f"#pv-{universe_id}-{_qualified_id(tree_out)}-code_version", + "code_version", + manifest.get("code_version", ""), + ), + ) + action.append_to( + "object", + self._add_property_value( + f"#pv-{universe_id}-{_qualified_id(tree_out)}-data_version", + "data_version", + manifest.get("data_version", ""), + ), + ) + + return action_id + + # ----- Resolved references ----- + + def _resolve_input_reference( + self, + *, + inp_id: str, + version_str: str, + consumer: TreeOutput, + universe_id: str, + tree_outputs: list[TreeOutput], + ) -> dict[str, str] | None: + """Return a ``{"@id": ...}`` reference for an action's input. + + For upstream-produced inputs, the @id points at the producing + dataset. For external inputs, we synthesize a File ContextEntity + with the fingerprint as its sha256 / mtime-size note. + """ + from lightcone.engine.tree import find_upstream_output + + upstream = find_upstream_output(consumer, inp_id, tree_outputs) + if upstream is not None: + # Reference an existing dataset @id (must already have been + # added — order-preserving collect_tree_outputs handles the + # common case; out-of-order DAGs still work because rocrate + # tolerates forward refs at write time). + cache_key = (universe_id, upstream.output_id) + if cache_key in self._dataset_ids: + return {"@id": self._dataset_ids[cache_key]} + # Forward reference: predict the @id deterministically. + out_dir = ( + resolve_output_path(self.project_path, upstream, universe_id) + / upstream.output_id + ) + return { + "@id": out_dir.relative_to(self.project_path).as_posix() + "/" + } + + # External input. Synthesize a stable @id from the input id + + # fingerprint so identical files de-duplicate across runs. + external_id = f"#ext-{inp_id}" + if not self.crate.dereference(external_id): + from rocrate.model import ContextEntity + + ext = ContextEntity( + self.crate, + external_id, + properties={ + "@type": "File", + "name": inp_id, + "description": f"External input fingerprint: {version_str}", + }, + ) + # Encode the fingerprint as a checksum/contentSize note. + if version_str.startswith("sha256:"): + ext["sha256"] = version_str.removeprefix("sha256:") + else: + ext["fingerprint"] = version_str + self.crate.add(ext) + return {"@id": external_id} + + def _link_code_repository(self, git_remote: str | None) -> None: + """Idempotently add a CodeRepository entity for the project repo. + + Called once per manifest read; later calls with the same URL are + no-ops. The repository entity is also linked from the workflow + via ``codeRepository`` so consumers can discover the source. + """ + if not git_remote: + return + if self._code_repo_id is not None: + return # already added + from rocrate.model import ContextEntity + + repo = ContextEntity(self.crate, git_remote, properties={ + "@type": ["CodeRepository", "SoftwareSourceCode"], + "name": git_remote.rsplit("/", 1)[-1] or git_remote, + "url": git_remote, + }) + self.crate.add(repo) + if self._workflow_id is not None: + wf = self.crate.dereference(self._workflow_id) + if wf is not None: + wf["codeRepository"] = {"@id": git_remote} + self._code_repo_id = git_remote + + def _add_recipe_software( + self, + recipe_cmd: str, + container_image: str | None, + *, + tool_name: str | None = None, + output_id: str | None = None, + ) -> str: + """De-duplicate recipes — return the @id of the SoftwareApplication. + + ``SoftwareApplication.name`` resolution order: + + 1. Explicit ``recipe.tool_name`` from astra.yaml (best — author-chosen). + 2. Heuristic from the command (e.g. ``scripts/analyze.py``). + 3. The output id (always available, always meaningful). + + The full command is always preserved as ``description``. + """ + from rocrate.model import ContextEntity + + key = recipe_cmd or "" + if key in self._software_ids: + return self._software_ids[key] + + sw_id = f"#recipe-{len(self._software_ids)}" + name = ( + tool_name + or _heuristic_tool_name(recipe_cmd) + or output_id + or "(empty recipe)" + ) + props: dict[str, Any] = { + "@type": "SoftwareApplication", + "name": name, + "description": recipe_cmd, + } + if container_image: + props["softwareRequirements"] = { + "@id": self._add_container_software(container_image) + } + sw = ContextEntity(self.crate, sw_id, properties=props) + self.crate.add(sw) + + # WRROC: ComputationalWorkflow MUST refer to its orchestrated + # tools via hasPart. Link each recipe back to the workflow. + if self._workflow_id is not None: + wf = self.crate.dereference(self._workflow_id) + if wf is not None: + wf.append_to("hasPart", sw) + + self._software_ids[key] = sw_id + return sw_id + + def _add_container_software(self, image_tag: str) -> str: + """De-duplicate container images.""" + from rocrate.model import ContextEntity + + if image_tag in self._container_ids: + return self._container_ids[image_tag] + cid = f"#container-{len(self._container_ids)}" + sw = ContextEntity( + self.crate, + cid, + properties={ + "@type": ["SoftwareApplication", "ContainerImage"], + "name": image_tag, + "softwareVersion": image_tag, + }, + ) + self.crate.add(sw) + self._container_ids[image_tag] = cid + return cid + + def _add_property_value(self, pv_id: str, name: str, value: Any) -> Any: + from rocrate.model import ContextEntity + + existing = self.crate.dereference(pv_id) + if existing: + return existing + pv = ContextEntity( + self.crate, + pv_id, + properties={ + "@type": "PropertyValue", + "name": name, + "value": _coerce_value(value), + }, + ) + self.crate.add(pv) + return pv + + def _add_person(self, author_str: str) -> str: + from rocrate.model import ContextEntity + + name, email = _parse_author(author_str) + person_id = f"#author-{(email or name).replace('@', '_at_')}" + if self.crate.dereference(person_id): + return person_id + props: dict[str, Any] = {"@type": "Person", "name": name} + if email: + props["email"] = email + person = ContextEntity(self.crate, person_id, properties=props) + self.crate.add(person) + return person_id + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _qualified_id(tree_out: TreeOutput) -> str: + """A filesystem-safe qualified id for use in @id minting.""" + if tree_out.analysis_id: + return f"{tree_out.analysis_id}.{tree_out.output_id}" + return tree_out.output_id + + +def _discover_all_universes(project_path: Path) -> list[str]: + """List every universe with at least one universe yaml present.""" + universes_dir = project_path / "universes" + if not universes_dir.exists(): + return ["baseline"] + ids = sorted(p.stem for p in universes_dir.glob("*.yaml")) + return ids or ["baseline"] + + +def _safe_load_universe_decisions( + project_path: Path, + spec: dict[str, Any], + universe_id: str, +) -> dict[str, Any]: + """Like resolve_universe_decisions but tolerant of missing/unreadable files.""" + universe_yaml = project_path / "universes" / f"{universe_id}.yaml" + try: + if not universe_yaml.exists(): + return {} + return resolve_universe_decisions(project_path, spec, universe_id) + except (FileNotFoundError, KeyError, OSError): + return {} + + +def _decision_sample_value(decision: dict[str, Any]) -> Any: + """Pick a representative value for a decision to infer its type. + + ASTRA's decisions schema uses a dict of options keyed by option id + (e.g. ``options: {bins_8: {label: '8 bins'}, ...}``). The default is + typically one of those keys. For type inference, prefer: + + 1. The `default` value if set (always a primitive). + 2. The first option key if options is a dict. + 3. The first option's `value` field if options is a list. + """ + if "default" in decision: + return decision["default"] + opts = decision.get("options") + if isinstance(opts, dict) and opts: + return next(iter(opts)) + if isinstance(opts, list) and opts: + first = opts[0] + if isinstance(first, dict): + return first.get("value") + return first + return None + + +def _heuristic_tool_name(recipe_cmd: str) -> str | None: + """Best-effort SoftwareApplication.name from a bash command. + + Looks for the first script-like token (``foo.py``, ``./bin/foo``, + ``foo.sh``) and returns just that. Returns None if no obvious tool + can be extracted, falling through to the output_id fallback. + """ + if not recipe_cmd: + return None + for token in recipe_cmd.split(): + # Skip env assignments, redirects, shell builtins + if "=" in token and not token.startswith("-"): + continue + if token in {"python", "python3", "bash", "sh", "uv", "run"}: + continue + # Must look like a path with an extension or a leading ./ + if "/" in token or token.startswith("./"): + return token + if "." in token and not token.startswith("-"): + ext = token.rsplit(".", 1)[-1] + if ext in {"py", "sh", "R", "jl", "rb", "pl"}: + return token + return None + + +def _infer_additional_type(value: Any) -> dict[str, str]: + """Map a sample decision value to a schema.org primitive type @id. + + WRROC's FormalParameter shape requires ``additionalType`` to indicate + the parameter's expected value type — Text/Integer/Float/Boolean. + """ + if isinstance(value, bool): + return {"@id": "http://schema.org/Boolean"} + if isinstance(value, int): + return {"@id": "http://schema.org/Integer"} + if isinstance(value, float): + return {"@id": "http://schema.org/Float"} + return {"@id": "http://schema.org/Text"} + + +def _coerce_value(value: Any) -> Any: + """schema.org PropertyValue.value should be a primitive (str/num/bool). + + Lightcone decisions can be arbitrary YAML — coerce non-primitives to + a JSON string so the PropertyValue stays valid. + """ + if isinstance(value, (str, int, float, bool)) or value is None: + return value + import json as _json + return _json.dumps(value, sort_keys=True) + + +def _format_finished_at(ts: float | None) -> str | None: + """Convert a unix timestamp to ISO 8601 for schema:endTime.""" + if ts is None: + return None + from datetime import UTC, datetime + return datetime.fromtimestamp(ts, tz=UTC).isoformat() + + +def _now() -> float: + import time + return time.time() + + +def _parse_author(s: str) -> tuple[str, str | None]: + """Parse ``"Name "`` or just ``"Name"`` into (name, email).""" + s = s.strip() + if "<" in s and s.endswith(">"): + name, _, rest = s.rpartition("<") + return name.strip(), rest.removesuffix(">").strip() or None + return s, None + + +def _detect_author(project_path: Path) -> str | None: + """Pull author from git config or environment, else return None.""" + if env := os.environ.get("LIGHTCONE_AUTHOR"): + return env + try: + name = subprocess.run( + ["git", "config", "user.name"], + cwd=project_path, capture_output=True, text=True, timeout=5, + ).stdout.strip() + email = subprocess.run( + ["git", "config", "user.email"], + cwd=project_path, capture_output=True, text=True, timeout=5, + ).stdout.strip() + if name and email: + return f"{name} <{email}>" + return name or None + except (subprocess.SubprocessError, FileNotFoundError, OSError): + return None + + +__all__ = [ + "ExportResult", + "PROCESS_RUN_CRATE_PROFILE", + "PROVENANCE_RUN_CRATE_PROFILE", + "WORKFLOW_RUN_CRATE_PROFILE", + "WRROCBuilder", + "export_wrroc", +] diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 2e882e65..18177521 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -191,6 +191,7 @@ def test_write_manifest_basic(tmp_path: Path) -> None: "decisions": {"k": 1}, "code_version": "sha256:abc", "git_sha": "deadbeef", + "git_remote": "https://github.com/dkn16/test-repo", "lc_version": "0.4.1", }, ) @@ -207,6 +208,7 @@ def test_write_manifest_basic(tmp_path: Path) -> None: assert m["decisions"] == {"k": 1} assert m["code_version"] == "sha256:abc" assert m["git_sha"] == "deadbeef" + assert m["git_remote"] == "https://github.com/dkn16/test-repo" assert m["lc_version"] == "0.4.1" assert m["data_version"].startswith("sha256:") assert "raw_data" in m["input_versions"] diff --git a/tests/test_snakefile.py b/tests/test_snakefile.py index 88cf439d..62b4a065 100644 --- a/tests/test_snakefile.py +++ b/tests/test_snakefile.py @@ -509,3 +509,55 @@ def test_external_input_flows_to_manifest(tmp_path: Path) -> None: assert '"union21_table": Path(input.union21_table)' in text + + +# --------------------------------------------------------------------------- +# _git_remote() URL normalisation +# --------------------------------------------------------------------------- + + +class TestGitRemote: + """Probe ``_git_remote`` indirectly via subprocess monkeypatching. + + The function shells out to ``git config --get remote.origin.url``; + we capture the stdout it sees and check the normalisation. + """ + + @pytest.fixture + def git_remote(self, monkeypatch: pytest.MonkeyPatch): + import subprocess as sp + + from lightcone.engine import snakefile as sf + + def factory(stdout: str, returncode: int = 0): + class _R: + pass + r = _R() + r.stdout = stdout + r.returncode = returncode + r.stderr = "" + + def fake_run(*args: Any, **kwargs: Any): + return r + monkeypatch.setattr(sp, "run", fake_run) + return sf._git_remote(Path(".")) + + return factory + + def test_https_url_preserved(self, git_remote) -> None: + assert git_remote("https://github.com/dkn16/test\n") \ + == "https://github.com/dkn16/test" + + def test_ssh_url_normalised(self, git_remote) -> None: + assert git_remote("git@github.com:dkn16/test.git\n") \ + == "https://github.com/dkn16/test" + + def test_strips_dot_git_suffix(self, git_remote) -> None: + assert git_remote("https://gitlab.com/team/proj.git\n") \ + == "https://gitlab.com/team/proj" + + def test_returns_none_when_no_remote(self, git_remote) -> None: + assert git_remote("", returncode=1) is None + + def test_returns_none_when_empty_url(self, git_remote) -> None: + assert git_remote("\n") is None diff --git a/tests/test_wrroc.py b/tests/test_wrroc.py new file mode 100644 index 00000000..733fadda --- /dev/null +++ b/tests/test_wrroc.py @@ -0,0 +1,785 @@ +"""Tests for engine/wrroc.py — Workflow Run RO-Crate exporter.""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +import yaml +from click.testing import CliRunner + +from lightcone.cli.commands import main +from lightcone.engine.manifest import code_version, write_manifest +from lightcone.engine.wrroc import ( + PROVENANCE_RUN_CRATE_PROFILE, + ExportResult, + export_wrroc, +) + +# --------------------------------------------------------------------------- +# Fixtures: tiny project + materialized outputs +# --------------------------------------------------------------------------- + + +def _write_spec(project: Path, spec: dict[str, Any]) -> None: + project.mkdir(parents=True, exist_ok=True) + (project / "astra.yaml").write_text(yaml.safe_dump(spec)) + + +def _write_universe(project: Path, universe_id: str, decisions: dict[str, Any]) -> None: + udir = project / "universes" + udir.mkdir(parents=True, exist_ok=True) + (udir / f"{universe_id}.yaml").write_text( + yaml.safe_dump({"decisions": decisions}) + ) + + +def _materialize( + project: Path, + output_id: str, + universe_id: str, + *, + recipe: str, + decisions: dict[str, Any] | None = None, + container_image: str | None = None, + inputs: dict[str, Path] | None = None, + body: str = "output bytes", +) -> Path: + out = project / "results" / universe_id / output_id + out.mkdir(parents=True, exist_ok=True) + (out / "data.txt").write_text(body) + cv = code_version( + recipe=recipe, + container_image=container_image, + decisions=decisions or {}, + ) + write_manifest( + output_dir=out, + inputs=inputs or {}, + cfg={ + "output_id": output_id, + "universe_id": universe_id, + "recipe": recipe, + "container_image": container_image, + "decisions": decisions or {}, + "code_version": cv, + "git_sha": "abc1234", + "lc_version": "0.0.1", + }, + ) + return out + + +@pytest.fixture +def minimal_project(tmp_path: Path) -> Path: + """A project with one universe and one materialized output.""" + _write_spec( + tmp_path, + { + "name": "minimal", + "description": "test", + "outputs": [ + {"id": "foo", "recipe": {"command": "echo foo > data.txt"}}, + ], + }, + ) + _write_universe(tmp_path, "baseline", {}) + _materialize(tmp_path, "foo", "baseline", recipe="echo foo > data.txt") + return tmp_path + + +@pytest.fixture +def chained_project(tmp_path: Path) -> Path: + """Two-step DAG: step_b depends on step_a.""" + _write_spec( + tmp_path, + { + "name": "chained", + "description": "Two-step chained DAG for WRROC tests.", + # ASTRA's decisions schema: dict keyed by decision id, with + # options also a dict keyed by option id. + "decisions": { + "method": { + "label": "Method", + "default": "A", + "options": { + "A": {"label": "Option A"}, + "B": {"label": "Option B"}, + }, + }, + }, + "outputs": [ + {"id": "step_a", "recipe": {"command": "echo a > data.txt"}}, + { + "id": "step_b", + "inputs": ["step_a"], + "recipe": {"command": "cat data/step_a/data.txt > data.txt"}, + }, + ], + }, + ) + _write_universe(tmp_path, "baseline", {"method": "A"}) + + out_a = _materialize( + tmp_path, "step_a", "baseline", + recipe="echo a > data.txt", + decisions={"method": "A"}, + ) + _materialize( + tmp_path, "step_b", "baseline", + recipe="cat data/step_a/data.txt > data.txt", + decisions={"method": "A"}, + inputs={"step_a": out_a}, + ) + return tmp_path + + +# --------------------------------------------------------------------------- +# Module-level tests +# --------------------------------------------------------------------------- + + +class TestMinimalExport: + def test_returns_export_result(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + result = export_wrroc(minimal_project, out, author="Tester ") + assert isinstance(result, ExportResult) + assert result.bundle_path == out + assert result.runs_included == 1 + assert result.universes_included == ["baseline"] + assert result.is_zip is False + + def test_bundle_has_metadata_file(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="Tester ") + assert (out / "ro-crate-metadata.json").is_file() + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + assert "@context" in meta + assert "@graph" in meta + + def test_bundle_includes_astra_yaml(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="Tester ") + assert (out / "astra.yaml").is_file() + + def test_root_conforms_to_wrroc_profiles(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="Tester ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + root = next(g for g in meta["@graph"] if g["@id"] == "./") + conforms_ids = [c["@id"] for c in root["conformsTo"]] + assert PROVENANCE_RUN_CRATE_PROFILE in conforms_ids + + +class TestChainPreserved: + def test_step_b_object_references_step_a_dataset( + self, chained_project: Path + ) -> None: + out = chained_project / "wrroc" + export_wrroc(chained_project, out, author="Tester ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + + # Find step_b's CreateAction + actions = [g for g in meta["@graph"] if g.get("@type") == "CreateAction"] + step_b_action = next(a for a in actions if "step_b" in a["@id"]) + + # Its `object` list should include step_a's dataset @id + object_ids = [o["@id"] for o in step_b_action["object"]] + assert "results/baseline/step_a/" in object_ids + + def test_both_steps_have_create_actions(self, chained_project: Path) -> None: + out = chained_project / "wrroc" + result = export_wrroc(chained_project, out, author="X ") + assert result.runs_included == 2 + + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + actions = [g for g in meta["@graph"] if g.get("@type") == "CreateAction"] + ids = {a["@id"] for a in actions} + assert any("step_a" in i for i in ids) + assert any("step_b" in i for i in ids) + + +class TestDecisionsAttached: + def test_decisions_emitted_as_property_values( + self, chained_project: Path + ) -> None: + out = chained_project / "wrroc" + export_wrroc(chained_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + + pvs = [g for g in meta["@graph"] if g.get("@type") == "PropertyValue"] + method_pvs = [p for p in pvs if p.get("name") == "method"] + assert len(method_pvs) >= 1 + assert all(p["value"] == "A" for p in method_pvs) + + def test_complex_decision_value_is_serialized(self, tmp_path: Path) -> None: + """Non-primitive decision values must be JSON-serialized for + PropertyValue.value compatibility. + """ + _write_spec( + tmp_path, + { + "outputs": [ + {"id": "foo", "recipe": {"command": "echo foo"}}, + ] + }, + ) + _write_universe(tmp_path, "u1", {"opts": {"a": 1, "b": [2, 3]}}) + _materialize( + tmp_path, "foo", "u1", + recipe="echo foo", + decisions={"opts": {"a": 1, "b": [2, 3]}}, + ) + + export_wrroc(tmp_path, tmp_path / "wrroc", author="X ") + meta = json.loads((tmp_path / "wrroc" / "ro-crate-metadata.json").read_text()) + opts_pv = next( + g for g in meta["@graph"] + if g.get("@type") == "PropertyValue" and g.get("name") == "opts" + ) + # Coerced to a JSON string + assert isinstance(opts_pv["value"], str) + assert json.loads(opts_pv["value"]) == {"a": 1, "b": [2, 3]} + + +class TestRoundTrip: + def test_load_via_rocrate_py(self, chained_project: Path) -> None: + """A bundle we wrote must be loadable by rocrate.Crate(path).""" + from rocrate.rocrate import ROCrate + + out = chained_project / "wrroc" + export_wrroc(chained_project, out, author="X ") + + crate = ROCrate(out) + assert crate.name == "chained" + actions = crate.get_by_type("CreateAction") + assert len(actions) == 2 + + def test_workflow_is_main_entity(self, chained_project: Path) -> None: + from rocrate.rocrate import ROCrate + + out = chained_project / "wrroc" + export_wrroc(chained_project, out, author="X ") + crate = ROCrate(out) + assert crate.mainEntity is not None + assert crate.mainEntity.id == "astra.yaml" + + +class TestMetadataOnly: + def test_skips_data_files(self, chained_project: Path) -> None: + out = chained_project / "wrroc" + export_wrroc( + chained_project, out, author="X ", include_data=False, + ) + # data.txt files should NOT be copied + assert not (out / "results" / "baseline" / "step_a" / "data.txt").exists() + # but manifests SHOULD be + assert ( + out / "results" / "baseline" / "step_a" / ".lightcone-manifest.json" + ).is_file() + + def test_chain_still_valid_in_metadata(self, chained_project: Path) -> None: + """Even without data files, the @id chain must still link upstream.""" + out = chained_project / "wrroc" + export_wrroc( + chained_project, out, author="X ", include_data=False, + ) + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + actions = [g for g in meta["@graph"] if g.get("@type") == "CreateAction"] + step_b = next(a for a in actions if "step_b" in a["@id"]) + ids = [o["@id"] for o in step_b["object"]] + assert "results/baseline/step_a/" in ids + + +class TestZipBundle: + def test_produces_zip(self, minimal_project: Path) -> None: + zip_path = minimal_project / "bundle.zip" + result = export_wrroc( + minimal_project, zip_path, author="X ", zip_bundle=True, + ) + assert result.is_zip is True + assert zip_path.is_file() + # Contains ro-crate-metadata.json + import zipfile + with zipfile.ZipFile(zip_path) as zf: + names = zf.namelist() + assert any("ro-crate-metadata.json" in n for n in names) + + def test_zip_raises_on_existing_directory(self, minimal_project: Path) -> None: + dir_path = minimal_project / "existing_dir" + dir_path.mkdir() + with pytest.raises(FileExistsError, match="existing directory"): + export_wrroc(minimal_project, dir_path, author="X ", zip_bundle=True) + + +class TestAuthor: + def test_explicit_author_overrides(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="Alice ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + persons = [g for g in meta["@graph"] if g.get("@type") == "Person"] + assert len(persons) == 1 + assert persons[0]["name"] == "Alice" + assert persons[0]["email"] == "a@b.c" + + def test_author_used_as_action_agent(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="Alice ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + action = next(g for g in meta["@graph"] if g.get("@type") == "CreateAction") + assert action["agent"]["@id"] == "#author-a_at_b.c" + + +class TestUniverseFilter: + def test_restricts_to_listed_universes(self, tmp_path: Path) -> None: + _write_spec( + tmp_path, + { + "outputs": [ + {"id": "foo", "recipe": {"command": "echo foo"}}, + ] + }, + ) + _write_universe(tmp_path, "u1", {}) + _write_universe(tmp_path, "u2", {}) + _materialize(tmp_path, "foo", "u1", recipe="echo foo") + _materialize(tmp_path, "foo", "u2", recipe="echo foo") + + result = export_wrroc( + tmp_path, tmp_path / "wrroc", + universes=["u1"], author="X ", + ) + assert result.universes_included == ["u1"] + assert result.runs_included == 1 + + +class TestEmptyProject: + def test_no_materializations_warns_but_succeeds( + self, tmp_path: Path, + ) -> None: + _write_spec( + tmp_path, + { + "outputs": [ + {"id": "foo", "recipe": {"command": "echo foo"}}, + ] + }, + ) + _write_universe(tmp_path, "u1", {}) + # No materialization + + result = export_wrroc(tmp_path, tmp_path / "wrroc", author="X ") + assert result.runs_included == 0 + # Bundle still has the workflow definition + assert (tmp_path / "wrroc" / "astra.yaml").is_file() + + +class TestRefuseClobber: + def test_non_empty_target_dir_errors(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + out.mkdir() + (out / "existing.txt").write_text("hi") + with pytest.raises(FileExistsError): + export_wrroc(minimal_project, out, author="X ") + + +class TestSubAnalyses: + """Sub-analysis outputs must be captured with the correct path-rooted + `@id`s and the chain back to root outputs preserved. + """ + + @pytest.fixture + def subanalysis_project(self, tmp_path: Path) -> Path: + # Root project declares a sub-analysis at analyses/sub/ + _write_spec( + tmp_path, + { + "name": "with-subs", + "description": "Project with one sub-analysis.", + "outputs": [ + {"id": "root_out", "recipe": {"command": "echo r"}}, + ], + "analyses": { + "sub": {"path": "./analyses/sub"}, + }, + }, + ) + # Sub-analysis has its own astra.yaml. + sub_dir = tmp_path / "analyses" / "sub" + sub_dir.mkdir(parents=True) + (sub_dir / "astra.yaml").write_text(yaml.safe_dump({ + "name": "sub", + "description": "Sub-analysis.", + "outputs": [ + {"id": "sub_out", "recipe": {"command": "echo s"}}, + ], + })) + _write_universe(tmp_path, "baseline", {}) + + # Materialize root output at /results/baseline/root_out/ + _materialize(tmp_path, "root_out", "baseline", recipe="echo r") + + # Materialize sub-analysis output at /results/baseline/sub_out/ + sub_out_dir = sub_dir / "results" / "baseline" / "sub_out" + sub_out_dir.mkdir(parents=True) + (sub_out_dir / "data.txt").write_text("sub bytes") + cv = code_version(recipe="echo s", container_image=None, decisions={}) + write_manifest( + output_dir=sub_out_dir, + inputs={}, + cfg={"output_id": "sub_out", "universe_id": "baseline", + "recipe": "echo s", "container_image": None, + "decisions": {}, "code_version": cv, + "git_sha": "abc", "lc_version": "0.0.1"}, + ) + return tmp_path + + def test_both_root_and_sub_outputs_captured( + self, subanalysis_project: Path, + ) -> None: + out = subanalysis_project / "wrroc" + result = export_wrroc(subanalysis_project, out, author="X ") + assert result.runs_included == 2 + + def test_sub_dataset_id_includes_sub_path( + self, subanalysis_project: Path, + ) -> None: + out = subanalysis_project / "wrroc" + export_wrroc(subanalysis_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + ids = {g["@id"] for g in meta["@graph"]} + # Root dataset uses results/// + assert "results/baseline/root_out/" in ids + # Sub-analysis dataset uses /results/// + assert "analyses/sub/results/baseline/sub_out/" in ids + + def test_sub_create_action_id_qualified( + self, subanalysis_project: Path, + ) -> None: + """CreateAction @ids include the analysis_id qualifier so sub + and root outputs with the same id never collide. + """ + out = subanalysis_project / "wrroc" + export_wrroc(subanalysis_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + action_ids = { + g["@id"] for g in meta["@graph"] + if g.get("@type") == "CreateAction" + } + assert "#run-baseline-root_out" in action_ids + assert "#run-baseline-sub.sub_out" in action_ids + + def test_sub_data_files_bundled( + self, subanalysis_project: Path, + ) -> None: + out = subanalysis_project / "wrroc" + export_wrroc(subanalysis_project, out, author="X ") + # Sub-analysis data file should be copied at the corresponding + # path inside the bundle. + assert ( + out / "analyses" / "sub" / "results" / "baseline" + / "sub_out" / "data.txt" + ).is_file() + + +class TestToolName: + """SoftwareApplication.name resolution: tool_name > heuristic > output_id.""" + + def _project_with_recipe( + self, + tmp_path: Path, + *, + recipe_command: str, + tool_name: str | None = None, + ) -> Path: + recipe: dict[str, Any] = {"command": recipe_command} + if tool_name: + recipe["tool_name"] = tool_name + _write_spec( + tmp_path, + { + "outputs": [{"id": "foo", "recipe": recipe}], + }, + ) + _write_universe(tmp_path, "u1", {}) + _materialize(tmp_path, "foo", "u1", recipe=recipe_command) + return tmp_path + + def _software_app(self, bundle: Path) -> dict[str, Any]: + meta = json.loads((bundle / "ro-crate-metadata.json").read_text()) + return next( + g for g in meta["@graph"] + if g.get("@type") == "SoftwareApplication" + and g["@id"].startswith("#recipe") + ) + + def test_explicit_tool_name_wins(self, tmp_path: Path) -> None: + project = self._project_with_recipe( + tmp_path, + recipe_command="python scripts/analyze.py --x 1", + tool_name="analyze (chi-squared)", + ) + out = project / "wrroc" + export_wrroc(project, out, author="X ") + sw = self._software_app(out) + assert sw["name"] == "analyze (chi-squared)" + assert sw["description"] == "python scripts/analyze.py --x 1" + + def test_heuristic_extracts_script_path(self, tmp_path: Path) -> None: + project = self._project_with_recipe( + tmp_path, + recipe_command="python scripts/analyze.py --x 1", + ) + out = project / "wrroc" + export_wrroc(project, out, author="X ") + sw = self._software_app(out) + assert sw["name"] == "scripts/analyze.py" + + def test_falls_back_to_output_id(self, tmp_path: Path) -> None: + # Recipe with no script-like token in it + project = self._project_with_recipe( + tmp_path, + recipe_command="echo hello", + ) + out = project / "wrroc" + export_wrroc(project, out, author="X ") + sw = self._software_app(out) + assert sw["name"] == "foo" # the output id + + +class TestGitRemote: + def test_emits_code_repository_entity(self, tmp_path: Path) -> None: + """When manifests carry git_remote, the bundle gets a CodeRepository.""" + _write_spec( + tmp_path, + {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, + ) + _write_universe(tmp_path, "u1", {}) + out = tmp_path / "results" / "u1" / "foo" + out.mkdir(parents=True) + (out / "data.txt").write_text("bytes") + cv = code_version(recipe="echo foo", container_image=None, decisions={}) + write_manifest( + output_dir=out, inputs={}, + cfg={ + "output_id": "foo", "universe_id": "u1", + "recipe": "echo foo", "container_image": None, + "decisions": {}, "code_version": cv, + "git_sha": "abc", + "git_remote": "https://github.com/dkn16/test-repo", + "lc_version": "0.0.1", + }, + ) + + bundle = tmp_path / "wrroc" + export_wrroc(tmp_path, bundle, author="X ") + meta = json.loads((bundle / "ro-crate-metadata.json").read_text()) + + repos = [ + g for g in meta["@graph"] + if "CodeRepository" in ( + g["@type"] if isinstance(g["@type"], list) else [g["@type"]] + ) + ] + assert len(repos) == 1 + assert repos[0]["@id"] == "https://github.com/dkn16/test-repo" + assert repos[0]["url"] == "https://github.com/dkn16/test-repo" + + wf = next( + g for g in meta["@graph"] + if "ComputationalWorkflow" in ( + g["@type"] if isinstance(g["@type"], list) else [g["@type"]] + ) + ) + assert wf.get("codeRepository", {}).get("@id") == \ + "https://github.com/dkn16/test-repo" + + def test_no_git_remote_no_repo_entity(self, minimal_project: Path) -> None: + """Without git_remote in manifests, no CodeRepository is emitted.""" + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + for g in meta["@graph"]: + t = g.get("@type") + tlist = t if isinstance(t, list) else [t] + assert "CodeRepository" not in tlist + + +class TestUnreadableManifest: + def test_skips_permission_denied( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + """If a manifest read raises OSError (permission, broken symlink), + the exporter must warn and skip rather than abort the whole run. + Mirrors what happens with cross-user symlinked results dirs. + """ + from lightcone.engine import wrroc as wrroc_mod + + _write_spec( + tmp_path, + {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, + ) + _write_universe(tmp_path, "u1", {}) + # No manifest exists, but inject one that raises PermissionError. + from lightcone.engine import manifest as manifest_mod + + def boom(out_dir: Path) -> dict[str, Any] | None: + raise PermissionError(f"denied: {out_dir}") + + monkeypatch.setattr(manifest_mod, "read_manifest", boom) + # wrroc.py imports read_manifest by name, so patch there too. + monkeypatch.setattr(wrroc_mod, "read_manifest", boom) + + # Should not raise; should warn and produce a (mostly empty) bundle. + result = export_wrroc(tmp_path, tmp_path / "wrroc", author="X ") + assert result.runs_included == 0 + + +class TestProfileConformance: + """The bundle's @graph must declare profile CreativeWork entities, + set a license, and include FormalParameter additionalType — the + Provenance Run Crate 0.5 validator's REQUIRED checks all hinge on + these. + """ + + def test_root_has_license(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + root = next(g for g in meta["@graph"] if g["@id"] == "./") + assert "license" in root + assert root["license"]["@id"].startswith("http") + + def test_explicit_license_passed_through(self, minimal_project: Path) -> None: + out = minimal_project / "wrroc" + export_wrroc( + minimal_project, out, author="X ", + license="https://opensource.org/licenses/MIT", + ) + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + root = next(g for g in meta["@graph"] if g["@id"] == "./") + assert root["license"]["@id"] == "https://opensource.org/licenses/MIT" + + def test_profile_creativework_entities_declared( + self, minimal_project: Path, + ) -> None: + """conformsTo profile URLs must each have a CreativeWork entity.""" + out = minimal_project / "wrroc" + export_wrroc(minimal_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + ids = {g["@id"] for g in meta["@graph"]} + assert PROVENANCE_RUN_CRATE_PROFILE in ids + + def test_formal_parameters_have_additional_type( + self, chained_project: Path, + ) -> None: + out = chained_project / "wrroc" + export_wrroc(chained_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + params = [g for g in meta["@graph"] if g.get("@type") == "FormalParameter"] + assert len(params) >= 1 + for p in params: + assert "additionalType" in p + assert p["additionalType"]["@id"].startswith("http://schema.org/") + + def test_workflow_haspart_recipes(self, chained_project: Path) -> None: + """ComputationalWorkflow MUST link recipes via hasPart.""" + out = chained_project / "wrroc" + export_wrroc(chained_project, out, author="X ") + meta = json.loads((out / "ro-crate-metadata.json").read_text()) + wf = next( + g for g in meta["@graph"] + if "ComputationalWorkflow" in ( + g["@type"] if isinstance(g["@type"], list) else [g["@type"]] + ) + ) + has_part = wf.get("hasPart") or [] + recipe_refs = [hp["@id"] for hp in has_part if hp["@id"].startswith("#recipe-")] + assert len(recipe_refs) >= 2 # both step_a and step_b recipes + + +# --------------------------------------------------------------------------- +# CLI tests +# --------------------------------------------------------------------------- + + +class TestCli: + def test_export_wrroc_help(self) -> None: + runner = CliRunner() + result = runner.invoke(main, ["export", "wrroc", "--help"]) + assert result.exit_code == 0 + assert "WRROC" in result.output + assert "--zip" in result.output + assert "--metadata-only" in result.output + + def test_export_wrroc_runs( + self, minimal_project: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.chdir(minimal_project) + runner = CliRunner() + result = runner.invoke( + main, + [ + "export", "wrroc", + "-o", "out-dir", + "--author", "Tester ", + ], + ) + assert result.exit_code == 0, result.output + assert (minimal_project / "out-dir" / "ro-crate-metadata.json").is_file() + assert "Wrote WRROC" in result.output + + def test_export_wrroc_zip( + self, minimal_project: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.chdir(minimal_project) + runner = CliRunner() + result = runner.invoke( + main, + [ + "export", "wrroc", + "-o", "bundle.zip", + "--zip", + "--author", "Tester ", + ], + ) + assert result.exit_code == 0, result.output + assert (minimal_project / "bundle.zip").is_file() + + def test_export_wrroc_metadata_only( + self, minimal_project: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.chdir(minimal_project) + runner = CliRunner() + result = runner.invoke( + main, + [ + "export", "wrroc", + "-o", "meta", + "--metadata-only", + "--author", "Tester ", + ], + ) + assert result.exit_code == 0, result.output + # Manifest yes, data no + out_dir = minimal_project / "meta" / "results" / "baseline" / "foo" + assert (out_dir / ".lightcone-manifest.json").is_file() + assert not (out_dir / "data.txt").exists() + + def test_export_wrroc_no_runs_warns( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + _write_spec( + tmp_path, + {"outputs": [{"id": "foo", "recipe": {"command": "echo foo"}}]}, + ) + _write_universe(tmp_path, "u1", {}) + monkeypatch.chdir(tmp_path) + runner = CliRunner() + result = runner.invoke( + main, + ["export", "wrroc", "-o", "out", "--author", "X "], + ) + assert result.exit_code == 0, result.output + assert "no materialized outputs" in result.output.lower() diff --git a/zensical.toml b/zensical.toml index d5205537..d3cc8b4c 100644 --- a/zensical.toml +++ b/zensical.toml @@ -28,6 +28,7 @@ nav = [ {"lc build" = "cli/build.md"}, {"lc status" = "cli/status.md"}, {"lc verify" = "cli/verify.md"}, + {"lc export" = "cli/export.md"}, {"lc setup" = "cli/setup.md"}, ]}, {"Python API" = [