Skip to content

Commit 2dcab80

Browse files
committed
feat(cli): add validation profiles and shared reports
1 parent e660c8c commit 2dcab80

3 files changed

Lines changed: 176 additions & 5 deletions

File tree

docs/source/reference/cli.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,28 @@ openenv import path/to/source --name my_env --output-dir ./envs --env-class MyEn
4040

4141
## `openenv validate`
4242

43+
Use an explicit profile to produce the RFC 008 validation plan and shared
44+
report schema:
45+
46+
```bash
47+
openenv validate path/to/env --profile static
48+
openenv validate path/to/env --profile runtime --json
49+
openenv validate --url http://127.0.0.1:8000 --profile runtime --json
50+
openenv validate path/to/env --profile full --output validation.json
51+
```
52+
53+
`static` checks source and packaging, `runtime` adds a launched or connected
54+
server, and `full` records every policy criterion while marking unavailable
55+
remote capabilities as skipped. Local reports never claim official
56+
certification. Automatic local launch executes the environment checkout as the
57+
current user and is intended only for trusted development source; use `--url`
58+
for a server you already isolated.
59+
60+
This command intentionally targets the served OpenEnv spec. Shared reports
61+
record spec, adapter, and execution-model provenance, but `openenv validate`
62+
does not auto-dispatch external task-package formats. A separate spec-selected
63+
task workflow is tracked in [issue #898](https://github.com/huggingface/OpenEnv/issues/898).
64+
4365
[[autodoc]] openenv.cli.commands.validate.validate
4466

4567
## `openenv push`

src/openenv/cli/commands/validate.py

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
validate_multi_mode_deployment,
2020
validate_running_environment,
2121
)
22+
from openenv.validation import (
23+
format_shared_validation_report,
24+
run_local_validation,
25+
ValidationProfile,
26+
)
2227

2328

2429
def _looks_like_url(value: str) -> bool:
@@ -48,9 +53,23 @@ def validate(
4853
bool,
4954
typer.Option(
5055
"--json",
51-
help="Output local validation report as JSON (runtime validation is JSON by default)",
56+
help="Output the RFC 008 shared validation report as JSON",
5257
),
5358
] = False,
59+
profile: Annotated[
60+
str | None,
61+
typer.Option(
62+
"--profile",
63+
help="Validation profile: static, runtime, or full",
64+
),
65+
] = None,
66+
output: Annotated[
67+
Path | None,
68+
typer.Option(
69+
"--output",
70+
help="Write the shared JSON validation report to this path",
71+
),
72+
] = None,
5473
timeout: Annotated[
5574
float,
5675
typer.Option(
@@ -74,6 +93,11 @@ def validate(
7493
7594
Runtime validation checks if a live OpenEnv server conforms to the
7695
versioned runtime API contract and returns a criteria-based JSON report.
96+
Explicit static, runtime, and full profiles emit the RFC 008 shared report.
97+
Reports identify the served OpenEnv spec and pinned adapter; external task
98+
package formats are intentionally outside this command's dispatch surface.
99+
Automatic runtime launch is intended for trusted local source; connect to
100+
an already isolated server with `--url` for untrusted environments.
77101
78102
Examples:
79103
@@ -91,6 +115,9 @@ def validate(
91115
92116
# Validate specific environment
93117
$ openenv validate envs/echo_env
118+
119+
# Run every locally available check and record remote-only skips
120+
$ openenv validate envs/echo_env --profile full --output report.json
94121
```
95122
"""
96123
runtime_target = url
@@ -114,6 +141,80 @@ def validate(
114141
raise typer.Exit(1)
115142
runtime_target = target
116143

144+
# Machine-readable and explicit-profile invocations use the RFC 008 shared
145+
# report. Only the unqualified human rendering stays on the legacy path.
146+
if profile is not None or output is not None or json_output:
147+
if profile is None:
148+
selected_profile = (
149+
ValidationProfile.RUNTIME
150+
if runtime_target is not None
151+
else ValidationProfile.STATIC
152+
)
153+
else:
154+
try:
155+
selected_profile = ValidationProfile(profile.lower())
156+
except ValueError as exc:
157+
typer.echo(
158+
"Error: --profile must be one of: static, runtime, full",
159+
err=True,
160+
)
161+
raise typer.Exit(1) from exc
162+
163+
if runtime_target is not None and selected_profile is ValidationProfile.STATIC:
164+
typer.echo(
165+
"Error: The static profile requires a local source directory",
166+
err=True,
167+
)
168+
raise typer.Exit(1)
169+
170+
if runtime_target is not None:
171+
shared_target: str | Path = runtime_target
172+
shared_runtime_url = runtime_target
173+
else:
174+
shared_target = Path.cwd() if target is None else Path(target)
175+
shared_runtime_url = None
176+
if not shared_target.exists():
177+
typer.echo(f"Error: Path does not exist: {shared_target}", err=True)
178+
raise typer.Exit(1)
179+
if not shared_target.is_dir():
180+
typer.echo(f"Error: Path is not a directory: {shared_target}", err=True)
181+
raise typer.Exit(1)
182+
183+
try:
184+
report = run_local_validation(
185+
shared_target,
186+
profile=selected_profile,
187+
runtime_url=shared_runtime_url,
188+
timeout_s=timeout,
189+
)
190+
except ValueError as exc:
191+
typer.echo(f"Error: {exc}", err=True)
192+
raise typer.Exit(1) from exc
193+
payload = report.to_dict()
194+
serialized = json.dumps(payload, indent=2)
195+
196+
if output is not None:
197+
try:
198+
output.parent.mkdir(parents=True, exist_ok=True)
199+
output.write_text(f"{serialized}\n", encoding="utf-8")
200+
except OSError as exc:
201+
typer.echo(
202+
f"Error: Unable to write validation report: {type(exc).__name__}",
203+
err=True,
204+
)
205+
raise typer.Exit(1) from exc
206+
207+
if json_output:
208+
typer.echo(serialized)
209+
else:
210+
typer.echo(format_shared_validation_report(report))
211+
if output is not None:
212+
typer.echo(f"Report written to {output}")
213+
214+
if not report.passed:
215+
raise typer.Exit(1)
216+
return
217+
117218
if runtime_target is not None:
118219
try:
119220
report = validate_running_environment(runtime_target, timeout_s=timeout)

tests/test_cli/test_validate.py

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ def _write_minimal_valid_env(
4141
(env_dir / "openenv.yaml").write_text(
4242
"spec_version: 1\nname: test_env\ntype: space\nruntime: fastapi\napp: server.app:app\nport: 8000\n"
4343
)
44-
(env_dir / "uv.lock").write_text("")
44+
(env_dir / "uv.lock").write_text(
45+
'version = 1\nrevision = 1\nrequires-python = ">=3.10"\n'
46+
)
4547
(env_dir / "pyproject.toml").write_text(
4648
"[project]\n"
4749
'name = "test-env"\n'
@@ -54,6 +56,7 @@ def _write_minimal_valid_env(
5456
(env_dir / "server" / "app.py").write_text(
5557
f"{main_signature}\n return None\n\nif __name__ == '__main__':\n {main_invocation}\n"
5658
)
59+
(env_dir / "server" / "Dockerfile").write_text("FROM python:3.12-slim\n")
5760

5861

5962
def test_validate_running_environment_success() -> None:
@@ -194,6 +197,16 @@ def test_validate_command_runtime_target_outputs_json() -> None:
194197
mock_validate.assert_called_once_with("https://example.com", timeout_s=5.0)
195198

196199

200+
def test_validate_shared_profile_reports_invalid_runtime_url() -> None:
201+
result = runner.invoke(
202+
app,
203+
["validate", "--url", "http://[::1", "--profile", "runtime", "--json"],
204+
)
205+
206+
assert result.exit_code == 1
207+
assert "Error: Invalid runtime URL" in result.output
208+
209+
197210
def test_validate_command_local_path_still_works(tmp_path: Path) -> None:
198211
"""CLI local validation remains backward compatible."""
199212
env_dir = tmp_path / "test_env"
@@ -214,13 +227,47 @@ def test_validate_command_local_json_output(tmp_path: Path) -> None:
214227

215228
assert result.exit_code == 0
216229
payload = json.loads(result.output)
217-
assert payload["validation_type"] == "local_environment"
230+
assert payload["validation_type"] == "openenv_validation"
231+
assert payload["report_schema_version"] == "1.0"
232+
assert payload["profile"] == "static"
233+
assert payload["spec"]["id"] == "openenv"
234+
assert payload["spec"]["adapter"] == {
235+
"id": "openenv-yaml",
236+
"version": "1",
237+
}
238+
assert payload["spec"]["execution_model"] == "served"
218239
assert payload["passed"] is True
219-
assert payload["summary"]["passed_count"] == 1
220-
assert payload["summary"]["total_count"] == 1
240+
assert payload["summary"]["passed_count"] >= 1
241+
assert payload["summary"]["total_count"] >= 1
221242
assert payload["summary"]["failed_criteria"] == []
222243

223244

245+
def test_profile_json_can_be_written_to_output_file(tmp_path: Path) -> None:
246+
env_dir = tmp_path / "test_env"
247+
report_path = tmp_path / "reports" / "validation.json"
248+
_write_minimal_valid_env(env_dir)
249+
250+
result = runner.invoke(
251+
app,
252+
[
253+
"validate",
254+
str(env_dir),
255+
"--profile",
256+
"static",
257+
"--json",
258+
"--output",
259+
str(report_path),
260+
],
261+
)
262+
263+
assert result.exit_code == 0, result.output
264+
stdout_payload = json.loads(result.output)
265+
file_payload = json.loads(report_path.read_text())
266+
assert file_payload == stdout_payload
267+
assert file_payload["profile"] == "static"
268+
assert file_payload["report_schema_version"] == "1.0"
269+
270+
224271
def test_validate_command_rejects_environment_package_as_runtime_dependency(
225272
tmp_path: Path,
226273
) -> None:
@@ -259,6 +306,7 @@ def test_validate_command_accepts_dockerfile_managed_openenv_runtime(
259306
'server = "server.app:main"\n'
260307
)
261308
(env_dir / "server" / "Dockerfile").write_text(
309+
"FROM python:3.12-slim\n"
262310
'RUN pip install --no-cache-dir --no-deps "openenv[core]>=0.2.2"\n'
263311
)
264312

0 commit comments

Comments
 (0)