[WIP] Ship BR schema as standalone llmdbenchmark-schema package

llmdbenchmark-schema/README.md

@@ -0,0 +1,42 @@
+# llmdbenchmark-schema
+
+Pydantic types for the **Benchmark Report (BR)** format used by
+[llm-d-benchmark](https://github.com/llm-d/llm-d-benchmark).
+
+Published as a standalone distribution so that **external benchmark
+tooling** (workload generators, dashboards, analysis pipelines) can
+depend on just the schema without pulling in the rest of `llmdbenchmark`
+(which carries Kubernetes/transformers/HF deps).
+
+## Install
+
+```bash
+pip install llmdbenchmark-schema
+```
+
+Runtime deps: `pydantic>=2.0`, `PyYAML`. That's it.
+
+## Use
+
+```python
+from llmdbenchmark_schema import BenchmarkReportV02
+
+with open("report.yaml") as f:
+    import yaml
+    BenchmarkReportV02.model_validate(yaml.safe_load(f))
+```
+
+The full `llmdbenchmark` package re-exports these types under
+`llmdbenchmark.analysis.benchmark_report` for back-compat, so existing
+code does not need to change.
+
+## Versioning
+
+The package version tracks the latest *schema* version it ships. Older
+schema versions remain accessible (e.g. `BenchmarkReportV01`).
+
+## Source
+
+This package's source lives inside the
+[llm-d-benchmark](https://github.com/llm-d/llm-d-benchmark) repository,
+under `llmdbenchmark-schema/`. Changes are made there.

llmdbenchmark-schema/llmdbenchmark_schema/__init__.py

@@ -0,0 +1,22 @@
+"""llmdbenchmark-schema: BR (Benchmark Report) schema types.
+
+This is the schema-only subset of ``llmdbenchmark``, published as a
+standalone distribution so external benchmark tooling can depend on just
+the report types without pulling in the rest of ``llmdbenchmark``
+(kubernetes client, transformers, etc.).
+
+The full ``llmdbenchmark`` package re-exports these types under
+``llmdbenchmark.analysis.benchmark_report`` for back-compat.
+"""
+
+from .base import BenchmarkReport, Units, WorkloadGenerator
+from .schema_v0_1 import BenchmarkReportV01
+from .schema_v0_2 import BenchmarkReportV02
+
+__all__ = [
+    "BenchmarkReport",
+    "BenchmarkReportV01",
+    "BenchmarkReportV02",
+    "Units",
+    "WorkloadGenerator",
+]

llmdbenchmark-schema/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "llmdbenchmark-schema"
+version = "0.2.0"
+description = "Benchmark Report (BR) schema types for llm-d-benchmark."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "pydantic>=2.0",
+    "PyYAML",
+]
+license = {text = "Apache-2.0"}
+authors = [{name = "llm-d-benchmark contributors"}]
+keywords = ["benchmarking", "llm", "schema", "llm-d"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Typing :: Typed",
+]
+
+[project.urls]
+Homepage = "https://github.com/llm-d/llm-d-benchmark"
+Source = "https://github.com/llm-d/llm-d-benchmark"
+
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["llmdbenchmark_schema*"]
+
+[tool.setuptools.package-data]
+"llmdbenchmark_schema" = [
+    "*.yaml",
+    "*.json",
+    "py.typed",
+]

llmdbenchmark/analysis/benchmark_report/__init__.py

@@ -1,8 +1,20 @@
 """
 Benchmark Report standardized reporting format.
+
+Schema types now live in the standalone ``llmdbenchmark-schema`` package
+(distributed independently so external benchmark tooling can depend on
+just the schema without pulling in the rest of ``llmdbenchmark``).
+
+This module re-exports the schema types and bundles them with the
+analysis utilities (``core``, ``cli``, ``native_to_*``) so existing
+``from llmdbenchmark.analysis.benchmark_report import ...`` callers keep
+working.
 """
 
-from .base import BenchmarkReport
+from llmdbenchmark_schema.base import BenchmarkReport
+from llmdbenchmark_schema.schema_v0_1 import BenchmarkReportV01
+from llmdbenchmark_schema.schema_v0_2 import BenchmarkReportV02
+
 from .core import (
     get_nested,
     import_benchmark_report,
@@ -12,8 +24,6 @@
     update_dict,
     yaml_str_to_benchmark_report,
 )
-from .schema_v0_1 import BenchmarkReportV01
-from .schema_v0_2 import BenchmarkReportV02
 
 __all__ = [
     "BenchmarkReport",

llmdbenchmark/analysis/benchmark_report/cli.py

@@ -15,7 +15,7 @@
 import sys
 
 from . import make_json_schema
-from .base import WorkloadGenerator
+from llmdbenchmark_schema.base import WorkloadGenerator
 
 
 def main() -> None:

llmdbenchmark/analysis/benchmark_report/core.py

@@ -10,9 +10,9 @@
 import yaml
 import numpy as np
 
-from .base import BenchmarkReport
-from .schema_v0_1 import BenchmarkReportV01
-from .schema_v0_2 import BenchmarkReportV02
+from llmdbenchmark_schema.base import BenchmarkReport
+from llmdbenchmark_schema.schema_v0_1 import BenchmarkReportV01
+from llmdbenchmark_schema.schema_v0_2 import BenchmarkReportV02
 
 
 def check_file(file_path: str) -> None:

llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py

@@ -12,15 +12,15 @@
 
 import numpy as np
 
-from .base import Units
+from llmdbenchmark_schema.base import Units
 from .core import (
     check_file,
     get_nested,
     import_yaml,
     load_benchmark_report,
     update_dict,
 )
-from .schema_v0_1 import BenchmarkReportV01, HostType, WorkloadGenerator
+from llmdbenchmark_schema.schema_v0_1 import BenchmarkReportV01, HostType, WorkloadGenerator
 
 
 def _get_llmd_benchmark_envars() -> dict:

llmdbenchmark/analysis/benchmark_report/native_to_br0_2.py

@@ -18,16 +18,16 @@
 import numpy as np
 import yaml
 
-from .base import Units, WorkloadGenerator
+from llmdbenchmark_schema.base import Units, WorkloadGenerator
 from .core import (
     check_file,
     get_nested,
     import_yaml,
     load_benchmark_report,
     update_dict,
 )
-from .schema_v0_2 import BenchmarkReportV02, Component, Distribution, LoadSource
-from .schema_v0_2_components import HostType
+from llmdbenchmark_schema.schema_v0_2 import BenchmarkReportV02, Component, Distribution, LoadSource
+from llmdbenchmark_schema.schema_v0_2_components import HostType
 
 
 def _load_run_metadata() -> dict:

pyproject.toml

@@ -17,6 +17,9 @@ dependencies = [
     "pydantic>=2.0",
     "google-auth",
     "google-cloud-storage>=2.10.0",
+    # Schema types live in their own distribution so external benchmark
+    # tooling can depend on just the schema. See ``llmdbenchmark-schema/``.
+    "llmdbenchmark-schema",
 ]
 
 #
@@ -29,14 +32,12 @@ dependencies = [
 [tool.setuptools.packages.find]
 where = ["."]
 include = ["llmdbenchmark*"]
-exclude = ["templates*", "specification*"]
+exclude = ["templates*", "specification*", "llmdbenchmark_schema*"]
 
 [tool.setuptools.package-data]
 "llmdbenchmark.analysis" = [
     "scripts/*.sh",
     "scripts/*.py",
-    "benchmark_report/*.yaml",
-    "benchmark_report/*.json",
 ]
 
 [project.urls]