Move most logic into separate scripts / workflows

Author: chmp

.github/workflows/bench.yml

@@ -8,7 +8,7 @@
     "CARGO_TERM_COLOR": "always"
   },
   "jobs": {
-    "build": {
+    "bench": {
       "runs-on": "ubuntu-latest",
       "steps": [
         {
@@ -24,7 +24,11 @@
         },
         {
           "name": "Bench",
-          "run": "cargo bench -p serde_arrow_bench\npython scripts/analyze-benchmark.py --criterion-root target/criterion --readme Readme.md --plot-output timings.png >> $GITHUB_STEP_SUMMARY"
+          "run": "cargo bench -p serde_arrow_bench"
+        },
+        {
+          "name": "Summarize",
+          "run": "python scripts/analyze-benchmark.py --criterion-root target/criterion --update-github-summary"
         }
       ]
     }

.github/workflows/local.yml

@@ -65,7 +65,7 @@
         },
         {
           "name": "Check Cargo.toml",
-          "run": "uv run python scripts/check-cargo-toml.py --arrow-features arrow-59 arrow-58 arrow-57 arrow-56 arrow-55 arrow-54 arrow-53"
+          "run": "uv run python scripts/check-cargo-toml.py"
         },
         {
           "name": "Check workspace",

.github/workflows/test.yml

@@ -74,7 +74,7 @@
         },
         {
           "name": "Check Cargo.toml",
-          "run": "uv run python scripts/check-cargo-toml.py --arrow-features arrow-59 arrow-58 arrow-57 arrow-56 arrow-55 arrow-54 arrow-53"
+          "run": "uv run python scripts/check-cargo-toml.py"
         },
         {
           "name": "Check",
@@ -144,10 +144,30 @@
           "name": "Check support packages",
           "run": "cargo check --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
         },
+        {
+          "name": "Clippy serde_arrow",
+          "run": "cargo clippy --all-targets --package serde_arrow --features arrow-59"
+        },
+        {
+          "name": "Clippy marrow",
+          "run": "cargo clippy --all-targets --package marrow --features serde,arrow-59"
+        },
+        {
+          "name": "Clippy support packages",
+          "run": "cargo clippy --all-targets --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
+        },
+        {
+          "name": "Check Python format",
+          "run": "uv run ruff format --check x.py"
+        },
         {
           "name": "Check format",
           "run": "cargo fmt --check"
         },
+        {
+          "name": "Check generated Rust format",
+          "run": "rustfmt --check marrow/src/impl_arrow/impl_api_53.rs marrow/src/impl_arrow/impl_api_base.rs marrow_integration/src/tests/arrays.rs marrow_integration/src/tests/data_types.rs marrow_integration/src/tests/fixed_size_binary_arrays.rs marrow_integration/src/tests/intervals.rs marrow_integration/src/tests/struct_arrays.rs marrow_integration/src/tests/union_arrays.rs marrow_integration/src/tests/utils.rs marrow_integration/src/tests/views.rs"
+        },
         {
           "name": "Build",
           "run": "cargo build --package serde_arrow --package marrow --features serde_arrow/arrow-59,marrow/serde,marrow/arrow-59"
@@ -159,6 +179,14 @@
         {
           "name": "Integration test",
           "run": "cargo test -p serde_arrow_integration"
+        },
+        {
+          "name": "Run example",
+          "run": "cargo run -p serde_arrow_example"
+        },
+        {
+          "name": "Read example output",
+          "run": "uv run python -c 'import polars as pl; print(pl.read_ipc(\"serde_arrow_example.ipc\"))'"
         }
       ]
     }

arrow-versions.toml

@@ -0,0 +1,10 @@
+arrow-features = [
+    # arrow-version:insert:     "arrow-{version}",
+    "arrow-59",
+    "arrow-58",
+    "arrow-57",
+    "arrow-56",
+    "arrow-55",
+    "arrow-54",
+    "arrow-53",
+]

scripts/add-arrow-version.py

@@ -35,6 +35,7 @@ def add_arrow_version(version):
 def arrow_version_paths():
     return [
         SELF_PATH / "x.py",
+        *SELF_PATH.glob("*.toml"),
         *SELF_PATH.glob("*/**/*.rs"),
         *SELF_PATH.glob("*/**/*.toml"),
     ]

scripts/analyze-benchmark.py

@@ -1,5 +1,6 @@
 import argparse
 import json
+import os
 import pathlib
 import statistics
 
@@ -19,32 +20,39 @@
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--criterion-root", type=pathlib.Path, required=True)
-    parser.add_argument("--plot-output", type=pathlib.Path, required=True)
-    parser.add_argument("--readme", type=pathlib.Path, required=True)
-    parser.add_argument("--update", action="store_true", default=False)
+    parser.add_argument(
+        "--plot-output", type=pathlib.Path, default=pathlib.Path("timings.png")
+    )
+    parser.add_argument("--update", type=pathlib.Path)
+    parser.add_argument("--update-github-summary", action="store_true", default=False)
     analyze_benchmark(parser.parse_args())
 
 
 def analyze_benchmark(args):
     root = resolve_path(args.criterion_root)
-    readme = resolve_path(args.readme)
+    update = resolve_path(args.update) if args.update else None
     plot_output = resolve_path(args.plot_output)
 
     mean_times = load_times(root)
-    print(format_benchmark(mean_times, ignore_groups=README_BENCHMARK_IGNORE_GROUPS))
+    benchmark = format_benchmark(
+        mean_times,
+        ignore_groups=README_BENCHMARK_IGNORE_GROUPS,
+    )
 
-    if args.update:
-        update_readme(
-            readme,
-            mean_times,
-            ignore_groups=README_BENCHMARK_IGNORE_GROUPS,
-        )
-        plot_times(
-            mean_times,
-            benchmark_baseline=BENCHMARK_BASELINE,
-            ignore_groups=README_BENCHMARK_IGNORE_GROUPS,
-            output=plot_output,
-        )
+    print(benchmark)
+
+    if update is not None:
+        update_marked_output(update, benchmark)
+
+    if args.update_github_summary:
+        update_github_summary(benchmark)
+
+    plot_times(
+        mean_times,
+        benchmark_baseline=BENCHMARK_BASELINE,
+        ignore_groups=README_BENCHMARK_IGNORE_GROUPS,
+        output=plot_output,
+    )
 
 
 def resolve_path(path):
@@ -131,21 +139,35 @@ def _parts():
     return "\n".join(_parts())
 
 
-def update_readme(readme, mean_times, ignore_groups=()):
-    print("Update readme")
-    with open(readme, "rt", encoding="utf8") as fobj:
+def update_marked_output(output, content):
+    print(f"Update markers in {output}")
+    with open(output, "rt", encoding="utf8") as fobj:
         lines = [line.rstrip() for line in fobj]
 
-    with open(readme, "wt", encoding="utf8", newline="\n") as fobj:
+    with open(output, "wt", encoding="utf8", newline="\n") as fobj:
         for line in replace_marked_section(
             lines,
             start_marker="<!-- start:benchmarks -->",
             end_marker="<!-- end:benchmarks -->",
-            content=format_benchmark(mean_times, ignore_groups=ignore_groups),
+            content=content,
         ):
             print(line, file=fobj)
 
 
+def update_github_summary(content):
+    path = os.environ.get("GITHUB_STEP_SUMMARY")
+    if path is None:
+        return
+
+    append_output(pathlib.Path(path), content)
+
+
+def append_output(path, content):
+    print(f"Append summary to {path}")
+    with open(path, "at", encoding="utf8", newline="\n") as fobj:
+        print(content, file=fobj)
+
+
 def replace_marked_section(lines, *, start_marker, end_marker, content):
     start = None
     end = None

scripts/check-cargo-toml.py

@@ -1,4 +1,3 @@
-import argparse
 import pathlib
 import tomllib
 
@@ -9,12 +8,10 @@
 
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--arrow-features", required=True, nargs="+")
-    check_cargo_toml(parser.parse_args())
+    check_cargo_toml(load_script_config()["arrow-features"])
 
 
-def check_cargo_toml(args):
+def check_cargo_toml(arrow_features):
     print(":: check Cargo.toml")
     serde_arrow_config = load_config("serde_arrow")
     marrow_config = load_config("marrow")
@@ -36,16 +33,16 @@ def check_cargo_toml(args):
         "serde_arrow",
         "docs.rs configuration",
         serde_arrow_config["package"]["metadata"]["docs"]["rs"]["features"],
-        (args.arrow_features[0],),
+        (arrow_features[0],),
     )
     check_feature_list(
         "marrow",
         "docs.rs configuration",
         marrow_config["package"]["metadata"]["docs"]["rs"]["features"],
-        ("serde", args.arrow_features[0]),
+        ("serde", arrow_features[0]),
     )
 
-    for feature in args.arrow_features:
+    for feature in arrow_features:
         check_arrow_feature(
             "serde_arrow",
             serde_arrow_config,
@@ -88,6 +85,11 @@ def load_config(crate):
         return tomllib.load(fobj)
 
 
+def load_script_config():
+    with open(SELF_PATH / "arrow-versions.toml", "rb") as fobj:
+        return tomllib.load(fobj)
+
+
 def check_feature_list(crate, label, actual, expected):
     actual_features = sorted(actual)
     expected_features = sorted(expected)

scripts/doc.py

@@ -0,0 +1,72 @@
+import argparse
+import pathlib
+import subprocess
+import tomllib
+
+
+SELF_PATH = pathlib.Path(__file__).parents[1].resolve()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--private", action="store_true", default=False)
+    parser.add_argument("--open", action="store_true", default=False)
+    doc(parser.parse_args())
+
+
+def doc(args):
+    extra_args = (("--document-private-items",) if args.private else ()) + (
+        ("--open",) if args.open else ()
+    )
+    subprocess.run(
+        split_command(
+            cargo(
+                "doc",
+                *extra_args,
+                packages=("serde_arrow", "marrow"),
+                features=(
+                    f"serde_arrow/{arrow_features()[0]}",
+                    "marrow/serde",
+                    f"marrow/{arrow_features()[0]}",
+                ),
+            )
+        ),
+        check=True,
+        cwd=SELF_PATH,
+    )
+
+
+def arrow_features():
+    return load_script_config()["arrow-features"]
+
+
+def load_script_config():
+    with open(SELF_PATH / "arrow-versions.toml", "rb") as fobj:
+        return tomllib.load(fobj)
+
+
+def cargo(command, *extra_args, packages=(), features=()):
+    return " ".join(
+        part
+        for part in [
+            "cargo",
+            command,
+            *(f"--package {package}" for package in packages),
+            "--features" if features else "",
+            ",".join(features) if features else "",
+            *extra_args,
+        ]
+        if part
+    )
+
+
+def split_command(command):
+    import shlex
+
+    args = shlex.split(command.replace("\n", " "))
+    print("::", *args)
+    return args
+
+
+if __name__ == "__main__":
+    main()