Skip to content

Commit 2b2a2e5

Browse files
chmpchmp
committed
Simplify x.py by moving functions into scripts
1 parent a2233ab commit 2b2a2e5

10 files changed

Lines changed: 1166 additions & 731 deletions

File tree

‎.github/workflows/bench.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
},
2525
{
2626
"name": "Bench",
27-
"run": "cargo bench -p serde_arrow_bench\npython x.py summarize-bench >> $GITHUB_STEP_SUMMARY"
27+
"run": "cargo bench -p serde_arrow_bench\npython scripts/analyze-benchmark.py --criterion-root target/criterion --readme Readme.md --plot-output timings.png >> $GITHUB_STEP_SUMMARY"
2828
}
2929
]
3030
}

‎.github/workflows/local.yml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"name": "Local",
3+
"on": {
4+
"workflow_dispatch": {}
5+
},
6+
"env": {
7+
"CARGO_TERM_COLOR": "always"
8+
},
9+
"jobs": {
10+
"format": {
11+
"runs-on": "ubuntu-latest",
12+
"steps": [
13+
{
14+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
15+
},
16+
{
17+
"name": "Install uv",
18+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
19+
"with": {
20+
"enable-cache": true
21+
}
22+
},
23+
{
24+
"name": "rustc",
25+
"run": "rustc --version"
26+
},
27+
{
28+
"name": "cargo",
29+
"run": "cargo --version"
30+
},
31+
{
32+
"name": "Format Python",
33+
"run": "uv run ruff format x.py"
34+
},
35+
{
36+
"name": "Format Cargo",
37+
"run": "cargo fmt"
38+
},
39+
{
40+
"name": "Format generated Rust",
41+
"run": "rustfmt marrow/src/impl_arrow/impl_api_53.rs marrow/src/impl_arrow/impl_api_base.rs marrow_integration/src/tests/arrays.rs marrow_integration/src/tests/data_types.rs marrow_integration/src/tests/fixed_size_binary_arrays.rs marrow_integration/src/tests/intervals.rs marrow_integration/src/tests/struct_arrays.rs marrow_integration/src/tests/union_arrays.rs marrow_integration/src/tests/utils.rs marrow_integration/src/tests/views.rs"
42+
}
43+
]
44+
},
45+
"check": {
46+
"runs-on": "ubuntu-latest",
47+
"steps": [
48+
{
49+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
50+
},
51+
{
52+
"name": "Install uv",
53+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
54+
"with": {
55+
"enable-cache": true
56+
}
57+
},
58+
{
59+
"name": "rustc",
60+
"run": "rustc --version"
61+
},
62+
{
63+
"name": "cargo",
64+
"run": "cargo --version"
65+
},
66+
{
67+
"name": "Check Cargo.toml",
68+
"run": "uv run python scripts/check-cargo-toml.py --arrow-features arrow-59 arrow-58 arrow-57 arrow-56 arrow-55 arrow-54 arrow-53"
69+
},
70+
{
71+
"name": "Check workspace",
72+
"run": "cargo check"
73+
},
74+
{
75+
"name": "Check serde_arrow",
76+
"run": "cargo check --all-targets --package serde_arrow --features arrow-59"
77+
},
78+
{
79+
"name": "Check marrow",
80+
"run": "cargo check --all-targets --package marrow --features serde,arrow-59"
81+
},
82+
{
83+
"name": "Clippy serde_arrow",
84+
"run": "cargo clippy --all-targets --package serde_arrow --features arrow-59"
85+
},
86+
{
87+
"name": "Clippy marrow",
88+
"run": "cargo clippy --all-targets --package marrow --features serde,arrow-59"
89+
},
90+
{
91+
"name": "Check support packages",
92+
"run": "cargo check --all-targets --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
93+
},
94+
{
95+
"name": "Clippy support packages",
96+
"run": "cargo clippy --all-targets --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
97+
}
98+
]
99+
},
100+
"test": {
101+
"runs-on": "ubuntu-latest",
102+
"steps": [
103+
{
104+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
105+
},
106+
{
107+
"name": "Install uv",
108+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
109+
"with": {
110+
"enable-cache": true
111+
}
112+
},
113+
{
114+
"name": "rustc",
115+
"run": "rustc --version"
116+
},
117+
{
118+
"name": "cargo",
119+
"run": "cargo --version"
120+
},
121+
{
122+
"name": "Test serde_arrow",
123+
"run": "cargo test -q --package serde_arrow --features arrow-59"
124+
},
125+
{
126+
"name": "Test marrow",
127+
"run": "cargo test -q --package marrow --features serde,arrow-59"
128+
},
129+
{
130+
"name": "Integration test",
131+
"run": "cargo test -p serde_arrow_integration"
132+
}
133+
]
134+
},
135+
"example": {
136+
"runs-on": "ubuntu-latest",
137+
"steps": [
138+
{
139+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
140+
},
141+
{
142+
"name": "Install uv",
143+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
144+
"with": {
145+
"enable-cache": true
146+
}
147+
},
148+
{
149+
"name": "rustc",
150+
"run": "rustc --version"
151+
},
152+
{
153+
"name": "cargo",
154+
"run": "cargo --version"
155+
},
156+
{
157+
"name": "Run example",
158+
"run": "cargo run -p serde_arrow_example"
159+
},
160+
{
161+
"name": "Read example output",
162+
"run": "uv run python -c 'import polars as pl; print(pl.read_ipc(\"serde_arrow_example.ipc\"))'"
163+
}
164+
]
165+
}
166+
}
167+
}

‎.github/workflows/test.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
}
5252
]
5353
},
54-
"build": {
54+
"test": {
5555
"runs-on": "ubuntu-latest",
5656
"steps": [
5757
{
@@ -72,6 +72,10 @@
7272
"name": "cargo",
7373
"run": "cargo --version"
7474
},
75+
{
76+
"name": "Check Cargo.toml",
77+
"run": "uv run python scripts/check-cargo-toml.py --arrow-features arrow-59 arrow-58 arrow-57 arrow-56 arrow-55 arrow-54 arrow-53"
78+
},
7579
{
7680
"name": "Check",
7781
"run": "cargo check"
@@ -154,7 +158,7 @@
154158
},
155159
{
156160
"name": "Integration test",
157-
"run": "uv run python x.py test-integration"
161+
"run": "cargo test -p serde_arrow_integration"
158162
}
159163
]
160164
}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ target/
33
/serde_arrow_example.ipc
44

55
__pycache__/
6+
*.py[cod]
67
.ipynb_checkpoints/
78

89
# Editor directories and files

‎scripts/add-arrow-version.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import argparse
2+
import pathlib
3+
import re
4+
import subprocess
5+
6+
7+
SELF_PATH = pathlib.Path(__file__).parents[1].resolve()
8+
9+
10+
def main():
11+
parser = argparse.ArgumentParser()
12+
parser.add_argument("version")
13+
add_arrow_version(parser.parse_args().version)
14+
15+
16+
def add_arrow_version(version):
17+
if (
18+
subprocess.run(
19+
["git", "diff-files", "--quiet"],
20+
check=False,
21+
cwd=SELF_PATH,
22+
).returncode
23+
!= 0
24+
):
25+
print(
26+
"WARNING: potentially destructive changes. "
27+
"Please stage or commit the working tree first."
28+
)
29+
raise SystemExit(1)
30+
31+
for path in arrow_version_paths():
32+
process_arrow_version_path(path, version)
33+
34+
35+
def arrow_version_paths():
36+
return [
37+
SELF_PATH / "x.py",
38+
*SELF_PATH.glob("*/**/*.rs"),
39+
*SELF_PATH.glob("*/**/*.toml"),
40+
]
41+
42+
43+
def process_arrow_version_path(path, version):
44+
content = path.read_text()
45+
if "arrow-version" not in content:
46+
return
47+
48+
print(f"process {path}")
49+
new_content = []
50+
include_next = True
51+
for line in content.splitlines():
52+
if (
53+
match := re.match(r"^.*(//|#) arrow-version:(replace|insert): (.*)$", line)
54+
) is not None:
55+
new_content.append(line)
56+
new_content.append(
57+
match.group(3).format_map({"version": version, "\\n": "\n"})
58+
)
59+
include_next = match.group(2) != "replace"
60+
61+
else:
62+
if include_next:
63+
new_content.append(line)
64+
65+
include_next = True
66+
67+
with open(path, "wt", newline="\n", encoding="utf-8") as fobj:
68+
fobj.write("\n".join(new_content))
69+
70+
71+
if __name__ == "__main__":
72+
main()

0 commit comments

Comments
 (0)