Skip to content

Commit dd16b68

Browse files
chmpchmp
authored
Feature/update dev docs (#322)
* Update contributing guide * Update tags * Add underdocumented aspects * Fix readme links * Simplify x.py workflow check generation * Simplify x.py local check commands * Represent x.py feature groups as tuples * Simplify x.py manifest validation * Name README benchmark exclusions * Check README benchmark markers * Move monorepo hint to contributing.md * Inline x.py cargo workflow steps * Fix clippy * Simplify x.py * Further simplify x.py * Simplify x.py * Remove unused variable * Add a simple workflow runner * Simplify x.py by moving functions into scripts * Move most logic into separate scripts / workflows * Bump Python requirements * Add a PR template * Update cargo deps * update contriuting docs
1 parent 806fd33 commit dd16b68

19 files changed

Lines changed: 1825 additions & 1215 deletions

File tree

‎.github/pull_request_template.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
## Summary
2+
3+
<!-- Describe the change and why it is needed. -->
4+
5+
## Checklist
6+
7+
- [ ] Run the pre-commit checks locally with `uv run python x.py precommit`
8+
- [ ] Add tests for any new functionality
9+
- [ ] Update the relevant changelog (`Changes.md` for `serde_arrow`,
10+
`marrow/Changes.md` for `marrow`)

‎.github/workflows/bench.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
"CARGO_TERM_COLOR": "always"
99
},
1010
"jobs": {
11-
"build": {
11+
"bench": {
1212
"runs-on": "ubuntu-latest",
1313
"steps": [
1414
{
@@ -24,7 +24,11 @@
2424
},
2525
{
2626
"name": "Bench",
27-
"run": "cargo bench -p serde_arrow_bench\npython x.py summarize-bench >> $GITHUB_STEP_SUMMARY"
27+
"run": "cargo bench -p serde_arrow_bench"
28+
},
29+
{
30+
"name": "Summarize",
31+
"run": "python scripts/analyze-benchmark.py --criterion-root target/criterion --update-github-summary"
2832
}
2933
]
3034
}

‎.github/workflows/local.yml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"name": "Local",
3+
"on": {
4+
"workflow_dispatch": {}
5+
},
6+
"env": {
7+
"CARGO_TERM_COLOR": "always"
8+
},
9+
"jobs": {
10+
"format": {
11+
"runs-on": "ubuntu-latest",
12+
"steps": [
13+
{
14+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
15+
},
16+
{
17+
"name": "Install uv",
18+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
19+
"with": {
20+
"enable-cache": true
21+
}
22+
},
23+
{
24+
"name": "rustc",
25+
"run": "rustc --version"
26+
},
27+
{
28+
"name": "cargo",
29+
"run": "cargo --version"
30+
},
31+
{
32+
"name": "Format Python",
33+
"run": "uv run ruff format x.py"
34+
},
35+
{
36+
"name": "Format Cargo",
37+
"run": "cargo fmt"
38+
},
39+
{
40+
"name": "Format generated Rust",
41+
"run": "rustfmt marrow/src/impl_arrow/impl_api_53.rs marrow/src/impl_arrow/impl_api_base.rs marrow_integration/src/tests/arrays.rs marrow_integration/src/tests/data_types.rs marrow_integration/src/tests/fixed_size_binary_arrays.rs marrow_integration/src/tests/intervals.rs marrow_integration/src/tests/struct_arrays.rs marrow_integration/src/tests/union_arrays.rs marrow_integration/src/tests/utils.rs marrow_integration/src/tests/views.rs"
42+
}
43+
]
44+
},
45+
"check": {
46+
"runs-on": "ubuntu-latest",
47+
"steps": [
48+
{
49+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
50+
},
51+
{
52+
"name": "Install uv",
53+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
54+
"with": {
55+
"enable-cache": true
56+
}
57+
},
58+
{
59+
"name": "rustc",
60+
"run": "rustc --version"
61+
},
62+
{
63+
"name": "cargo",
64+
"run": "cargo --version"
65+
},
66+
{
67+
"name": "Check Cargo.toml",
68+
"run": "uv run python scripts/check-cargo-toml.py"
69+
},
70+
{
71+
"name": "Check workspace",
72+
"run": "cargo check"
73+
},
74+
{
75+
"name": "Check serde_arrow",
76+
"run": "cargo check --all-targets --package serde_arrow --features arrow-59"
77+
},
78+
{
79+
"name": "Check marrow",
80+
"run": "cargo check --all-targets --package marrow --features serde,arrow-59"
81+
},
82+
{
83+
"name": "Clippy serde_arrow",
84+
"run": "cargo clippy --all-targets --package serde_arrow --features arrow-59"
85+
},
86+
{
87+
"name": "Clippy marrow",
88+
"run": "cargo clippy --all-targets --package marrow --features serde,arrow-59"
89+
},
90+
{
91+
"name": "Check support packages",
92+
"run": "cargo check --all-targets --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
93+
},
94+
{
95+
"name": "Clippy support packages",
96+
"run": "cargo clippy --all-targets --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
97+
}
98+
]
99+
},
100+
"test": {
101+
"runs-on": "ubuntu-latest",
102+
"steps": [
103+
{
104+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
105+
},
106+
{
107+
"name": "Install uv",
108+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
109+
"with": {
110+
"enable-cache": true
111+
}
112+
},
113+
{
114+
"name": "rustc",
115+
"run": "rustc --version"
116+
},
117+
{
118+
"name": "cargo",
119+
"run": "cargo --version"
120+
},
121+
{
122+
"name": "Test serde_arrow",
123+
"run": "cargo test -q --package serde_arrow --features arrow-59"
124+
},
125+
{
126+
"name": "Test marrow",
127+
"run": "cargo test -q --package marrow --features serde,arrow-59"
128+
},
129+
{
130+
"name": "Integration test",
131+
"run": "cargo test -p serde_arrow_integration"
132+
}
133+
]
134+
},
135+
"example": {
136+
"runs-on": "ubuntu-latest",
137+
"steps": [
138+
{
139+
"uses": "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0"
140+
},
141+
{
142+
"name": "Install uv",
143+
"uses": "astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39",
144+
"with": {
145+
"enable-cache": true
146+
}
147+
},
148+
{
149+
"name": "rustc",
150+
"run": "rustc --version"
151+
},
152+
{
153+
"name": "cargo",
154+
"run": "cargo --version"
155+
},
156+
{
157+
"name": "Run example",
158+
"run": "cargo run -p serde_arrow_example"
159+
},
160+
{
161+
"name": "Read example output",
162+
"run": "uv run python -c 'import polars as pl; print(pl.read_ipc(\"serde_arrow_example.ipc\"))'"
163+
}
164+
]
165+
}
166+
}
167+
}

‎.github/workflows/test.yml‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
}
5252
]
5353
},
54-
"build": {
54+
"test": {
5555
"runs-on": "ubuntu-latest",
5656
"steps": [
5757
{
@@ -72,6 +72,10 @@
7272
"name": "cargo",
7373
"run": "cargo --version"
7474
},
75+
{
76+
"name": "Check Cargo.toml",
77+
"run": "uv run python scripts/check-cargo-toml.py"
78+
},
7579
{
7680
"name": "Check",
7781
"run": "cargo check"
@@ -140,10 +144,30 @@
140144
"name": "Check support packages",
141145
"run": "cargo check --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
142146
},
147+
{
148+
"name": "Clippy serde_arrow",
149+
"run": "cargo clippy --all-targets --package serde_arrow --features arrow-59"
150+
},
151+
{
152+
"name": "Clippy marrow",
153+
"run": "cargo clippy --all-targets --package marrow --features serde,arrow-59"
154+
},
155+
{
156+
"name": "Clippy support packages",
157+
"run": "cargo clippy --all-targets --all-features --package serde_arrow_bench --package serde_arrow_example --package serde_arrow_integration --package marrow_integration"
158+
},
159+
{
160+
"name": "Check Python format",
161+
"run": "uv run ruff format --check x.py"
162+
},
143163
{
144164
"name": "Check format",
145165
"run": "cargo fmt --check"
146166
},
167+
{
168+
"name": "Check generated Rust format",
169+
"run": "rustfmt --check marrow/src/impl_arrow/impl_api_53.rs marrow/src/impl_arrow/impl_api_base.rs marrow_integration/src/tests/arrays.rs marrow_integration/src/tests/data_types.rs marrow_integration/src/tests/fixed_size_binary_arrays.rs marrow_integration/src/tests/intervals.rs marrow_integration/src/tests/struct_arrays.rs marrow_integration/src/tests/union_arrays.rs marrow_integration/src/tests/utils.rs marrow_integration/src/tests/views.rs"
170+
},
147171
{
148172
"name": "Build",
149173
"run": "cargo build --package serde_arrow --package marrow --features serde_arrow/arrow-59,marrow/serde,marrow/arrow-59"
@@ -154,7 +178,15 @@
154178
},
155179
{
156180
"name": "Integration test",
157-
"run": "uv run python x.py test-integration"
181+
"run": "cargo test -p serde_arrow_integration"
182+
},
183+
{
184+
"name": "Run example",
185+
"run": "cargo run -p serde_arrow_example"
186+
},
187+
{
188+
"name": "Read example output",
189+
"run": "uv run python -c 'import polars as pl; print(pl.read_ipc(\"serde_arrow_example.ipc\"))'"
158190
}
159191
]
160192
}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ target/
33
/serde_arrow_example.ipc
44

55
__pycache__/
6+
*.py[cod]
67
.ipynb_checkpoints/
78

89
# Editor directories and files

0 commit comments

Comments
 (0)