Skip to content

Commit 205b85c

Browse files
Crocodoctopusthedataking
Crocodoctopus
authored and
thedataking
committed
ci: Add daily postprocess cronjob
1 parent bac286b commit 205b85c

3 files changed

Lines changed: 160 additions & 5 deletions

File tree

‎.github/workflows/ci-c2rust-postprocess.yml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
name: ci-c2rust-postprocess
2+
3+
on:
4+
schedule:
5+
- cron: "0 4 * * *"
6+
7+
jobs:
8+
run-testsuite:
9+
strategy:
10+
matrix:
11+
include:
12+
- runner: ubuntu-latest
13+
os: Linux
14+
arch: x86_64
15+
clang-version: 18
16+
- runner: ubuntu-22.04
17+
os: Linux
18+
arch: x86_64
19+
clang-version: 15
20+
fail-fast: false
21+
name: "postprocess-test-and-cache (${{ matrix.runner }}: ${{ matrix.os }} ${{ matrix.arch}}, Clang ${{ matrix.clang-version }})"
22+
# The type of runner that the job will run on
23+
runs-on: ${{ matrix.runner }}
24+
25+
env:
26+
# TODO: Remove the ident filter when we know the cronjob works
27+
C2RUST_POSTPROCESS_ARGS: >-
28+
--ident-filter '^(json_object_set_userdata|json_object_to_file)$'
29+
--cache-dir ${{ runner.temp }}/llm-cache
30+
31+
# Steps represent a sequence of tasks that will be executed as part of the job
32+
steps:
33+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
34+
# Working dir is /home/runner/work/c2rust/c2rust
35+
- name: Checkout c2rust
36+
uses: actions/checkout@v4
37+
with:
38+
repository: immunant/c2rust
39+
# `update = none` is set, which makes this not work, so we need to update them manually.
40+
# See `.gitmodules` for more info.
41+
submodules: false
42+
43+
- name: Checkout submodules
44+
run: |
45+
git submodule update --init --checkout tests/integration/tests/json-c/repo
46+
47+
- uses: astral-sh/setup-uv@v6
48+
49+
- run: uv python install
50+
51+
- uses: Swatinem/rust-cache@v2
52+
with:
53+
cache-workspace-crates: true
54+
55+
- name: Install Rust toolchains
56+
run: |
57+
rustup toolchain install nightly-2022-11-03 \
58+
--profile minimal --component rustfmt,rustc-dev
59+
rustup toolchain install nightly-2023-04-15 \
60+
--profile minimal --component rustfmt
61+
62+
- name: Provision Debian Packages
63+
run: |
64+
sudo apt-get -qq update
65+
sudo apt-get -qq install \
66+
build-essential \
67+
libbrotli-dev \
68+
libbz2-dev \
69+
libclang-${{ matrix.clang-version }}-dev \
70+
libdb-dev \
71+
libgcrypt20 \
72+
libgdbm-dev \
73+
libreadline-dev \
74+
libidn2-dev \
75+
libldap2-dev \
76+
liblzma-dev \
77+
libnghttp2-dev \
78+
libpcre3-dev \
79+
libpsl-dev \
80+
librtmp-dev \
81+
libtool \
82+
libzstd-dev \
83+
rcs \
84+
tcl-dev \
85+
tk-dev \
86+
zlib1g-dev
87+
88+
# Runs a single command using the runners shell
89+
# Working dir is /home/runner/work/c2rust/c2rust
90+
- name: Build c2rust
91+
env:
92+
LLVM_CONFIG_PATH: /usr/bin/llvm-config-${{ matrix.clang-version }}
93+
run: cargo build --release
94+
95+
- name: Build tools
96+
run: |
97+
(cd tools && cargo build --release --manifest-path split_rust/Cargo.toml)
98+
(cd tools && cargo build --release --manifest-path merge_rust/Cargo.toml)
99+
100+
# Use separate restore/save steps so we still save the cache even if the
101+
# testsuite fails.
102+
- name: Restore postprocess cache
103+
uses: actions/cache/restore@v4
104+
with:
105+
path: ${{ runner.temp }}/llm-cache
106+
# Restore the newest cache matching this runner/compiler prefix.
107+
key: postprocess-cache-${{ matrix.runner }}-${{ matrix.arch }}-clang-${{ matrix.clang-version }}-
108+
109+
# Entry mtimes drive LRU pruning (see `DirectoryCache.prune`) and must
110+
# survive the tar round-trip through `actions/cache`; log the oldest
111+
# entries so this can be verified.
112+
- name: Show restored cache entry ages
113+
run: |
114+
find ${{ runner.temp }}/llm-cache -name metadata.toml -printf '%TF %p\n' 2>/dev/null | sort | head -n 20
115+
116+
- name: Run c2rust testsuite
117+
env:
118+
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
119+
run: |
120+
export PATH=$PWD/target/release:$PWD/c2rust-postprocess:$HOME/.local/bin:$PATH
121+
echo "PATH=$PATH"
122+
export C2RUST_DIR=$PWD
123+
# Needs to be run from `tests/integration/` (or further inside)
124+
# to correctly load the `pyproject.toml`.
125+
(cd tests/integration && ./test.py json-c)
126+
127+
# Stale entries are pruned by postprocess itself (--prune-cache-days).
128+
- name: Save postprocess cache
129+
if: always()
130+
uses: actions/cache/save@v4
131+
with:
132+
path: ${{ runner.temp }}/llm-cache
133+
# Save a fresh snapshot every run: caches are immutable, and entry
134+
# mtimes (which drive pruning) only persist via a new snapshot even
135+
# when entry contents are unchanged. Old snapshots stop being
136+
# restored and age out via GitHub's cache eviction.
137+
key: postprocess-cache-${{ matrix.runner }}-${{ matrix.arch }}-clang-${{ matrix.clang-version }}-${{ github.run_id }}

‎.github/workflows/internal-testsuite.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ jobs:
1414
# This workflow contains a single job called "build"
1515
run-testsuite:
1616
strategy:
17+
# ci-c2rust-postprocess.yml generates postprocess caches for this matrix.
18+
# Keep both matrices in sync so this workflow can restore those caches.
1719
matrix:
1820
include:
1921
- runner: ubuntu-latest
@@ -29,6 +31,11 @@ jobs:
2931
# The type of runner that the job will run on
3032
runs-on: ${{ matrix.runner }}
3133

34+
env:
35+
# Uses cache-only responses; skips uncached work and exercises cached
36+
# postprocess output usage.
37+
C2RUST_POSTPROCESS_ARGS: --on-error warn --cache-dir ${{ runner.temp }}/llm-cache
38+
3239
# Steps represent a sequence of tasks that will be executed as part of the job
3340
steps:
3441
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -106,6 +113,13 @@ jobs:
106113
(cd tools && cargo build --release --manifest-path split_rust/Cargo.toml)
107114
(cd tools && cargo build --release --manifest-path merge_rust/Cargo.toml)
108115
116+
- name: Restore postprocess cache
117+
uses: actions/cache/restore@v4
118+
with:
119+
path: ${{ runner.temp }}/llm-cache
120+
# Restore the newest available cache entry that matches this prefix.
121+
key: postprocess-cache-${{ matrix.runner }}-${{ matrix.arch }}-clang-${{ matrix.clang-version }}-
122+
109123
- name: Run c2rust testsuite
110124
run: |
111125
export PATH=$PWD/target/release:$PWD/c2rust-postprocess:$HOME/.local/bin:$PATH

‎tests/integration/tests/templates.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,17 +199,21 @@ def autogen_postprocess(
199199
if not (ag and isinstance(ag, bool)):
200200
return
201201

202-
params = {"args": ""}
202+
args: list[str] = []
203203

204204
exclude_file = conf.with_name("postprocess-exclude.yml")
205205
if exclude_file.exists():
206206
# args are relative to script dir
207-
params["args"] = shlex.join(
208-
["--exclude-file", str(exclude_file.relative_to(conf.parent))]
209-
)
207+
args.extend(["--exclude-file", str(exclude_file.relative_to(conf.parent))])
208+
209+
extra_args = os.getenv("C2RUST_POSTPROCESS_ARGS")
210+
if extra_args:
211+
args.extend(shlex.split(extra_args))
210212

211213
if verbose:
212-
params["args"] = f"--log-level DEBUG {params['args']}".rstrip()
214+
args[0:0] = ["--log-level", "DEBUG"]
215+
216+
params = {"args": shlex.join(args)}
213217

214218
out_path = conf.with_name("postprocess.gen.sh")
215219
render_script(POSTPROCESS_SH, out_path, params)

0 commit comments

Comments
 (0)