Skip to content

Commit e52577e

Browse files
Crocodoctopusthedataking
Crocodoctopus
authored and
thedataking
committed
ci: Add daily postprocess cronjob
1 parent bac286b commit e52577e

3 files changed

Lines changed: 168 additions & 7 deletions

File tree

‎.github/workflows/ci-c2rust-postprocess.yml‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
name: ci-c2rust-postprocess
2+
3+
on:
4+
workflow_dispatch: # Enables manual triggering
5+
schedule:
6+
- cron: "0 4 * * *"
7+
8+
jobs:
9+
run-testsuite:
10+
strategy:
11+
matrix:
12+
include:
13+
- runner: ubuntu-latest
14+
os: Linux
15+
arch: x86_64
16+
clang-version: 18
17+
- runner: ubuntu-22.04
18+
os: Linux
19+
arch: x86_64
20+
clang-version: 15
21+
fail-fast: false
22+
name: "postprocess-test-and-cache (${{ matrix.runner }}: ${{ matrix.os }} ${{ matrix.arch}}, Clang ${{ matrix.clang-version }})"
23+
# The type of runner that the job will run on
24+
runs-on: ${{ matrix.runner }}
25+
26+
env:
27+
# TODO: Remove the ident filter when we know the cronjob works.
28+
# The value is expanded unquoted by postprocess.gen.sh, so no shell
29+
# quoting: args must not contain whitespace.
30+
C2RUST_POSTPROCESS_EXTRA_ARGS: >-
31+
--ident-filter ^(json_object_set_userdata|json_object_to_file)$
32+
--cache-dir ${{ runner.temp }}/llm-cache
33+
34+
# Steps represent a sequence of tasks that will be executed as part of the job
35+
steps:
36+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
37+
# Working dir is /home/runner/work/c2rust/c2rust
38+
- name: Checkout c2rust
39+
uses: actions/checkout@v4
40+
with:
41+
repository: immunant/c2rust
42+
# `update = none` is set, which makes this not work, so we need to update them manually.
43+
# See `.gitmodules` for more info.
44+
submodules: false
45+
46+
- name: Checkout submodules
47+
run: |
48+
git submodule update --init --checkout tests/integration/tests/json-c/repo
49+
50+
- uses: astral-sh/setup-uv@v6
51+
52+
- run: uv python install
53+
54+
- uses: Swatinem/rust-cache@v2
55+
with:
56+
cache-workspace-crates: true
57+
58+
- name: Install Rust toolchains
59+
run: |
60+
rustup toolchain install nightly-2022-11-03 \
61+
--profile minimal --component rustfmt,rustc-dev
62+
rustup toolchain install nightly-2023-04-15 \
63+
--profile minimal --component rustfmt
64+
65+
- name: Provision Debian Packages
66+
run: |
67+
sudo apt-get -qq update
68+
sudo apt-get -qq install \
69+
build-essential \
70+
libbrotli-dev \
71+
libbz2-dev \
72+
libclang-${{ matrix.clang-version }}-dev \
73+
libdb-dev \
74+
libgcrypt20 \
75+
libgdbm-dev \
76+
libreadline-dev \
77+
libidn2-dev \
78+
libldap2-dev \
79+
liblzma-dev \
80+
libnghttp2-dev \
81+
libpcre3-dev \
82+
libpsl-dev \
83+
librtmp-dev \
84+
libtool \
85+
libzstd-dev \
86+
rcs \
87+
tcl-dev \
88+
tk-dev \
89+
zlib1g-dev
90+
91+
# Runs a single command using the runners shell
92+
# Working dir is /home/runner/work/c2rust/c2rust
93+
- name: Build c2rust
94+
env:
95+
LLVM_CONFIG_PATH: /usr/bin/llvm-config-${{ matrix.clang-version }}
96+
run: cargo build --release
97+
98+
- name: Build tools
99+
run: |
100+
(cd tools && cargo build --release --manifest-path split_rust/Cargo.toml)
101+
(cd tools && cargo build --release --manifest-path merge_rust/Cargo.toml)
102+
103+
# Use separate restore/save steps so we still save the cache even if the
104+
# testsuite fails.
105+
- name: Restore postprocess cache
106+
uses: actions/cache/restore@v4
107+
with:
108+
path: ${{ runner.temp }}/llm-cache
109+
# Restore the newest cache matching this runner/compiler prefix.
110+
key: postprocess-cache-${{ matrix.runner }}-${{ matrix.arch }}-clang-${{ matrix.clang-version }}-
111+
112+
# Entry mtimes drive LRU pruning (see `DirectoryCache.prune`) and must
113+
# survive the tar round-trip through `actions/cache`; log the oldest
114+
# entries so this can be verified.
115+
- name: Show restored cache entry ages
116+
run: |
117+
find ${{ runner.temp }}/llm-cache -name metadata.toml -printf '%TF %p\n' 2>/dev/null | sort | head -n 20
118+
119+
- name: Run c2rust testsuite
120+
env:
121+
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
122+
run: |
123+
export PATH=$PWD/target/release:$PWD/c2rust-postprocess:$HOME/.local/bin:$PATH
124+
echo "PATH=$PATH"
125+
export C2RUST_DIR=$PWD
126+
# Needs to be run from `tests/integration/` (or further inside)
127+
# to correctly load the `pyproject.toml`.
128+
(cd tests/integration && ./test.py json-c)
129+
130+
# Stale entries are pruned by postprocess itself (--prune-cache-days).
131+
- name: Save postprocess cache
132+
if: always()
133+
uses: actions/cache/save@v4
134+
with:
135+
path: ${{ runner.temp }}/llm-cache
136+
# Save a fresh snapshot every run: caches are immutable, and entry
137+
# mtimes (which drive pruning) only persist via a new snapshot even
138+
# when entry contents are unchanged. Old snapshots stop being
139+
# restored and age out via GitHub's cache eviction.
140+
key: postprocess-cache-${{ matrix.runner }}-${{ matrix.arch }}-clang-${{ matrix.clang-version }}-${{ github.run_id }}
141+
142+
- uses: actions/upload-artifact@v4
143+
with:
144+
name: testsuite-${{ matrix.runner }}-artifacts
145+
path: |
146+
${{ github.workspace }}/tests/integration/**/*.log
147+
if: always()

‎.github/workflows/internal-testsuite.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ jobs:
1414
# This workflow contains a single job called "build"
1515
run-testsuite:
1616
strategy:
17+
# ci-c2rust-postprocess.yml generates postprocess caches for this matrix.
18+
# Keep both matrices in sync so this workflow can restore those caches.
1719
matrix:
1820
include:
1921
- runner: ubuntu-latest
@@ -29,6 +31,11 @@ jobs:
2931
# The type of runner that the job will run on
3032
runs-on: ${{ matrix.runner }}
3133

34+
env:
35+
# Uses cache-only responses; skips uncached work and exercises cached
36+
# postprocess output usage.
37+
C2RUST_POSTPROCESS_EXTRA_ARGS: --on-error warn --cache-dir ${{ runner.temp }}/llm-cache
38+
3239
# Steps represent a sequence of tasks that will be executed as part of the job
3340
steps:
3441
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -106,6 +113,13 @@ jobs:
106113
(cd tools && cargo build --release --manifest-path split_rust/Cargo.toml)
107114
(cd tools && cargo build --release --manifest-path merge_rust/Cargo.toml)
108115
116+
- name: Restore postprocess cache
117+
uses: actions/cache/restore@v4
118+
with:
119+
path: ${{ runner.temp }}/llm-cache
120+
# Restore the newest available cache entry that matches this prefix.
121+
key: postprocess-cache-${{ matrix.runner }}-${{ matrix.arch }}-clang-${{ matrix.clang-version }}-
122+
109123
- name: Run c2rust testsuite
110124
run: |
111125
export PATH=$PWD/target/release:$PWD/c2rust-postprocess:$HOME/.local/bin:$PATH

‎tests/integration/tests/templates.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import os
22
from pathlib import Path
3-
import shlex
43
import stat
54
from collections.abc import Mapping
65
from typing import Any, Generator
@@ -88,7 +87,8 @@
8887
8988
cd "${SCRIPT_DIR}"
9089
91-
c2rust-postprocess --update-rust {{args}} repo/lib.rs 2>&1 | tee "$LOG_FILE"
90+
# C2RUST_POSTPROCESS_EXTRA_ARGS is whitespace-split; quoting is not re-parsed.
91+
c2rust-postprocess --update-rust {{args}} ${C2RUST_POSTPROCESS_EXTRA_ARGS} repo/lib.rs 2>&1 | tee "$LOG_FILE"
9292
"""
9393

9494
CARGO_SH: str = r"""#!/usr/bin/env bash
@@ -199,17 +199,17 @@ def autogen_postprocess(
199199
if not (ag and isinstance(ag, bool)):
200200
return
201201

202-
params = {"args": ""}
202+
args: list[str] = []
203203

204204
exclude_file = conf.with_name("postprocess-exclude.yml")
205205
if exclude_file.exists():
206206
# args are relative to script dir
207-
params["args"] = shlex.join(
208-
["--exclude-file", str(exclude_file.relative_to(conf.parent))]
209-
)
207+
args.extend(["--exclude-file", str(exclude_file.relative_to(conf.parent))])
210208

211209
if verbose:
212-
params["args"] = f"--log-level DEBUG {params['args']}".rstrip()
210+
args[0:0] = ["--log-level", "DEBUG"]
211+
212+
params = {"args": " ".join(args)}
213213

214214
out_path = conf.with_name("postprocess.gen.sh")
215215
render_script(POSTPROCESS_SH, out_path, params)

0 commit comments

Comments
 (0)