Skip to content

Commit aa65a39

Browse files
jservjserv
authored
Merge pull request #369 from sysprog21/ci-refine
Refine CI/CD for faster, consolidated pipelines
2 parents fb18fd7 + 731c25d commit aa65a39

6 files changed

Lines changed: 457 additions & 150 deletions

File tree

‎.ci/build-n-run.sh‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,56 @@
11
#!/usr/bin/env bash
22

3-
function build_example()
4-
{
5-
make -C examples || exit 1
3+
set -euo pipefail
4+
5+
LOG_DIR=${STATUS_CHECK_LOG_DIR:-$(pwd)}
6+
mkdir -p "${LOG_DIR}"
7+
8+
build_example()
9+
{
10+
make -C examples 2>&1 | tee "${LOG_DIR}/build-n-run-build.log"
611
}
712

8-
function list_mod()
13+
list_mod()
914
{
10-
# Filter out the modules specified in non-working
11-
ls examples/*.ko | awk -F "[/|.]" '{print $2}' | grep -vFxf .ci/non-working
15+
local exclude_file=".ci/non-working"
16+
17+
if [ ! -f "${exclude_file}" ]; then
18+
exclude_file="/dev/null"
19+
fi
20+
21+
find examples -maxdepth 1 -name '*.ko' -print | sort \
22+
| sed 's#^examples/##; s#\.ko$##' \
23+
| { grep -vFxf "${exclude_file}" || test $? -eq 1; }
1224
}
1325

14-
function run_mod()
26+
run_mod()
1527
{
16-
# insert/remove twice to ensure resource allocations
17-
( sudo insmod "examples/$1.ko" && sudo rmmod "$1" ) || exit 1
18-
( sudo insmod "examples/$1.ko" && sudo rmmod "$1" ) || exit 1
28+
local module=$1
29+
local module_log="${LOG_DIR}/${module}.log"
30+
31+
{
32+
echo "=== insmod/rmmod pass 1: ${module} ==="
33+
sudo insmod "examples/${module}.ko"
34+
sudo rmmod "${module}"
35+
echo "=== insmod/rmmod pass 2: ${module} ==="
36+
sudo insmod "examples/${module}.ko"
37+
sudo rmmod "${module}"
38+
} 2>&1 | tee "${module_log}"
1939
}
2040

21-
function run_examples()
41+
run_examples()
2242
{
23-
for module in $(list_mod); do
24-
echo "Running $module"
25-
run_mod "$module"
43+
local modules=()
44+
45+
mapfile -t modules < <(list_mod)
46+
if [ ${#modules[@]} -eq 0 ]; then
47+
echo "No runnable modules found."
48+
return
49+
fi
50+
51+
for module in "${modules[@]}"; do
52+
echo "Running ${module}"
53+
run_mod "${module}"
2654
done
2755
}
2856

‎.ci/check-format.sh‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,35 @@
11
#!/usr/bin/env bash
22

3-
SOURCES=$(find $(git rev-parse --show-toplevel) | grep -E "\.(cpp|cc|c|h)\$")
3+
set -euo pipefail
44

5-
CLANG_FORMAT=$(which clang-format)
6-
if [ $? -ne 0 ]; then
7-
CLANG_FORMAT=$(which clang-format)
8-
if [ $? -ne 0 ]; then
9-
echo "[!] clang-format not installed. Unable to check source file format policy." >&2
10-
exit 1
11-
fi
5+
mapfile -t SOURCES < <(git ls-files '*.c' '*.cc' '*.cpp' '*.h')
6+
if [ ${#SOURCES[@]} -eq 0 ]; then
7+
exit 0
128
fi
139

14-
set -x
10+
CLANG_FORMAT=$(command -v clang-format)
11+
if [ -z "${CLANG_FORMAT}" ]; then
12+
echo "[!] clang-format not installed. Unable to check source file format policy." >&2
13+
exit 1
14+
fi
15+
16+
LOG_DIR=${STATUS_CHECK_LOG_DIR:-$(pwd)}
17+
mkdir -p "${LOG_DIR}"
18+
DIFF_LOG="${LOG_DIR}/check-format.diff"
19+
TMP_FILE=$(mktemp)
20+
trap 'rm -f "${TMP_FILE}"' EXIT
21+
22+
: > "${DIFF_LOG}"
1523

16-
for file in ${SOURCES};
17-
do
18-
$CLANG_FORMAT ${file} > expected-format
19-
diff -u -p --label="${file}" --label="expected coding style" ${file} expected-format
24+
for file in "${SOURCES[@]}"; do
25+
"${CLANG_FORMAT}" "${file}" > "${TMP_FILE}"
26+
if ! diff -u -p --label="${file}" --label="expected coding style" \
27+
"${file}" "${TMP_FILE}" >> "${DIFF_LOG}"; then
28+
cat "${DIFF_LOG}"
29+
exit 1
30+
fi
2031
done
21-
exit $($CLANG_FORMAT --output-replacements-xml ${SOURCES} | grep -E -c "</replacement>")
32+
33+
if "${CLANG_FORMAT}" --output-replacements-xml "${SOURCES[@]}" | grep -Eq "</replacement>"; then
34+
exit 1
35+
fi

‎.ci/static-analysis.sh‎

Lines changed: 157 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,199 @@
11
#!/usr/bin/env bash
22

3-
function do_cppcheck()
3+
set -euo pipefail
4+
5+
ROOT_DIR=$(git rev-parse --show-toplevel)
6+
LOG_DIR=${STATUS_CHECK_LOG_DIR:-"${ROOT_DIR}/.status-check-logs"}
7+
TOOL_CACHE_DIR=${STATUS_CHECK_TOOL_CACHE_DIR:-"${ROOT_DIR}/.status-check-tools"}
8+
mkdir -p "${LOG_DIR}" "${TOOL_CACHE_DIR}"
9+
10+
mapfile -t SOURCES < <(git -C "${ROOT_DIR}" ls-files '*.c' '*.cc' '*.cpp' '*.h')
11+
12+
count_matches()
413
{
5-
local SOURCES=$(find $(git rev-parse --show-toplevel) | grep -E "\.(cpp|cc|c|h)\$")
14+
local pattern=$1
15+
local file=$2
616

7-
local CPPCHECK=$(which cppcheck)
8-
if [ $? -ne 0 ]; then
9-
echo "[!] cppcheck not installed. Failed to run static analysis the source code." >&2
10-
exit 1
17+
grep -E -c "${pattern}" "${file}" 2>/dev/null || true
18+
}
19+
20+
needs_rebuild()
21+
{
22+
local repo_url=$1
23+
local repo_dir=$2
24+
local binary=$3
25+
local stamp_file=$4
26+
27+
if [ ! -x "${binary}" ] || ! "${binary}" --version >/dev/null 2>&1; then
28+
return 0
29+
fi
30+
31+
if [ ! -d "${repo_dir}/.git" ]; then
32+
return 0
1133
fi
1234

13-
## Suppression list ##
14-
# This list will explain the detail of suppressed warnings.
15-
# The prototype of the item should be like:
16-
# "- [{file}] {spec}: {reason}"
17-
#
18-
# - [hello-1.c] unusedFunction: False positive of init_module and cleanup_module.
19-
# - [*.c] missingIncludeSystem: Focus on the example code, not the kernel headers.
35+
local remote_head local_head
36+
remote_head=$(git ls-remote "${repo_url}" HEAD 2>/dev/null | cut -f1)
37+
local_head=$(git -C "${repo_dir}" rev-parse HEAD 2>/dev/null)
2038

21-
local OPTS="
22-
--enable=warning,performance,information
23-
--suppress=unusedFunction:hello-1.c
24-
--suppress=missingIncludeSystem
25-
--std=c89 "
39+
if [ -z "${remote_head}" ] || [ -z "${local_head}" ]; then
40+
return 0
41+
fi
2642

27-
$CPPCHECK $OPTS --xml ${SOURCES} 2> cppcheck.xml
28-
local ERROR_COUNT=$(cat cppcheck.xml | grep -E -c "</error>" )
43+
if [ ! -f "${stamp_file}" ] || [ "$(cat "${stamp_file}")" != "${local_head}" ]; then
44+
return 0
45+
fi
2946

30-
if [ $ERROR_COUNT -gt 0 ]; then
31-
echo "Cppcheck failed: $ERROR_COUNT error(s)"
32-
cat cppcheck.xml
33-
exit 1
47+
[ "${remote_head}" != "${local_head}" ]
48+
}
49+
50+
sync_repo()
51+
{
52+
local repo_url=$1
53+
local repo_dir=$2
54+
55+
if [ -d "${repo_dir}/.git" ]; then
56+
git -C "${repo_dir}" fetch --depth=1 origin
57+
git -C "${repo_dir}" reset --hard FETCH_HEAD
58+
return
3459
fi
60+
61+
rm -rf "${repo_dir}"
62+
git clone --depth=1 "${repo_url}" "${repo_dir}"
3563
}
3664

37-
function do_sparse()
65+
do_cppcheck()
3866
{
39-
git clone git://git.kernel.org/pub/scm/devel/sparse/sparse.git --depth=1
40-
if [ $? -ne 0 ]; then
41-
echo "Failed to download sparse."
67+
local cppcheck_log="${LOG_DIR}/cppcheck.xml"
68+
local cppcheck_bin
69+
70+
cppcheck_bin=$(command -v cppcheck || true)
71+
if [ -z "${cppcheck_bin}" ]; then
72+
echo "[!] cppcheck not installed. Failed to run static analysis the source code." >&2
4273
exit 1
4374
fi
44-
pushd sparse
45-
make sparse || exit 1
46-
sudo make INST_PROGRAMS=sparse PREFIX=/usr install || exit 1
47-
popd
48-
local SPARSE=$(which sparse)
4975

50-
make -C examples C=2 CHECK="$SPARSE" 2> sparse.log
76+
"${cppcheck_bin}" \
77+
--enable=warning,performance,information \
78+
--suppress=unusedFunction:hello-1.c \
79+
--suppress=missingIncludeSystem \
80+
--std=c89 \
81+
--xml \
82+
"${SOURCES[@]}" \
83+
2> "${cppcheck_log}"
5184

52-
local WARNING_COUNT=$(cat sparse.log | grep -E -c " warning:" )
53-
local ERROR_COUNT=$(cat sparse.log | grep -E -c " error:" )
54-
local COUNT=`expr $WARNING_COUNT + $ERROR_COUNT`
55-
if [ $COUNT -gt 0 ]; then
56-
echo "Sparse failed: $WARNING_COUNT warning(s), $ERROR_COUNT error(s)"
57-
cat sparse.log
85+
local error_count
86+
error_count=$(count_matches "</error>" "${cppcheck_log}")
87+
if [ "${error_count}" -gt 0 ]; then
88+
echo "Cppcheck failed: ${error_count} error(s)"
89+
cat "${cppcheck_log}"
5890
exit 1
5991
fi
60-
make -C examples clean
6192
}
6293

63-
function do_gcc()
94+
do_sparse()
6495
{
65-
local GCC=$(which gcc)
66-
if [ $? -ne 0 ]; then
96+
local sparse_url="https://git.kernel.org/pub/scm/devel/sparse/sparse.git"
97+
local sparse_dir="${TOOL_CACHE_DIR}/sparse"
98+
local sparse_log="${LOG_DIR}/sparse.log"
99+
local sparse_bin="${sparse_dir}/sparse"
100+
local sparse_stamp="${sparse_dir}/.build-head"
101+
local warning_count
102+
local error_count
103+
local count
104+
105+
if needs_rebuild "${sparse_url}" "${sparse_dir}" "${sparse_bin}" "${sparse_stamp}"; then
106+
sync_repo "${sparse_url}" "${sparse_dir}"
107+
make -C "${sparse_dir}" HAVE_LLVM=no sparse
108+
git -C "${sparse_dir}" rev-parse HEAD > "${sparse_stamp}"
109+
fi
110+
111+
make -C examples clean >/dev/null 2>&1 || true
112+
if ! make -C examples C=2 CHECK="${sparse_bin}" 2> "${sparse_log}"; then
113+
cat "${sparse_log}"
114+
exit 1
115+
fi
116+
117+
warning_count=$(count_matches " warning:" "${sparse_log}")
118+
error_count=$(count_matches " error:" "${sparse_log}")
119+
count=$((warning_count + error_count))
120+
if [ "${count}" -gt 0 ]; then
121+
echo "Sparse failed: ${warning_count} warning(s), ${error_count} error(s)"
122+
cat "${sparse_log}"
123+
exit 1
124+
fi
125+
make -C examples clean >/dev/null 2>&1 || true
126+
}
127+
128+
do_gcc()
129+
{
130+
local gcc_log="${LOG_DIR}/gcc.log"
131+
local gcc_bin
132+
local warning_count
133+
local error_count
134+
local count
135+
136+
gcc_bin=$(command -v gcc || true)
137+
if [ -z "${gcc_bin}" ]; then
67138
echo "[!] gcc is not installed. Failed to run static analysis with GCC." >&2
68139
exit 1
69140
fi
70141

71-
make -C examples CONFIG_STATUS_CHECK_GCC=y STATUS_CHECK_GCC=$GCC 2> gcc.log
142+
make -C examples clean >/dev/null 2>&1 || true
143+
if ! make -C examples CONFIG_STATUS_CHECK_GCC=y STATUS_CHECK_GCC="${gcc_bin}" \
144+
2> "${gcc_log}"; then
145+
cat "${gcc_log}"
146+
exit 1
147+
fi
72148

73-
local WARNING_COUNT=$(cat gcc.log | grep -E -c " warning:" )
74-
local ERROR_COUNT=$(cat gcc.log | grep -E -c " error:" )
75-
local COUNT=`expr $WARNING_COUNT + $ERROR_COUNT`
76-
if [ $COUNT -gt 0 ]; then
77-
echo "gcc failed: $WARNING_COUNT warning(s), $ERROR_COUNT error(s)"
78-
cat gcc.log
149+
warning_count=$(count_matches " warning:" "${gcc_log}")
150+
error_count=$(count_matches " error:" "${gcc_log}")
151+
count=$((warning_count + error_count))
152+
if [ "${count}" -gt 0 ]; then
153+
echo "gcc failed: ${warning_count} warning(s), ${error_count} error(s)"
154+
cat "${gcc_log}"
79155
exit 1
80156
fi
81-
make -C examples CONFIG_STATUS_CHECK_GCC=y STATUS_CHECK_GCC=$GCC clean
157+
make -C examples CONFIG_STATUS_CHECK_GCC=y STATUS_CHECK_GCC="${gcc_bin}" clean \
158+
>/dev/null 2>&1 || true
82159
}
83160

84-
function do_smatch()
161+
do_smatch()
85162
{
86-
git clone https://github.com/error27/smatch.git --depth=1
87-
if [ $? -ne 0 ]; then
88-
echo "Failed to download smatch."
163+
local smatch_url="https://github.com/error27/smatch.git"
164+
local smatch_dir="${TOOL_CACHE_DIR}/smatch"
165+
local smatch_log="${LOG_DIR}/smatch.log"
166+
local smatch_bin="${smatch_dir}/smatch"
167+
local smatch_stamp="${smatch_dir}/.build-head"
168+
local warning_count
169+
local error_count
170+
local count
171+
172+
if needs_rebuild "${smatch_url}" "${smatch_dir}" "${smatch_bin}" "${smatch_stamp}"; then
173+
sync_repo "${smatch_url}" "${smatch_dir}"
174+
make -C "${smatch_dir}" smatch
175+
git -C "${smatch_dir}" rev-parse HEAD > "${smatch_stamp}"
176+
fi
177+
178+
make -C examples clean >/dev/null 2>&1 || true
179+
if ! make -C examples C=2 CHECK="${smatch_bin} -p=kernel" > "${smatch_log}" \
180+
2>&1; then
181+
cat "${smatch_log}"
89182
exit 1
90183
fi
91-
pushd smatch
92-
make smatch || exit 1
93-
local SMATCH=$(pwd)/smatch
94-
popd
95-
96-
make -C examples C=2 CHECK="$SMATCH -p=kernel" > smatch.log
97-
local WARNING_COUNT=$(cat smatch.log | egrep -c " warn:" )
98-
local ERROR_COUNT=$(cat smatch.log | egrep -c " error:" )
99-
local COUNT=`expr $WARNING_COUNT + $ERROR_COUNT`
100-
if [ $COUNT -gt 0 ]; then
101-
echo "Smatch failed: $WARNING_COUNT warning(s), $ERROR_COUNT error(s)"
102-
cat smatch.log | grep "warn:\|error:"
184+
185+
warning_count=$(count_matches " warn:" "${smatch_log}")
186+
error_count=$(count_matches " error:" "${smatch_log}")
187+
count=$((warning_count + error_count))
188+
if [ "${count}" -gt 0 ]; then
189+
echo "Smatch failed: ${warning_count} warning(s), ${error_count} error(s)"
190+
grep -E "warn:|error:" "${smatch_log}" || true
103191
exit 1
104192
fi
105-
make -C examples clean
193+
make -C examples clean >/dev/null 2>&1 || true
106194
}
107195

108196
do_cppcheck
109197
do_sparse
110198
do_gcc
111199
do_smatch
112-
exit 0

0 commit comments

Comments
 (0)