Skip to content

Commit a86cec6

Browse files
achamayouCopilot
achamayou
and
Copilot
authored
Copilot-enabled and faster CI checks (#7738)
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
1 parent c35077c commit a86cec6

14 files changed

Lines changed: 377 additions & 137 deletions

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,5 @@ tests/perf-system/analyzer/*.png
4545
**/*.ipynb*
4646
scripts/azure_deployment/.env
4747
.env
48-
python/src/ccf/version.py
48+
python/src/ccf/version.py
49+
scripts/env-*

‎scripts/ci-checks.sh‎

Lines changed: 76 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -2,164 +2,104 @@
22
# Copyright (c) Microsoft Corporation. All rights reserved.
33
# Licensed under the Apache 2.0 License.
44

5-
if [ "$1" == "-f" ]; then
6-
FIX=1
5+
if [ "${1:-}" == "-f" ]; then
6+
FIX_ARG="-f"
77
else
8-
FIX=0
8+
FIX_ARG=""
99
fi
1010

11-
# Replace numeric flag with list of failed groups
12-
FAIL=""
13-
1411
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
1512

1613
ROOT_DIR=$( dirname "$SCRIPT_DIR" )
1714
pushd "$ROOT_DIR" > /dev/null || exit 1
1815

1916
# GitHub actions workflow commands: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions
2017
function group(){
21-
# Track current group name
22-
CURRENT_GROUP="$1"
23-
# Only do this in GitHub actions, where CI is defined according to
24-
# https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
25-
if [[ ${CI} ]]; then
18+
if [[ ${CI:-} ]]; then
2619
echo "::group::$1"
2720
else
2821
echo "-=[ $1 ]=-"
2922
fi
3023
}
3124
function endgroup() {
32-
if [[ ${CI} ]]; then
25+
if [[ ${CI:-} ]]; then
3326
echo "::endgroup::"
3427
fi
3528
}
3629

37-
# Helper to record a failing group
38-
function fail() {
39-
if [[ -z "$FAIL" ]]; then
40-
FAIL="$CURRENT_GROUP"
30+
# --- Concurrent execution of all checks ---
31+
# Each check runs as a background job with output captured to a temp file.
32+
# After all jobs complete, outputs are printed in order with CI group annotations.
33+
34+
TMPDIR_CHECKS=$(mktemp -d) || { echo "Failed to create temporary directory for ci-checks" >&2; exit 1; }
35+
trap 'rm -rf "$TMPDIR_CHECKS"' EXIT
36+
37+
# Declare the checks: "group_name:script_name"
38+
CHECKS=(
39+
"Shell scripts:shellcheck-checks.sh"
40+
"TODOs:todo-checks.sh"
41+
"Includes:includes-checks.sh"
42+
"Release notes:release-notes-checks.sh"
43+
"C/C++ format:cpp-format-checks.sh"
44+
"TypeScript, JavaScript, Markdown, TypeSpec, YAML and JSON format:prettier-checks.sh"
45+
"OpenAPI:openapi-checks.sh"
46+
"Copyright notice headers:copyright-checks.sh"
47+
"CMake format:cmake-format-checks.sh"
48+
"Python format:python-format-checks.sh"
49+
"Python lint:python-lint-checks.sh"
50+
"Python types:python-types-checks.sh"
51+
)
52+
53+
PIDS=()
54+
for i in "${!CHECKS[@]}"; do
55+
IFS=: read -r _name script <<< "${CHECKS[$i]}"
56+
(
57+
start=$SECONDS
58+
# shellcheck disable=SC2086
59+
"$SCRIPT_DIR"/$script $FIX_ARG
60+
echo $? > "$TMPDIR_CHECKS/$i.rc"
61+
echo $((SECONDS - start)) > "$TMPDIR_CHECKS/$i.time"
62+
) > "$TMPDIR_CHECKS/$i.out" 2>&1 &
63+
PIDS+=($!)
64+
done
65+
66+
# Wait for all background jobs
67+
for pid in "${PIDS[@]}"; do
68+
wait "$pid" 2>/dev/null || true
69+
done
70+
71+
# Print results in order, track failures
72+
FAIL=""
73+
for i in "${!CHECKS[@]}"; do
74+
IFS=: read -r name _script <<< "${CHECKS[$i]}"
75+
rc=$(cat "$TMPDIR_CHECKS/$i.rc")
76+
77+
group "$name"
78+
cat "$TMPDIR_CHECKS/$i.out"
79+
if [ "$rc" != "0" ]; then
80+
if [ -z "$FAIL" ]; then
81+
FAIL="$name"
82+
else
83+
FAIL="$FAIL;$name"
84+
fi
85+
fi
86+
endgroup
87+
done
88+
89+
group "Timing"
90+
printf "%-70s %6s %s\n" "Check" "Time" "Status"
91+
printf "%-70s %6s %s\n" "-----" "----" "------"
92+
for i in "${!CHECKS[@]}"; do
93+
IFS=: read -r name _script <<< "${CHECKS[$i]}"
94+
rc=$(cat "$TMPDIR_CHECKS/$i.rc")
95+
elapsed=$(cat "$TMPDIR_CHECKS/$i.time")
96+
if [ "$rc" = "0" ]; then
97+
status="OK"
4198
else
42-
FAIL="$FAIL;$CURRENT_GROUP"
99+
status="FAIL"
43100
fi
44-
return 0
45-
}
46-
47-
group "Shell scripts"
48-
git ls-files | grep -e '\.sh$' | grep -E -v "^3rdparty" | xargs shellcheck -S warning -s bash || fail
49-
endgroup
50-
51-
# No inline TODOs in the codebase, use tickets, with a pointer to the code if necessary.
52-
group "TODOs"
53-
"$SCRIPT_DIR"/check-todo.sh . || fail
54-
endgroup
55-
56-
group "Public includes"
57-
# Enforce that no private headers are included from public header files
58-
violations=$(find "$ROOT_DIR/include/ccf" -type f -print0 | xargs --null grep -e "#include \"" | grep -v "#include \"ccf" | sort)
59-
if [[ -n "$violations" ]]; then
60-
echo "Public headers include private implementation files:"
61-
echo "$violations"
62-
fail
63-
else
64-
echo "No public-private include violations"
65-
fi
66-
endgroup
67-
68-
group "Public header namespaces"
69-
# Enforce that all public headers namespace their exports
70-
# NB: This only greps for a namespace definition in each file, doesn't precisely enforce that no types escape that namespace - mistakes are possible
71-
violations=$(find "$ROOT_DIR/include/ccf" -type f -name "*.h" -print0 | xargs --null grep -L "namespace ccf" | sort || true)
72-
if [[ -n "$violations" ]]; then
73-
echo "Public headers missing ccf namespace:"
74-
echo "$violations"
75-
fail
76-
else
77-
echo "No public header namespace violations"
78-
fi
79-
endgroup
80-
81-
group "Release notes"
82-
if [ $FIX -ne 0 ]; then
83-
"$SCRIPT_DIR"/extract-release-notes.py -f || fail
84-
else
85-
"$SCRIPT_DIR"/extract-release-notes.py || fail
86-
fi
87-
endgroup
88-
89-
group "C/C++ format"
90-
if [ $FIX -ne 0 ]; then
91-
"$SCRIPT_DIR"/check-format.sh -f include src samples || fail
92-
else
93-
"$SCRIPT_DIR"/check-format.sh include src samples || fail
94-
fi
95-
endgroup
96-
97-
group "Headers are included"
98-
"$SCRIPT_DIR"/headers-are-included.sh || fail
99-
endgroup
100-
101-
group "TypeScript, JavaScript, Markdown, TypeSpec, YAML and JSON format"
102-
npm install --loglevel=error --no-save prettier @typespec/prettier-plugin-typespec 1>/dev/null || fail
103-
if [ $FIX -ne 0 ]; then
104-
git ls-files | grep -e '\.ts$' -e '\.js$' -e '\.md$' -e '\.yaml$' -e '\.yml$' -e '\.json$' | grep -v -e 'tests/sandbox/' | xargs npx prettier --write || fail
105-
else
106-
git ls-files | grep -e '\.ts$' -e '\.js$' -e '\.md$' -e '\.yaml$' -e '\.yml$' -e '\.json$' | grep -v -e 'tests/sandbox/' | xargs npx prettier --check || fail
107-
fi
108-
endgroup
109-
110-
group "OpenAPI"
111-
npm install --loglevel=error --no-save @apidevtools/swagger-cli 1>/dev/null || fail
112-
find doc/schemas/*.json -exec npx swagger-cli validate {} \; || fail
113-
find doc/schemas/gov/*/*.json -exec npx swagger-cli validate {} \; || fail
114-
endgroup
115-
116-
group "Copyright notice headers"
117-
python3 "$SCRIPT_DIR"/notice-check.py || fail
118-
endgroup
119-
120-
group "CMake format"
121-
if [ $FIX -ne 0 ]; then
122-
"$SCRIPT_DIR"/check-cmake-format.sh -f cmake samples src tests CMakeLists.txt || fail
123-
else
124-
"$SCRIPT_DIR"/check-cmake-format.sh cmake samples src tests CMakeLists.txt || fail
125-
fi
126-
endgroup
127-
128-
group "Python dependencies"
129-
# Virtual Environment w/ dependencies for Python steps
130-
if [ ! -f "scripts/env/bin/activate" ]
131-
then
132-
python3 -m venv scripts/env
133-
fi
134-
135-
source scripts/env/bin/activate
136-
pip install -U pip || fail
137-
pip install -U wheel black pytest-mypy mypy ruff 1>/dev/null || fail
138-
endgroup
139-
140-
group "Python format"
141-
if [ $FIX -ne 0 ]; then
142-
git ls-files tests/ python/ scripts/ tla/ .cmake-format.py | grep -e '\.py$' | xargs black || fail
143-
else
144-
git ls-files tests/ python/ scripts/ tla/ .cmake-format.py | grep -e '\.py$' | xargs black --check || fail
145-
fi
146-
endgroup
147-
148-
group "Python lint dependencies"
149-
pip install -U -r tests/requirements.txt 1>/dev/null || fail
150-
pip install -U -e python 1>/dev/null || fail
151-
endgroup
152-
153-
group "Python lint"
154-
if [ $FIX -ne 0 ]; then
155-
ruff check --fix python/ tests/ || fail
156-
else
157-
ruff check python/ tests/ || fail
158-
fi
159-
endgroup
160-
161-
group "Python types"
162-
git ls-files python/ | grep -e '\.py$' | xargs mypy || fail
101+
printf "%-70s %5ds %s\n" "$name" "$elapsed" "$status"
102+
done
163103
endgroup
164104

165105
group "Summary"

‎scripts/cmake-format-checks.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the Apache 2.0 License.
4+
5+
# Checks (and optionally fixes) CMake file formatting.
6+
# Pass -f to auto-fix formatting issues.
7+
8+
set -uo pipefail
9+
10+
if [ "${1:-}" == "-f" ]; then
11+
FIX=1
12+
else
13+
FIX=0
14+
fi
15+
16+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
17+
ROOT_DIR=$( dirname "$SCRIPT_DIR" )
18+
cd "$ROOT_DIR" || exit 1
19+
20+
if [ $FIX -ne 0 ]; then
21+
"$SCRIPT_DIR"/check-cmake-format.sh -f cmake samples src tests CMakeLists.txt
22+
else
23+
"$SCRIPT_DIR"/check-cmake-format.sh cmake samples src tests CMakeLists.txt
24+
fi

‎scripts/copyright-checks.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the Apache 2.0 License.
4+
5+
# Checks that all source files have the correct copyright notice headers.
6+
# Accepts -f for interface consistency, but no auto-fix is available.
7+
8+
set -uo pipefail
9+
10+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
11+
ROOT_DIR=$( dirname "$SCRIPT_DIR" )
12+
cd "$ROOT_DIR" || exit 1
13+
14+
python3 "$SCRIPT_DIR"/notice-check.py

‎scripts/cpp-format-checks.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the Apache 2.0 License.
4+
5+
# Checks (and optionally fixes) C/C++ formatting via clang-format.
6+
# Pass -f to auto-fix formatting issues.
7+
8+
set -uo pipefail
9+
10+
if [ "${1:-}" == "-f" ]; then
11+
FIX=1
12+
else
13+
FIX=0
14+
fi
15+
16+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
17+
ROOT_DIR=$( dirname "$SCRIPT_DIR" )
18+
cd "$ROOT_DIR" || exit 1
19+
20+
if [ $FIX -ne 0 ]; then
21+
"$SCRIPT_DIR"/check-format.sh -f include src samples
22+
else
23+
"$SCRIPT_DIR"/check-format.sh include src samples
24+
fi

‎scripts/includes-checks.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the Apache 2.0 License.
4+
5+
# Validates public C++ headers:
6+
# 1. No private headers included from public headers
7+
# 2. All public headers declare a ccf namespace
8+
# 3. All exported headers are actually included somewhere
9+
# Accepts -f for interface consistency, but no auto-fix is available.
10+
11+
set -uo pipefail
12+
13+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
14+
ROOT_DIR=$( dirname "$SCRIPT_DIR" )
15+
cd "$ROOT_DIR" || exit 1
16+
17+
STATUS=0
18+
19+
# 1. Public includes: no private headers included from public header files
20+
echo "Checking public includes..."
21+
violations=$(find "$ROOT_DIR/include/ccf" -type f -print0 | xargs --null grep -e "#include \"" | grep -v "#include \"ccf" | sort)
22+
if [[ -n "$violations" ]]; then
23+
echo "Public headers include private implementation files:"
24+
echo "$violations"
25+
STATUS=1
26+
else
27+
echo "No public-private include violations"
28+
fi
29+
30+
# 2. Public header namespaces: all public headers namespace their exports
31+
echo "Checking public header namespaces..."
32+
violations=$(find "$ROOT_DIR/include/ccf" -type f -name "*.h" -print0 | xargs --null grep -L "namespace ccf" | sort || true)
33+
if [[ -n "$violations" ]]; then
34+
echo "Public headers missing ccf namespace:"
35+
echo "$violations"
36+
STATUS=1
37+
else
38+
echo "No public header namespace violations"
39+
fi
40+
41+
# 3. Headers are included: all exported headers are actually included somewhere
42+
echo "Checking headers are included..."
43+
"$SCRIPT_DIR"/headers-are-included.sh || STATUS=1
44+
45+
exit $STATUS

‎scripts/openapi-checks.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# Licensed under the Apache 2.0 License.
4+
5+
# Validates OpenAPI schema files via swagger-cli.
6+
# Accepts -f for interface consistency, but no auto-fix is available.
7+
8+
set -uo pipefail
9+
10+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
11+
ROOT_DIR=$( dirname "$SCRIPT_DIR" )
12+
cd "$ROOT_DIR" || exit 1
13+
14+
NPM_DIR=$(mktemp -d) || exit 1
15+
trap 'rm -rf "$NPM_DIR"' EXIT
16+
npm install --loglevel=error --no-save --prefix "$NPM_DIR" @apidevtools/swagger-cli 1>/dev/null || exit 1
17+
18+
find doc/schemas/*.json -exec npx --prefix "$NPM_DIR" swagger-cli validate {} \; || exit 1
19+
find doc/schemas/gov/*/*.json -exec npx --prefix "$NPM_DIR" swagger-cli validate {} \;

0 commit comments

Comments
 (0)