Skip to content

Commit 70cd15b

Browse files
Saumya-RSaumya-R
committed
removing the filters
1 parent a11b8ce commit 70cd15b

2 files changed

Lines changed: 173 additions & 189 deletions

File tree

‎.github/workflows/codeql-multiple-repo-scan.yml‎

Lines changed: 158 additions & 174 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,7 @@
1010
#
1111
# SPDX-License-Identifier: Apache-2.0
1212
# *******************************************************************************
13-
1413
name: "CodeQL - Multi-Repo Source Scan"
15-
1614
on:
1715
pull_request:
1816
types: [opened, reopened, synchronize]
@@ -24,10 +22,8 @@ on:
2422
release:
2523
types: [created]
2624
workflow_dispatch:
27-
2825
permissions:
2926
contents: write
30-
3127
jobs:
3228
analyze-repos:
3329
name: Analyze Multiple Repositories
@@ -37,176 +33,164 @@ jobs:
3733
packages: read
3834
actions: read
3935
contents: read
40-
4136
steps:
42-
- name: Checkout central repository
43-
uses: actions/checkout@v4
44-
45-
- name: Checkout CodeQL Coding Standards scripts
46-
uses: actions/checkout@v4
47-
with:
48-
repository: github/codeql-coding-standards
49-
path: codeql-coding-standards-repo # Klonen in diesen Ordner
50-
ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev'
51-
52-
# Add coding standard packages and dependencies
53-
- name: Install Python dependencies for Coding Standards scripts
54-
run: |
55-
python3 -m pip install --upgrade pip
56-
pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools
57-
58-
- name: Parse known_good.json and create repos.json
59-
id: parse-repos
60-
run: |
61-
sudo apt-get update && sudo apt-get install -y jq
62-
JSON_FILE="./known_good.json"
63-
64-
# Check if the file exists
65-
if [ ! -f "$JSON_FILE" ]; then
66-
echo "Error file not found '$JSON_FILE' "
67-
ls -la .
68-
exit 1
69-
fi
70-
71-
# Create repos.json from known_good.json
72-
# This jq command transforms the 'modules' object into an array of repository objects
73-
# with 'name', 'url', 'version' (branch/tag/hash), and 'path'.
74-
jq '[.modules.target_sw | to_entries[] | {
75-
name: .key,
76-
url: .value.repo,
77-
version: (.value.branch // .value.hash // .value.version),
78-
path: ("repos/" + .key)
79-
}]' "$JSON_FILE" > repos.json
80-
81-
echo "Generated repos.json:"
82-
cat repos.json
83-
echo "" # Add a newline for better readability
84-
85-
# The following GITHUB_OUTPUT variables are set for each module.
86-
# These might be useful for other steps, but are not directly used by the 'checkout-repos' step
87-
# which now reads 'repos.json' directly.
88-
echo "MODULE_COUNT=$(jq '.modules.target_sw | length' "$JSON_FILE")" >> $GITHUB_OUTPUT
89-
90-
jq -c '.modules.target_sw | to_entries[]' "$JSON_FILE" | while read -r module_entry; do
91-
module_name=$(echo "$module_entry" | jq -r '.key')
92-
repo_url=$(echo "$module_entry" | jq -r '.value.repo // empty')
93-
version=$(echo "$module_entry" | jq -r '.value.version // empty')
94-
branch=$(echo "$module_entry" | jq -r '.value.branch // empty')
95-
hash=$(echo "$module_entry" | jq -r '.value.hash // empty')
96-
97-
echo "${module_name}_url=$repo_url" >> $GITHUB_OUTPUT
98-
99-
if [ -n "$version" ]; then
100-
echo "${module_name}_version=$version" >> $GITHUB_OUTPUT
37+
- name: Checkout central repository
38+
uses: actions/checkout@v4
39+
- name: Checkout CodeQL Coding Standards scripts
40+
uses: actions/checkout@v4
41+
with:
42+
repository: github/codeql-coding-standards
43+
path: codeql-coding-standards-repo # Klonen in diesen Ordner
44+
ref: main # Oder eine spezifische Release-Version, z.B. 'v2.53.0-dev'
45+
# Add coding standard packages and dependencies
46+
- name: Install Python dependencies for Coding Standards scripts
47+
run: |
48+
python3 -m pip install --upgrade pip
49+
pip3 install pyyaml jsonpath-ng jsonschema jsonpatch jsonpointer pytest sarif-tools
50+
- name: Parse known_good.json and create repos.json
51+
id: parse-repos
52+
run: |
53+
sudo apt-get update && sudo apt-get install -y jq
54+
JSON_FILE="./known_good.json"
55+
56+
# Check if the file exists
57+
if [ ! -f "$JSON_FILE" ]; then
58+
echo "Error file not found '$JSON_FILE' "
59+
ls -la .
60+
exit 1
10161
fi
10262
103-
if [ -n "$branch" ]; then
104-
echo "${module_name}_branch=$branch" >> $GITHUB_OUTPUT
105-
fi
106-
107-
if [ -n "$hash" ]; then
108-
echo "${module_name}_hash=$hash" >> $GITHUB_OUTPUT
109-
fi
110-
done
111-
112-
- name: Checkout all pinned repositories
113-
id: checkout-repos
114-
run: |
115-
# jq is already installed by the previous step.
116-
117-
# Read repositories from the repos.json file created by the previous step
118-
repos=$(cat repos.json)
119-
repo_count=$(echo "$repos" | jq length)
120-
121-
# Initialize an empty string for paths to be outputted
122-
repo_paths_output=""
123-
124-
for i in $(seq 0 $((repo_count-1))); do
125-
name=$(echo "$repos" | jq -r ".[$i].name")
126-
url=$(echo "$repos" | jq -r ".[$i].url")
127-
ref=$(echo "$repos" | jq -r ".[$i].version") # This can be a branch, tag, or commit hash
128-
path=$(echo "$repos" | jq -r ".[$i].path") # e.g., "repos/score_baselibs"
129-
130-
echo "Checking out $name ($ref) to $path"
131-
132-
# Create the parent directory if it doesn't exist
133-
mkdir -p "$(dirname "$path")"
134-
135-
# Check if 'ref' looks like a commit hash (e.g., 40 hex characters)
136-
# This is a heuristic; a more robust check might involve fetching refs first.
137-
if [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then
138-
echo " Detected commit hash. Cloning and then checking out."
139-
git clone "$url" "$path"
140-
(cd "$path" && git checkout "$ref")
141-
else
142-
echo " Detected branch/tag. Cloning with --branch."
143-
git clone --depth 1 --branch v"$ref" "$url" "$path"
144-
fi
145-
146-
# Append the path to the list, separated by commas
147-
if [ -z "$repo_paths_output" ]; then
148-
repo_paths_output="$path"
149-
else
150-
repo_paths_output="$repo_paths_output,$path"
151-
fi
152-
done
153-
154-
# Output all paths as a single variable
155-
echo "repo_paths=$repo_paths_output" >> $GITHUB_OUTPUT
156-
157-
- name: Initialize CodeQL for all repositories
158-
uses: github/codeql-action/init@v4
159-
with:
160-
languages: cpp
161-
build-mode: none
162-
packs: codeql/misra-cpp-coding-standards
163-
config-file: ./.github/codeql/codeql-config.yml
164-
165-
- name: Perform CodeQL Analysis
166-
uses: github/codeql-action/analyze@v4
167-
with:
168-
upload-database: false # Don't upload databases for each repo
169-
output: sarif-results/
170-
category: "multi-repo-scan"
171-
172-
- name: Recategorize Guidelines
173-
if: always()
174-
run: |
175-
RECATEGORIZE_SCRIPT="codeql-coding-standards-repo/scripts/guideline_recategorization/recategorize.py"
176-
CODING_STANDARDS_CONFIG="./.github/codeql/coding-standards.yml"
177-
178-
CODING_STANDARDS_SCHEMA="codeql-coding-standards-repo/schemas/coding-standards-schema-1.0.0.json"
179-
SARIF_SCHEMA="codeql-coding-standards-repo/schemas/sarif-schema-2.1.0.json"
180-
181-
182-
SARIF_FILE="sarif-results/cpp.sarif"
183-
184-
mkdir -p sarif-results-recategorized
185-
echo "Processing $SARIF_FILE for recategorization..."
186-
python3 "$RECATEGORIZE_SCRIPT" \
187-
--coding-standards-schema-file "$CODING_STANDARDS_SCHEMA" \
188-
--sarif-schema-file "$SARIF_SCHEMA" \
189-
"$CODING_STANDARDS_CONFIG" \
190-
"$SARIF_FILE" \
191-
"sarif-results-recategorized/$(basename "$SARIF_FILE")"
192-
193-
rm "$SARIF_FILE"
194-
mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE"
195-
196-
- name: Generate HTML Report from SARIF
197-
run: |
198-
SARIF_FILE="sarif-results/cpp.sarif"
199-
sarif html "$SARIF_FILE" --output codeql-report.html
200-
201-
- name: Upload SARIF results as artifact
202-
uses: actions/upload-artifact@v4
203-
with:
204-
name: codeql-sarif-results
205-
path: sarif-results/
206-
207-
208-
- name: Upload HTML Report as artifact
209-
uses: actions/upload-artifact@v4
210-
with:
211-
name: codeql-html-report
212-
path: codeql-report.html
63+
# Create repos.json from known_good.json
64+
# This jq command transforms the 'modules' object into an array of repository objects
65+
# with 'name', 'url', 'version' (branch/tag/hash), and 'path'.
66+
jq '[.modules.target_sw | to_entries[] | {
67+
name: .key,
68+
url: .value.repo,
69+
version: (.value.branch // .value.hash // .value.version),
70+
path: ("repos/" + .key)
71+
}]' "$JSON_FILE" > repos.json
72+
73+
echo "Generated repos.json:"
74+
cat repos.json
75+
echo "" # Add a newline for better readability
76+
77+
# The following GITHUB_OUTPUT variables are set for each module.
78+
# These might be useful for other steps, but are not directly used by the 'checkout-repos' step
79+
# which now reads 'repos.json' directly.
80+
echo "MODULE_COUNT=$(jq '.modules.target_sw | length' "$JSON_FILE")" >> $GITHUB_OUTPUT
81+
82+
jq -c '.modules.target_sw | to_entries[]' "$JSON_FILE" | while read -r module_entry; do
83+
module_name=$(echo "$module_entry" | jq -r '.key')
84+
repo_url=$(echo "$module_entry" | jq -r '.value.repo // empty')
85+
version=$(echo "$module_entry" | jq -r '.value.version // empty')
86+
branch=$(echo "$module_entry" | jq -r '.value.branch // empty')
87+
hash=$(echo "$module_entry" | jq -r '.value.hash // empty')
88+
89+
echo "${module_name}_url=$repo_url" >> $GITHUB_OUTPUT
90+
91+
if [ -n "$version" ]; then
92+
echo "${module_name}_version=$version" >> $GITHUB_OUTPUT
93+
fi
94+
95+
if [ -n "$branch" ]; then
96+
echo "${module_name}_branch=$branch" >> $GITHUB_OUTPUT
97+
fi
98+
99+
if [ -n "$hash" ]; then
100+
echo "${module_name}_hash=$hash" >> $GITHUB_OUTPUT
101+
fi
102+
done
103+
- name: Checkout all pinned repositories
104+
id: checkout-repos
105+
run: |
106+
# jq is already installed by the previous step.
107+
108+
# Read repositories from the repos.json file created by the previous step
109+
repos=$(cat repos.json)
110+
repo_count=$(echo "$repos" | jq length)
111+
112+
# Initialize an empty string for paths to be outputted
113+
repo_paths_output=""
114+
115+
for i in $(seq 0 $((repo_count-1))); do
116+
name=$(echo "$repos" | jq -r ".[$i].name")
117+
url=$(echo "$repos" | jq -r ".[$i].url")
118+
ref=$(echo "$repos" | jq -r ".[$i].version") # This can be a branch, tag, or commit hash
119+
path=$(echo "$repos" | jq -r ".[$i].path") # e.g., "repos/score_baselibs"
120+
121+
echo "Checking out $name ($ref) to $path"
122+
123+
# Create the parent directory if it doesn't exist
124+
mkdir -p "$(dirname "$path")"
125+
126+
# Check if 'ref' looks like a commit hash (e.g., 40 hex characters)
127+
# This is a heuristic; a more robust check might involve fetching refs first.
128+
if [[ "$ref" =~ ^[0-9a-fA-F]{40}$ ]]; then
129+
echo " Detected commit hash. Cloning and then checking out."
130+
git clone "$url" "$path"
131+
(cd "$path" && git checkout "$ref")
132+
else
133+
echo " Detected branch/tag. Cloning with --branch."
134+
git clone --depth 1 --branch "$ref" "$url" "$path"
135+
fi
136+
137+
# Append the path to the list, separated by commas
138+
if [ -z "$repo_paths_output" ]; then
139+
repo_paths_output="$path"
140+
else
141+
repo_paths_output="$repo_paths_output,$path"
142+
fi
143+
done
144+
145+
# Output all paths as a single variable
146+
echo "repo_paths=$repo_paths_output" >> $GITHUB_OUTPUT
147+
- name: Initialize CodeQL for all repositories
148+
uses: github/codeql-action/init@v4
149+
with:
150+
languages: cpp
151+
build-mode: none
152+
packs: codeql/misra-cpp-coding-standards
153+
config-file: ./.github/codeql/codeql-config.yml
154+
- name: Perform CodeQL Analysis
155+
uses: github/codeql-action/analyze@v4
156+
with:
157+
upload-database: false # Don't upload databases for each repo
158+
output: sarif-results/
159+
category: "multi-repo-scan"
160+
- name: Recategorize Guidelines
161+
if: always()
162+
run: |
163+
RECATEGORIZE_SCRIPT="codeql-coding-standards-repo/scripts/guideline_recategorization/recategorize.py"
164+
CODING_STANDARDS_CONFIG="./.github/codeql/coding-standards.yml"
165+
166+
CODING_STANDARDS_SCHEMA="codeql-coding-standards-repo/schemas/coding-standards-schema-1.0.0.json"
167+
SARIF_SCHEMA="codeql-coding-standards-repo/schemas/sarif-schema-2.1.0.json"
168+
169+
170+
SARIF_FILE="sarif-results/cpp.sarif"
171+
172+
mkdir -p sarif-results-recategorized
173+
echo "Processing $SARIF_FILE for recategorization..."
174+
python3 "$RECATEGORIZE_SCRIPT" \
175+
--coding-standards-schema-file "$CODING_STANDARDS_SCHEMA" \
176+
--sarif-schema-file "$SARIF_SCHEMA" \
177+
"$CODING_STANDARDS_CONFIG" \
178+
"$SARIF_FILE" \
179+
"sarif-results-recategorized/$(basename "$SARIF_FILE")"
180+
181+
rm "$SARIF_FILE"
182+
mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE"
183+
- name: Generate HTML Report from SARIF
184+
run: |
185+
SARIF_FILE="sarif-results/cpp.sarif"
186+
sarif html "$SARIF_FILE" --output codeql-report.html
187+
- name: Upload SARIF results as artifact
188+
uses: actions/upload-artifact@v4
189+
with:
190+
name: codeql-sarif-results
191+
path: sarif-results/
192+
- name: Upload HTML Report as artifact
193+
uses: actions/upload-artifact@v4
194+
with:
195+
name: codeql-html-report
196+
path: codeql-report.html

‎scripts/workflow/recategorize_guidelines.sh‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -33,19 +33,19 @@ fi
3333
rm "$SARIF_FILE"
3434
mv "sarif-results-recategorized/$(basename "$SARIF_FILE")" "$SARIF_FILE"
3535

36-
# Ensure jq is available
37-
if ! command -v jq >/dev/null 2>&1; then
38-
echo "Error: jq is required but not installed. Please install jq and rerun this script." >&2
39-
exit 1
40-
fi
36+
# # Ensure jq is available
37+
# if ! command -v jq >/dev/null 2>&1; then
38+
# echo "Error: jq is required but not installed. Please install jq and rerun this script." >&2
39+
# exit 1
40+
# fi
4141

42-
# Filter SARIF to only include results from repos/* (relative or absolute)
43-
echo "Filtering SARIF results to only include entries with paths matching (^|/)repos/ ..."
44-
jq '(.runs) |= map(.results |= map(select((.locations // [] | length > 0) and ((.locations[0].physicalLocation.artifactLocation.uri // "") | test("(^|/)repos/")))) )' "$SARIF_FILE" > "${SARIF_FILE}.filtered"
45-
if [ $? -eq 0 ]; then
46-
mv "${SARIF_FILE}.filtered" "$SARIF_FILE"
47-
else
48-
echo "jq filtering failed. SARIF file was not modified." >&2
49-
rm -f "${SARIF_FILE}.filtered"
50-
exit 1
51-
fi
42+
# # Filter SARIF to only include results from repos/* (relative or absolute)
43+
# echo "Filtering SARIF results to only include entries with paths matching (^|/)repos/ ..."
44+
# jq '(.runs) |= map(.results |= map(select((.locations // [] | length > 0) and ((.locations[0].physicalLocation.artifactLocation.uri // "") | test("(^|/)repos/")))) )' "$SARIF_FILE" > "${SARIF_FILE}.filtered"
45+
# if [ $? -eq 0 ]; then
46+
# mv "${SARIF_FILE}.filtered" "$SARIF_FILE"
47+
# else
48+
# echo "jq filtering failed. SARIF file was not modified." >&2
49+
# rm -f "${SARIF_FILE}.filtered"
50+
# exit 1
51+
# fi

0 commit comments

Comments
 (0)