Skip to content

Commit e0c6143

Browse files
trz42trz42
authored
Merge pull request #1149 from hvelab/licenses
Determine missing license information for packages to be installed
2 parents 0d6d54e + 3b5280b commit e0c6143

10 files changed

Lines changed: 43474 additions & 0 deletions

File tree

‎.github/workflows/check_licenses.yml‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
name: Check and update licenses
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
pull_request:
7+
branches: [ "main" ]
8+
permissions:
9+
contents: read # we dont need to write
10+
11+
jobs:
12+
license_update:
13+
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
# we just do these two architectures for now as they are ones causing more discrepancies
18+
include:
19+
- runs_on: ubuntu-24.04-arm
20+
EESSI_VERSION: 2023.06
21+
EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic
22+
NO_SLASH_NAME: aarch64-generic
23+
- runs_on: ubuntu-24.04
24+
EESSI_VERSION: 2023.06
25+
EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
26+
NO_SLASH_NAME: x86_64-generic
27+
28+
runs-on: ${{ matrix.runs_on }}
29+
30+
steps:
31+
- name: Check out software-layer repository
32+
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
33+
34+
- name: Mount EESSI CernVM-FS repository
35+
uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
36+
with:
37+
cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
38+
cvmfs_http_proxy: DIRECT
39+
cvmfs_repositories: software.eessi.io
40+
41+
- name: Check for missing installations
42+
env:
43+
PR_NUMBER: ${{ github.event.number }}
44+
45+
run: |
46+
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
47+
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
48+
49+
# set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
50+
# to prevent issues with checks in the Easybuild configuration that use this variable
51+
export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
52+
export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
53+
export EESSI_OS_TYPE=linux
54+
env | grep ^EESSI | sort
55+
module load EasyBuild
56+
57+
# create a temporary directory to store the output
58+
LOCAL_TMPDIR=$(mktemp -d)
59+
eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
60+
echo "eb_missing_out=$LOCAL_TMPDIR/eb_missing.out" >> $GITHUB_ENV
61+
echo "Temporary directory created: ${eb_missing_out}"
62+
file_list=$(curl -sS \
63+
-H "Accept: application/vnd.github+json" \
64+
"https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?per_page=100" |
65+
jq -r '.[].filename | select(test("easystack"))')
66+
67+
if [ -z "$file_list" ]; then
68+
echo "No easystack files found. Skipping license check."
69+
exit 0
70+
fi
71+
72+
echo "Files to check:"
73+
echo $file_list
74+
75+
for easystack_file in $file_list; do
76+
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
77+
echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
78+
module purge
79+
module load EasyBuild/${eb_version}
80+
module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
81+
which eb
82+
${EB:-eb} --version
83+
${EB:-eb} --missing --easystack ${easystack_file} 2>&1 | tee ${eb_missing_out}
84+
85+
exit_code=${PIPESTATUS[0]}
86+
#echo "exit code for eb --missing --easystack ${easystack_file} is ${exit_code}"
87+
grep " required modules missing:" ${eb_missing_out} # > /dev/null
88+
exit_code=$?
89+
90+
if [[ ${exit_code} -eq 0 ]]; then
91+
echo "missing installations found for ${easystack_file}!" >&2;
92+
echo "PROCESS_LICENSES=true" >> $GITHUB_ENV
93+
else
94+
echo "no missing installations found for ${easystack_file}."
95+
exit 0
96+
fi
97+
done
98+
99+
- name: Check for software existing in licenses.yml file
100+
if: env.PROCESS_LICENSES == 'true'
101+
run: |
102+
# double check this
103+
if [ -s licenses/licenses.yml ]; then
104+
echo "licenses.yml file exists, checking for software versions that are not in the file..."
105+
echo "tmp file check: ${eb_missing_out}"
106+
# cat ${eb_missing_out}
107+
grep -oP '^\* \K[^ ]+' "${eb_missing_out}" | sort -u > missing.txt
108+
echo "Modules to check"
109+
cat missing.txt
110+
111+
# Check if software exists as key in YAML
112+
113+
while IFS= read -r module; do
114+
# module format: NAME/VERSION-TOOLCHAIN
115+
# e.g. ALL/0.9.2-foss-2023a
116+
117+
name="${module%%/*}" # ALL
118+
rest="${module#*/}" # 0.9.2-foss-2023a
119+
version="${rest%%-*}" # 0.9.2
120+
121+
# Check if licenses.yml has: NAME -> VERSION
122+
if ! yq -e ".\"$name\".\"$version\"" licenses/licenses.yml >/dev/null 2>&1; then
123+
echo "$module" >> missing_modules.txt
124+
fi
125+
done < missing.txt
126+
127+
echo "Modules not in licenses.yml: "
128+
cat missing_modules.txt
129+
130+
else
131+
echo "licenses.yml file does not exist? what happened?"
132+
exit 1
133+
fi
134+
135+
- name : Search sources for missing modules
136+
if: env.PROCESS_LICENSES == 'true'
137+
run: |
138+
if [ -s missing_modules.txt ]; then
139+
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
140+
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
141+
142+
echo "Searching sources for missing modules..."
143+
# Generates a "modules_results.json" file
144+
module load Python-bundle-PyPI/2023.06-GCCcore-12.3.0
145+
python licenses/parsing_easyconfigs.py missing_modules.txt
146+
cat modules_results.json
147+
fi
148+
149+
- name : Try to fetch the license
150+
if: env.PROCESS_LICENSES == 'true'
151+
run: |
152+
if [ -s modules_results.json ]; then
153+
export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
154+
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
155+
156+
echo "modules_results.json file exists, trying to fetch the license..."
157+
ml Python-bundle-PyPI/2023.06-GCCcore-12.3.0 BeautifulSoup/4.12.2-GCCcore-12.3.0 PyYAML/6.0-GCCcore-12.3.0
158+
python licenses/parse_licenses.py modules_results.json licenses/licenses.yml
159+
cat temporal_print.yaml
160+
else
161+
echo "modules_results.json file does not exist, skipping license fetch."
162+
fi
163+
164+
- name: Check and generate report on missing licenses
165+
if: env.PROCESS_LICENSES == 'true'
166+
run: |
167+
echo ""
168+
# Look for missing licences in licenses_aux.yaml
169+
OUTPUT=$(yq eval '.. | select(has("License")) | select(.License == "not found" or .License == "Other") | (path | join(" --> ")) + ": " + .License' licenses_aux.yaml)
170+
171+
# Check if the variable is NOT empty (-n)
172+
if [[ -n "$OUTPUT" ]]; then
173+
echo "Missing licenses found, please check the missing_report.yaml file."
174+
echo "$OUTPUT"
175+
echo "$OUTPUT" > missing_report.yaml
176+
else
177+
echo "No missing licenses found."
178+
fi
179+
180+
# Create a patch file
181+
diff -Naur licenses/licenses.yml licenses_aux.yaml > patch.txt || true
182+
echo "patch.txt file generated."
183+
184+
185+
- name: Generate artifacts
186+
if: env.PROCESS_LICENSES == 'true'
187+
uses: actions/upload-artifact@v4
188+
with:
189+
name: license-results-${{ matrix.NO_SLASH_NAME }}
190+
path: |
191+
missing_report.yaml
192+
patch.txt
193+
194+
195+
- name: How to edit artifacts and apply patch
196+
if: env.PROCESS_LICENSES == 'true'
197+
run: |
198+
echo "Artifacts generated. To resolve the missing licenses, please edit 'patch.txt' manually, making sure you follow the following format: "
199+
echo ""
200+
echo "<package_name>:"
201+
echo " <version_id>:"
202+
echo " License: <license_info>"
203+
echo " Permission to redistribute: <true/false>"
204+
echo " Retrieved from: <source_link>"
205+
echo ""
206+
echo " Once edited, you can apply the patch automatically using the patch command from the licenses directory: "
207+
208+
echo " patch < <path/to/patch.txt> "

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
*.pyc
22
*.pyo
3+
venv/

‎licenses/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
see https://spdx.org/licenses
2+
3+
Python function to download SPDX list of licenses is available in `spdx.py`

0 commit comments

Comments
 (0)