From c0473e8ee32092c64e40dc640f41e1d4a7da91ad Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 1 Apr 2026 16:02:51 +0100 Subject: [PATCH 1/7] :gear: ensure `setuptools` is installed --- .github/workflows/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml index c2d921a..89ed884 100644 --- a/.github/workflows/dependabot.yml +++ b/.github/workflows/dependabot.yml @@ -19,7 +19,7 @@ jobs: - name: Install CI/CD tools run: | python -m pip install --upgrade pip - python -m pip install continuous-delivery-scripts + python -m pip install setuptools continuous-delivery-scripts python -m pip list - name: Dependabot metadata id: dependabot-metadata From a603c39157a2b241d7382547ea75c81458e5bcf6 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 1 Apr 2026 16:06:47 +0100 Subject: [PATCH 2/7] Add setuptools installation to CI/CD workflows --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0be6df..6fd1aa5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - name: Install CI/CD tools run: | python -m pip install --upgrade pip - python -m pip install continuous-delivery-scripts + python -m pip install setuptools continuous-delivery-scripts python -m pip list - name: Assert news run: cd-assert-news -b ${CI_ACTION_REF_NAME} @@ -123,6 +123,7 @@ jobs: # location for development dependencies. run: | python -m pip install -r dev-requirements.txt + python -m pip install setuptools python -m pip list - name: Generate SPDX documents run: | @@ -146,7 +147,7 @@ jobs: - name: Install tools run: | python -m pip install --upgrade pip - python -m pip install detect-secrets==1.0.3 + python -m pip install setuptools detect-secrets==1.0.3 python -m pip list - uses: actions/checkout@v5 with: @@ -246,7 +247,7 @@ jobs: # is superfluous and eliminating pipenv in CI reduces overhead and reduce complexity, while retaining a single # location for development dependencies. run: | - pip install --upgrade flake8 + pip install --upgrade flake8 setuptools pip install -r dev-requirements.txt pip list - if: ${{ startsWith(matrix.python-version, '3.10') }} From 5b7a215760b49cd748e655d48fec206474082cc1 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 1 Apr 2026 16:07:03 +0100 Subject: [PATCH 3/7] Upgrade pipenv installation to include setuptools --- .github/workflows/mypy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index cd617bc..fcf5128 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,8 +19,8 @@ jobs: - name: Install pipenv shell: bash run: | - python -m pip install --upgrade pipenv + python -m pip install --upgrade pipenv setuptools pipenv install --dev - name: Type check with mypy run: | - pipenv run mypy ./continuous_delivery_scripts \ No newline at end of file + pipenv run mypy ./continuous_delivery_scripts From 1a39aec7c212cdbc2fdde589965603e1565c9ba9 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 1 Apr 2026 16:08:23 +0100 Subject: [PATCH 4/7] Add note to ensure setuptools is installed --- news/202604011608.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/202604011608.misc diff --git a/news/202604011608.misc b/news/202604011608.misc new file mode 100644 index 0000000..5527a78 --- /dev/null +++ b/news/202604011608.misc @@ -0,0 +1 @@ +:gear: Ensure `setuptools` is installed From b3e1bf6b45f913e8b9b9ca5b2c1924946b009fd8 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Thu, 2 Apr 2026 09:53:33 +0100 Subject: [PATCH 5/7] detect secrets --- .github/workflows/ci.yml | 2 +- .secrets.baseline | 361 ++++++--------------------------------- 2 files changed, 52 insertions(+), 311 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fd1aa5..9b7a97f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,7 +147,7 @@ jobs: - name: Install tools run: | python -m pip install --upgrade pip - python -m pip install setuptools detect-secrets==1.0.3 + python -m pip install setuptools detect-secrets[gibberish]==1.5.0 python -m pip list - uses: actions/checkout@v5 with: diff --git a/.secrets.baseline b/.secrets.baseline index fd5d6e4..3511e83 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,5 +1,5 @@ { - "version": "1.0.3", + "version": "1.5.0", "plugins_used": [ { "name": "ArtifactoryDetector" @@ -20,6 +20,15 @@ { "name": "CloudantDetector" }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, { "name": "HexHighEntropyString", "limit": 3.0 @@ -30,6 +39,9 @@ { "name": "IbmCosHmacDetector" }, + { + "name": "IPPublicDetector" + }, { "name": "JwtTokenDetector" }, @@ -43,9 +55,18 @@ { "name": "NpmDetector" }, + { + "name": "OpenAIDetector" + }, { "name": "PrivateKeyDetector" }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, { "name": "SlackDetector" }, @@ -58,6 +79,9 @@ { "name": "StripeDetector" }, + { + "name": "TelegramBotTokenDetector" + }, { "name": "TwilioKeyDetector" } @@ -70,12 +94,22 @@ "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", "min_level": 2 }, + { + "path": "detect_secrets.filters.gibberish.should_exclude_secret", + "limit": 3.7 + }, { "path": "detect_secrets.filters.heuristic.is_indirect_reference" }, { "path": "detect_secrets.filters.heuristic.is_likely_id_string" }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, { "path": "detect_secrets.filters.heuristic.is_potential_uuid" }, @@ -85,332 +119,39 @@ { "path": "detect_secrets.filters.heuristic.is_sequential_string" }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, { "path": "detect_secrets.filters.heuristic.is_templated_secret" }, { "path": "detect_secrets.filters.regex.should_exclude_file", "pattern": [ - ".*lock$", + ".*version\\.py$", ".*\\.html$", ".*\\.properties$", - "ci.yml", - "\\.git", - ".*version\\.py$" + "^\\.git[\\\\/]", + ".*lock$", + "^workflows/.*" ] } ], "results": { - "continuous_delivery_scripts/plugins/python.py": [ - { - "type": "Secret Keyword", - "filename": "continuous_delivery_scripts/plugins/python.py", - "hashed_secret": "a5c8a5463c338b832f4afbe616ae77b4ecbfadc0", - "is_verified": false, - "line_number": 26 - } - ], - "continuous_delivery_scripts/utils/git_helpers.py": [ + ".idea/workspace.xml": [ { - "type": "Basic Auth Credentials", - "filename": "continuous_delivery_scripts/utils/git_helpers.py", - "hashed_secret": "9af3842326eadff7d85a55edfec67f6d8a3a259f", + "type": "Base64 High Entropy String", + "filename": ".idea/workspace.xml", + "hashed_secret": "2966c0d874788d9ab39b18d8a2d66643ca16e2ce", "is_verified": false, - "line_number": 48 + "line_number": 19 } ], - "tests/git_helper/htmlcov/status.json": [ - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "04d419908b7252b2ef212169cc9c21c2bd29ea40", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "050f2a359af8be6414fc7d687faad6ff0052e3b2", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "09b3ebdf06940026dcb3db9d7b58a678ab017d17", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "0c3863eb9120ba7da836f60d03b94842253d83cc", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "110fe0824bd97ad868e38ae15fc09cd6bc316044", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "2aa79091fb83bc451cae02a5209c00cfb6797ffa", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "2ed89a3b458cb19feeab91e5feca1f10e673ed83", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "34165846b4183d699107a8dea1b9863c4a701edb", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "45305fd86bbc707df345a6b61afb1cb4da102ce5", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "453085ba3abdc747d861c5bda0e40e5510a5579a", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "4a79e4bcd8f22c177a9fa75cd2ed72c3f677b8c3", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "4a962c73e5aea51ec01adb3c66fcf7eec1da2302", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "5d17fb8cef1f8b8c387f03b8aeeb1a6939d68eb3", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "610ef218d9dc8a979f1aae7305165bec17c7e462", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "650f1262ee224b19477559521e05d06fc5aa73c2", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "65a7f66288425b59aafe4b055a289dc64c0f2e41", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "65d23e2c392259aaacf9d305c1f2534f330fa5ba", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "6c6297bca47843d380ece16525c1fd65cba9e476", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "7a308e32d2a838a8b30bd50df47af433e807e176", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "7cf5ba01ea7fb43d6e5e754f3633d98dfe60f4dc", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "802151a187d206f1a4a9bb18c1cc1380c88debbe", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "8a060895e913a6287477a2979211375c1158903f", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "9bf1485b635cd899e75560f2e12d961b4de8a8b5", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "a13a58d66198efa2034d79a8ceca89489e42dc51", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "adec03f51f25ee3a2f1e08eb1ef37a3903f5a598", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "b1c37f9e4bac61cda8e65db8d2fa078134c08f42", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "b451e734449989581d706bd871226d4e69c82fd8", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "bad28fd1f8ced89042db54a99489ff218adf2781", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "be8545bbf9db8142dafbbe009f0c3270e1706149", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "bfef4f4099f1a0e67da504d7585a62dd9637debb", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "d0475b0dc36315158ed370f8593753c73e45fcd9", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "d1bb83f1dfb98f75005cdf2e4b411aa4988c450a", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "d37db94cfcc9e9b9f5dfc076e6ee9b61a3f05a73", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "ddc3f7b1eaf9f7c9594b24f42f64c3d20b5859f1", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "de7742fd0fe190d6005dd9ab2f9199d16653c09a", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "df8e94f550ba44920e1d9bf0b5a329c08d04be1c", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "e0d5118fc04eb60902c0b261d7caca26ebf54195", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "e66b8c7ee2964ff11609b6ba2b8a0fb4ad8acfc3", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "ec918cae5924a393015b1609c799773191d4b6a9", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "f1772b3a0bc082a6e7efac4458858fb4c9a9c1c4", - "is_verified": false, - "line_number": 1 - }, - { - "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "fa60226ffdd4e7d8041f769e0ded5e2523633485", - "is_verified": false, - "line_number": 1 - }, + ".ruff_cache/CACHEDIR.TAG": [ { "type": "Hex High Entropy String", - "filename": "tests/git_helper/htmlcov/status.json", - "hashed_secret": "ff01e21ec0e3ff778db86bd6d2b23e66c8dd31b5", + "filename": ".ruff_cache/CACHEDIR.TAG", + "hashed_secret": "e8f8c345877b2411a59897798e422b15b0c16d76", "is_verified": false, "line_number": 1 } @@ -425,5 +166,5 @@ } ] }, - "generated_at": "2023-02-28T00:31:43Z" + "generated_at": "2026-04-02T08:50:35Z" } From 9fce459a9b3e09b389d5d78ff4be39941ebbfa9b Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Thu, 2 Apr 2026 10:13:03 +0100 Subject: [PATCH 6/7] :gear: reformatted and added `setuptools` to the list of dependencies --- .secrets.baseline | 2 +- continuous_delivery_scripts/__init__.py | 1 + continuous_delivery_scripts/_version.py | 1 + continuous_delivery_scripts/assert_news.py | 1 + continuous_delivery_scripts/create_news_file.py | 1 + continuous_delivery_scripts/generate_docs.py | 1 + continuous_delivery_scripts/generate_news.py | 1 + continuous_delivery_scripts/get_config.py | 1 + continuous_delivery_scripts/get_version.py | 1 + continuous_delivery_scripts/license_files.py | 1 + continuous_delivery_scripts/plugins/__init__.py | 1 + continuous_delivery_scripts/plugins/basic.py | 1 + continuous_delivery_scripts/plugins/ci.py | 1 + continuous_delivery_scripts/plugins/docker.py | 1 + continuous_delivery_scripts/plugins/github_actions.py | 1 + continuous_delivery_scripts/plugins/golang.py | 1 + continuous_delivery_scripts/plugins/noop.py | 1 + continuous_delivery_scripts/plugins/python.py | 1 + continuous_delivery_scripts/report_third_party_ip.py | 1 + continuous_delivery_scripts/spdx_report/spdx_document.py | 1 + continuous_delivery_scripts/spdx_report/spdx_helpers.py | 1 + continuous_delivery_scripts/spdx_report/spdx_summary.py | 1 + continuous_delivery_scripts/tag_and_release.py | 1 + continuous_delivery_scripts/utils/aws_helpers.py | 1 + continuous_delivery_scripts/utils/configuration.py | 1 + continuous_delivery_scripts/utils/definitions.py | 1 + continuous_delivery_scripts/utils/filesystem_helpers.py | 1 + continuous_delivery_scripts/utils/git_helpers.py | 1 + continuous_delivery_scripts/utils/hash_helpers.py | 1 + continuous_delivery_scripts/utils/logging.py | 1 + continuous_delivery_scripts/utils/news_file.py | 1 + continuous_delivery_scripts/utils/string_helpers.py | 1 + continuous_delivery_scripts/utils/third_party_licences.py | 1 + setup.py | 3 +++ tests/tag_and_release/test_update_documentation_python.py | 1 + 35 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.secrets.baseline b/.secrets.baseline index 3511e83..af9362d 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -166,5 +166,5 @@ } ] }, - "generated_at": "2026-04-02T08:50:35Z" + "generated_at": "2026-04-02T09:12:28Z" } diff --git a/continuous_delivery_scripts/__init__.py b/continuous_delivery_scripts/__init__.py index 0ce526b..75547a6 100644 --- a/continuous_delivery_scripts/__init__.py +++ b/continuous_delivery_scripts/__init__.py @@ -3,4 +3,5 @@ # SPDX-License-Identifier: Apache-2.0 # """Scripts and utilities used by the CI pipeline.""" + from continuous_delivery_scripts._version import __version__ # noqa: F401 diff --git a/continuous_delivery_scripts/_version.py b/continuous_delivery_scripts/_version.py index a5a0fe3..f2fc8f4 100644 --- a/continuous_delivery_scripts/_version.py +++ b/continuous_delivery_scripts/_version.py @@ -10,6 +10,7 @@ This file is autogenerated, do not modify by hand. """ + __version__ = "3.2.6" COMMIT = "39ef9fc9203e6633b9e8df6ce7f7ee0067de27ff" MAJOR = 3 diff --git a/continuous_delivery_scripts/assert_news.py b/continuous_delivery_scripts/assert_news.py index 4370f9b..b08cb5d 100644 --- a/continuous_delivery_scripts/assert_news.py +++ b/continuous_delivery_scripts/assert_news.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Checks if valid news files are created for changes in the project.""" + import argparse import logging import pathlib diff --git a/continuous_delivery_scripts/create_news_file.py b/continuous_delivery_scripts/create_news_file.py index 4312c6c..f15fcda 100644 --- a/continuous_delivery_scripts/create_news_file.py +++ b/continuous_delivery_scripts/create_news_file.py @@ -7,6 +7,7 @@ Usage: create-news-file "Fixed a bug" --type bugfix """ + import argparse import logging diff --git a/continuous_delivery_scripts/generate_docs.py b/continuous_delivery_scripts/generate_docs.py index e7ca081..4ee3c29 100644 --- a/continuous_delivery_scripts/generate_docs.py +++ b/continuous_delivery_scripts/generate_docs.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Generates documentation.""" + import argparse import logging import os diff --git a/continuous_delivery_scripts/generate_news.py b/continuous_delivery_scripts/generate_news.py index 1b7ed25..e1cb48a 100644 --- a/continuous_delivery_scripts/generate_news.py +++ b/continuous_delivery_scripts/generate_news.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Handles usage of towncrier for automated changelog generation and pyautoversion for versioning.""" + import sys import argparse diff --git a/continuous_delivery_scripts/get_config.py b/continuous_delivery_scripts/get_config.py index d6148fd..7de935a 100644 --- a/continuous_delivery_scripts/get_config.py +++ b/continuous_delivery_scripts/get_config.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Retrieves configuration values.""" + import argparse import sys import logging diff --git a/continuous_delivery_scripts/get_version.py b/continuous_delivery_scripts/get_version.py index 1584e65..48aa96c 100644 --- a/continuous_delivery_scripts/get_version.py +++ b/continuous_delivery_scripts/get_version.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Determine the project new version.""" + import sys from typing import Optional diff --git a/continuous_delivery_scripts/license_files.py b/continuous_delivery_scripts/license_files.py index ed9abb3..7867109 100644 --- a/continuous_delivery_scripts/license_files.py +++ b/continuous_delivery_scripts/license_files.py @@ -7,6 +7,7 @@ This is to comply with OpenChain certification; https://github.com/OpenChain-Project/Curriculum/blob/master/guides/reusing_software.md#2-include-a-copyright-notice-and-license-in-each-file """ + import argparse import logging import subprocess diff --git a/continuous_delivery_scripts/plugins/__init__.py b/continuous_delivery_scripts/plugins/__init__.py index 1b775d6..59134d8 100644 --- a/continuous_delivery_scripts/plugins/__init__.py +++ b/continuous_delivery_scripts/plugins/__init__.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Language specific actions.""" + import glob from pathlib import Path diff --git a/continuous_delivery_scripts/plugins/basic.py b/continuous_delivery_scripts/plugins/basic.py index 721a964..ef5675d 100644 --- a/continuous_delivery_scripts/plugins/basic.py +++ b/continuous_delivery_scripts/plugins/basic.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Basic plugin.""" + import logging from pathlib import Path from typing import Optional diff --git a/continuous_delivery_scripts/plugins/ci.py b/continuous_delivery_scripts/plugins/ci.py index 975b99c..90ff42d 100644 --- a/continuous_delivery_scripts/plugins/ci.py +++ b/continuous_delivery_scripts/plugins/ci.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Plugin for CI projects.""" + import logging from pathlib import Path from typing import Optional diff --git a/continuous_delivery_scripts/plugins/docker.py b/continuous_delivery_scripts/plugins/docker.py index 9e140f2..00d7839 100644 --- a/continuous_delivery_scripts/plugins/docker.py +++ b/continuous_delivery_scripts/plugins/docker.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Plugin for Docker projects.""" + import logging from pathlib import Path from typing import Optional diff --git a/continuous_delivery_scripts/plugins/github_actions.py b/continuous_delivery_scripts/plugins/github_actions.py index a592789..663596c 100644 --- a/continuous_delivery_scripts/plugins/github_actions.py +++ b/continuous_delivery_scripts/plugins/github_actions.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Plugin for CI projects.""" + import logging import os from pathlib import Path diff --git a/continuous_delivery_scripts/plugins/golang.py b/continuous_delivery_scripts/plugins/golang.py index 8c3204e..0da7d1b 100644 --- a/continuous_delivery_scripts/plugins/golang.py +++ b/continuous_delivery_scripts/plugins/golang.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Plugin for Golang projects.""" + import logging import os from pathlib import Path diff --git a/continuous_delivery_scripts/plugins/noop.py b/continuous_delivery_scripts/plugins/noop.py index 050a663..cd9cad7 100644 --- a/continuous_delivery_scripts/plugins/noop.py +++ b/continuous_delivery_scripts/plugins/noop.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """No Operation plugin.""" + import logging from pathlib import Path from typing import Optional, Dict diff --git a/continuous_delivery_scripts/plugins/python.py b/continuous_delivery_scripts/plugins/python.py index c69f2ce..7769263 100644 --- a/continuous_delivery_scripts/plugins/python.py +++ b/continuous_delivery_scripts/plugins/python.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Plugin for Python projects.""" + import logging import shutil import sys diff --git a/continuous_delivery_scripts/report_third_party_ip.py b/continuous_delivery_scripts/report_third_party_ip.py index 741599c..9805419 100644 --- a/continuous_delivery_scripts/report_third_party_ip.py +++ b/continuous_delivery_scripts/report_third_party_ip.py @@ -11,6 +11,7 @@ supported so that third-party IP gets documented as described by the specification (i.e. with relationships). """ + import argparse import logging import sys diff --git a/continuous_delivery_scripts/spdx_report/spdx_document.py b/continuous_delivery_scripts/spdx_report/spdx_document.py index 82d7fe8..54281c1 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_document.py +++ b/continuous_delivery_scripts/spdx_report/spdx_document.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Definition of an SPDX Document.""" + from pathlib import Path from spdx.creationinfo import Person, Organization, Tool from spdx.document import Document, License diff --git a/continuous_delivery_scripts/spdx_report/spdx_helpers.py b/continuous_delivery_scripts/spdx_report/spdx_helpers.py index 4987b8d..adbbfca 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_helpers.py +++ b/continuous_delivery_scripts/spdx_report/spdx_helpers.py @@ -13,6 +13,7 @@ - https://spdx.org/spdx-tagvalue-example - https://github.com/spdx/tools/blob/master/Examples/SPDXTagExample-v2.1.spdx """ + import re import logging diff --git a/continuous_delivery_scripts/spdx_report/spdx_summary.py b/continuous_delivery_scripts/spdx_report/spdx_summary.py index 18f3890..7b4ab8f 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_summary.py +++ b/continuous_delivery_scripts/spdx_report/spdx_summary.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Summary generators.""" + import datetime import jinja2 import logging diff --git a/continuous_delivery_scripts/tag_and_release.py b/continuous_delivery_scripts/tag_and_release.py index 70a935a..d4fe719 100644 --- a/continuous_delivery_scripts/tag_and_release.py +++ b/continuous_delivery_scripts/tag_and_release.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Orchestrates release process.""" + import argparse import datetime import logging diff --git a/continuous_delivery_scripts/utils/aws_helpers.py b/continuous_delivery_scripts/utils/aws_helpers.py index eb9a345..45ef2f9 100644 --- a/continuous_delivery_scripts/utils/aws_helpers.py +++ b/continuous_delivery_scripts/utils/aws_helpers.py @@ -6,6 +6,7 @@ Based on the Python SDK called BOTO https://aws.amazon.com/sdk-for-python/. """ + import boto3 import logging import os diff --git a/continuous_delivery_scripts/utils/configuration.py b/continuous_delivery_scripts/utils/configuration.py index 01e36f4..db805ec 100644 --- a/continuous_delivery_scripts/utils/configuration.py +++ b/continuous_delivery_scripts/utils/configuration.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Utilities in charge of fetching configuration values for the ci scripts.""" + import enum import logging import os diff --git a/continuous_delivery_scripts/utils/definitions.py b/continuous_delivery_scripts/utils/definitions.py index 3297a6b..a25efd1 100644 --- a/continuous_delivery_scripts/utils/definitions.py +++ b/continuous_delivery_scripts/utils/definitions.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Place to store generic project concepts for the ci scripts.""" + import enum from typing import List diff --git a/continuous_delivery_scripts/utils/filesystem_helpers.py b/continuous_delivery_scripts/utils/filesystem_helpers.py index 0ca2100..56a0195 100644 --- a/continuous_delivery_scripts/utils/filesystem_helpers.py +++ b/continuous_delivery_scripts/utils/filesystem_helpers.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Helpers with regards to actions on the filesystem.""" + import os import tempfile import shutil diff --git a/continuous_delivery_scripts/utils/git_helpers.py b/continuous_delivery_scripts/utils/git_helpers.py index 30d3f01..08b700e 100644 --- a/continuous_delivery_scripts/utils/git_helpers.py +++ b/continuous_delivery_scripts/utils/git_helpers.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Utility script to abstract git operations for our CI scripts.""" + import logging import os import re diff --git a/continuous_delivery_scripts/utils/hash_helpers.py b/continuous_delivery_scripts/utils/hash_helpers.py index f6aa879..cb14db0 100644 --- a/continuous_delivery_scripts/utils/hash_helpers.py +++ b/continuous_delivery_scripts/utils/hash_helpers.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Helpers for calculating hashes or UUID.""" + import hashlib import uuid from pathlib import Path diff --git a/continuous_delivery_scripts/utils/logging.py b/continuous_delivery_scripts/utils/logging.py index 16e5117..a03373c 100644 --- a/continuous_delivery_scripts/utils/logging.py +++ b/continuous_delivery_scripts/utils/logging.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Helpers for logging errors according to severity of the exception.""" + import logging from .configuration import configuration, ConfigurationVariable from typing import Any, Optional diff --git a/continuous_delivery_scripts/utils/news_file.py b/continuous_delivery_scripts/utils/news_file.py index 68a4975..c60c9f7 100644 --- a/continuous_delivery_scripts/utils/news_file.py +++ b/continuous_delivery_scripts/utils/news_file.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Helpers with regards to news files.""" + import enum import pathlib import logging diff --git a/continuous_delivery_scripts/utils/string_helpers.py b/continuous_delivery_scripts/utils/string_helpers.py index 2c58a78..4948c0b 100644 --- a/continuous_delivery_scripts/utils/string_helpers.py +++ b/continuous_delivery_scripts/utils/string_helpers.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Utilities regarding string handling.""" + from functools import total_ordering import jellyfish diff --git a/continuous_delivery_scripts/utils/third_party_licences.py b/continuous_delivery_scripts/utils/third_party_licences.py index 4860f4a..5a6bf91 100644 --- a/continuous_delivery_scripts/utils/third_party_licences.py +++ b/continuous_delivery_scripts/utils/third_party_licences.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Third party licences.""" + import re import json diff --git a/setup.py b/setup.py index 51bd61e..a7c408f 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Package definition for PyPI.""" + import os from setuptools import setup @@ -52,6 +53,8 @@ keywords="Arm Tools CI CD Continuous Delivery Scripts Automation", include_package_data=True, install_requires=[ + # spdx-tools imports pkg_resources from setuptools at runtime. + "setuptools", "gitpython", "towncrier==22.12.0", "pyautoversion~=1.2.0", diff --git a/tests/tag_and_release/test_update_documentation_python.py b/tests/tag_and_release/test_update_documentation_python.py index 4a60735..0a254c8 100644 --- a/tests/tag_and_release/test_update_documentation_python.py +++ b/tests/tag_and_release/test_update_documentation_python.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # """Tests for documentation update functions.""" + import os from unittest import mock, TestCase From 4247506e92697dd05ee3444608b562040093d850 Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Thu, 2 Apr 2026 11:40:03 +0100 Subject: [PATCH 7/7] :green_heart: Fix CI --- .gitignore | 2 +- .secrets.baseline | 20 +---- continuous_delivery_scripts/plugins/basic.py | 13 ++- continuous_delivery_scripts/plugins/ci.py | 18 ++-- continuous_delivery_scripts/plugins/docker.py | 13 ++- continuous_delivery_scripts/plugins/golang.py | 29 +++++-- continuous_delivery_scripts/plugins/noop.py | 13 ++- continuous_delivery_scripts/plugins/python.py | 44 +++++++--- .../report_third_party_ip.py | 24 ++++-- .../spdx_report/spdx_dependency.py | 11 ++- .../spdx_report/spdx_document.py | 56 ++++++++---- .../spdx_report/spdx_file.py | 34 +++++--- .../spdx_report/spdx_helpers.py | 44 ++++++++-- .../spdx_report/spdx_package.py | 46 ++++++---- .../spdx_report/spdx_project.py | 28 ++++-- .../spdx_report/spdx_summary.py | 39 ++++++--- .../tag_and_release.py | 39 +++++++-- .../utils/language_specifics_base.py | 13 ++- .../utils/package_helpers.py | 31 ++++--- .../utils/python/package_helpers.py | 86 ++++++++++++++++--- .../utils/third_party_licences.py | 29 +++++-- 21 files changed, 448 insertions(+), 184 deletions(-) diff --git a/.gitignore b/.gitignore index 3a169aa..ca3c9b8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ Pipfile.lock # PyCharm -.idea/ +.idea # macOS .DS_Store diff --git a/.secrets.baseline b/.secrets.baseline index af9362d..304882c 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -138,24 +138,6 @@ } ], "results": { - ".idea/workspace.xml": [ - { - "type": "Base64 High Entropy String", - "filename": ".idea/workspace.xml", - "hashed_secret": "2966c0d874788d9ab39b18d8a2d66643ca16e2ce", - "is_verified": false, - "line_number": 19 - } - ], - ".ruff_cache/CACHEDIR.TAG": [ - { - "type": "Hex High Entropy String", - "filename": ".ruff_cache/CACHEDIR.TAG", - "hashed_secret": "e8f8c345877b2411a59897798e422b15b0c16d76", - "is_verified": false, - "line_number": 1 - } - ], "tests/hash/test_hash_generation.py": [ { "type": "Hex High Entropy String", @@ -166,5 +148,5 @@ } ] }, - "generated_at": "2026-04-02T09:12:28Z" + "generated_at": "2026-04-02T13:54:15Z" } diff --git a/continuous_delivery_scripts/plugins/basic.py b/continuous_delivery_scripts/plugins/basic.py index ef5675d..63968e7 100644 --- a/continuous_delivery_scripts/plugins/basic.py +++ b/continuous_delivery_scripts/plugins/basic.py @@ -6,11 +6,16 @@ import logging from pathlib import Path -from typing import Optional +from typing import TYPE_CHECKING, Optional -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject from continuous_delivery_scripts.utils.definitions import CommitType -from continuous_delivery_scripts.utils.language_specifics_base import BaseLanguage, get_language_from_file_name +from continuous_delivery_scripts.utils.language_specifics_base import ( + BaseLanguage, + get_language_from_file_name, +) + +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject logger = logging.getLogger(__name__) @@ -46,6 +51,6 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """Basic.""" return None diff --git a/continuous_delivery_scripts/plugins/ci.py b/continuous_delivery_scripts/plugins/ci.py index 90ff42d..bcbfa0e 100644 --- a/continuous_delivery_scripts/plugins/ci.py +++ b/continuous_delivery_scripts/plugins/ci.py @@ -6,12 +6,20 @@ import logging from pathlib import Path -from typing import Optional +from typing import TYPE_CHECKING, Optional -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject -from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable +from continuous_delivery_scripts.utils.configuration import ( + configuration, + ConfigurationVariable, +) from continuous_delivery_scripts.utils.definitions import CommitType -from continuous_delivery_scripts.utils.language_specifics_base import BaseLanguage, get_language_from_file_name +from continuous_delivery_scripts.utils.language_specifics_base import ( + BaseLanguage, + get_language_from_file_name, +) + +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject logger = logging.getLogger(__name__) @@ -53,7 +61,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None diff --git a/continuous_delivery_scripts/plugins/docker.py b/continuous_delivery_scripts/plugins/docker.py index 00d7839..42f2287 100644 --- a/continuous_delivery_scripts/plugins/docker.py +++ b/continuous_delivery_scripts/plugins/docker.py @@ -6,11 +6,16 @@ import logging from pathlib import Path -from typing import Optional +from typing import TYPE_CHECKING, Optional -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject from continuous_delivery_scripts.utils.definitions import CommitType -from continuous_delivery_scripts.utils.language_specifics_base import BaseLanguage, get_language_from_file_name +from continuous_delivery_scripts.utils.language_specifics_base import ( + BaseLanguage, + get_language_from_file_name, +) + +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject logger = logging.getLogger(__name__) @@ -48,7 +53,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None diff --git a/continuous_delivery_scripts/plugins/golang.py b/continuous_delivery_scripts/plugins/golang.py index 0da7d1b..e1b1af1 100644 --- a/continuous_delivery_scripts/plugins/golang.py +++ b/continuous_delivery_scripts/plugins/golang.py @@ -8,13 +8,24 @@ import os from pathlib import Path from subprocess import check_call -from typing import Optional, List, Dict +from typing import TYPE_CHECKING, Optional, List, Dict -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject -from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable +from continuous_delivery_scripts.utils.configuration import ( + configuration, + ConfigurationVariable, +) from continuous_delivery_scripts.utils.definitions import CommitType -from continuous_delivery_scripts.utils.git_helpers import LocalProjectRepository, GitWrapper -from continuous_delivery_scripts.utils.language_specifics_base import BaseLanguage, get_language_from_file_name +from continuous_delivery_scripts.utils.git_helpers import ( + LocalProjectRepository, + GitWrapper, +) +from continuous_delivery_scripts.utils.language_specifics_base import ( + BaseLanguage, + get_language_from_file_name, +) + +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject logger = logging.getLogger(__name__) @@ -160,7 +171,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """Gets current SPDX description.""" # TODO return None @@ -195,4 +206,8 @@ def _call_goreleaser_release(self, version: str) -> None: changelogPath = configuration.get_value(ConfigurationVariable.CHANGELOG_FILE_PATH) env[ENVVAR_GORELEASER_CUSTOMISED_TAG] = tag env[ENVVAR_GORELEASER_GIT_TOKEN] = configuration.get_value(ConfigurationVariable.GIT_TOKEN) - check_call(_generate_goreleaser_release_command_list(changelogPath), cwd=ROOT_DIR, env=env) + check_call( + _generate_goreleaser_release_command_list(changelogPath), + cwd=ROOT_DIR, + env=env, + ) diff --git a/continuous_delivery_scripts/plugins/noop.py b/continuous_delivery_scripts/plugins/noop.py index cd9cad7..e6666c2 100644 --- a/continuous_delivery_scripts/plugins/noop.py +++ b/continuous_delivery_scripts/plugins/noop.py @@ -6,12 +6,17 @@ import logging from pathlib import Path -from typing import Optional, Dict +from typing import TYPE_CHECKING, Optional, Dict -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject from continuous_delivery_scripts.utils.definitions import CommitType from continuous_delivery_scripts.utils.git_helpers import GitWrapper -from continuous_delivery_scripts.utils.language_specifics_base import BaseLanguage, get_language_from_file_name +from continuous_delivery_scripts.utils.language_specifics_base import ( + BaseLanguage, + get_language_from_file_name, +) + +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject logger = logging.getLogger(__name__) @@ -48,7 +53,7 @@ def can_get_project_metadata(self) -> bool: """States whether project metadata can be retrieved.""" return False - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """No Op.""" return None diff --git a/continuous_delivery_scripts/plugins/python.py b/continuous_delivery_scripts/plugins/python.py index 7769263..afe8f02 100644 --- a/continuous_delivery_scripts/plugins/python.py +++ b/continuous_delivery_scripts/plugins/python.py @@ -9,20 +9,28 @@ import sys from pathlib import Path from subprocess import check_call -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject -from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable +from continuous_delivery_scripts.utils.configuration import ( + configuration, + ConfigurationVariable, +) from continuous_delivery_scripts.utils.definitions import CommitType from continuous_delivery_scripts.utils.filesystem_helpers import TemporaryDirectory from continuous_delivery_scripts.utils.filesystem_helpers import cd -from continuous_delivery_scripts.utils.language_specifics_base import BaseLanguage, get_language_from_file_name +from continuous_delivery_scripts.utils.language_specifics_base import ( + BaseLanguage, + get_language_from_file_name, +) from continuous_delivery_scripts.utils.logging import log_exception from continuous_delivery_scripts.utils.python.package_helpers import ( CurrentPythonProjectMetadataFetcher, generate_package_info, ) +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject + ENVVAR_TWINE_USERNAME = "TWINE_USERNAME" ENVVAR_TWINE_PASSWORD = "TWINE_PASSWORD" OUTPUT_DIRECTORY = "release-dist" @@ -110,15 +118,25 @@ def _generate_pdoc_in_correct_structure(module_to_document: str, output_director Pdoc nests its docs output in a folder with the module's name. This process removes this unwanted folder. """ - with TemporaryDirectory() as temp_dir: - _call_pdoc(temp_dir, module_to_document) - docs_contents_dir = temp_dir.joinpath(module_to_document) - if docs_contents_dir.exists() and docs_contents_dir.is_dir(): - for element in docs_contents_dir.iterdir(): - shutil.move(str(element), str(output_directory)) + temp_directory = TemporaryDirectory() + if hasattr(temp_directory, "__enter__") and hasattr(temp_directory, "__exit__"): + with temp_directory as temp_dir: + _call_pdoc(temp_dir, module_to_document) + docs_contents_dir = temp_dir.joinpath(module_to_document) + if docs_contents_dir.exists() and docs_contents_dir.is_dir(): + for element in docs_contents_dir.iterdir(): + shutil.move(str(element), str(output_directory)) + return + + temp_dir = Path(str(temp_directory)) + _call_pdoc(temp_dir, module_to_document) + docs_contents_dir = temp_dir.joinpath(module_to_document) + if docs_contents_dir.exists() and docs_contents_dir.is_dir(): + for element in docs_contents_dir.iterdir(): + shutil.move(str(element), str(output_directory)) -def _get_current_spdx_project() -> SpdxProject: +def _get_current_spdx_project() -> "SpdxProject": """Gets information about the current project/package.""" logger.info("Generating package information.") try: @@ -126,6 +144,8 @@ def _get_current_spdx_project() -> SpdxProject: generate_package_info() except Exception as e: log_exception(logger, e) + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject + return SpdxProject(CurrentPythonProjectMetadataFetcher()) @@ -174,6 +194,6 @@ def should_include_spdx_in_package(self) -> bool: # FIXME Comment out SPDX package as no longer working return False - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """Gets the current SPDX description.""" return _get_current_spdx_project() diff --git a/continuous_delivery_scripts/report_third_party_ip.py b/continuous_delivery_scripts/report_third_party_ip.py index 9805419..b3a5f2e 100644 --- a/continuous_delivery_scripts/report_third_party_ip.py +++ b/continuous_delivery_scripts/report_third_party_ip.py @@ -16,16 +16,18 @@ import logging import sys from pathlib import Path -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional from continuous_delivery_scripts.language_specifics import get_language_specifics -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject from continuous_delivery_scripts.utils.logging import set_log_level, log_exception +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject + logger = logging.getLogger(__name__) -def generate_spdx_project_reports(project: SpdxProject, output_directory: Path) -> SpdxProject: +def generate_spdx_project_reports(project: "SpdxProject", output_directory: Path) -> "SpdxProject": """Generates all the SPDX reports for a given project.""" logger.info("Generating SPDX report.") project.generate_tag_value_files(output_directory) @@ -34,7 +36,7 @@ def generate_spdx_project_reports(project: SpdxProject, output_directory: Path) return project -def generate_spdx_reports(output_directory: Path) -> Optional[SpdxProject]: +def generate_spdx_reports(output_directory: Path) -> Optional["SpdxProject"]: """Generates all the SPDX reports for the current project.""" project = get_language_specifics().get_current_spdx_project() if not project: @@ -51,10 +53,20 @@ def convert_to_path(arg: Any) -> Path: return Path(arg) parser.add_argument( - "-o", "--output-dir", help="Output directory where the files are generated", required=True, type=convert_to_path + "-o", + "--output-dir", + help="Output directory where the files are generated", + required=True, + type=convert_to_path, ) - parser.add_argument("-v", "--verbose", action="count", default=0, help="Verbosity, by default errors are reported.") + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity, by default errors are reported.", + ) args = parser.parse_args() set_log_level(args.verbose) diff --git a/continuous_delivery_scripts/spdx_report/spdx_dependency.py b/continuous_delivery_scripts/spdx_report/spdx_dependency.py index 9e901a9..9e5a0c4 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_dependency.py +++ b/continuous_delivery_scripts/spdx_report/spdx_dependency.py @@ -4,8 +4,10 @@ # """Definition of dependency SPDX Document.""" -from spdx.checksum import Algorithm -from spdx.document import ExternalDocumentRef +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from spdx.document import ExternalDocumentRef class DependencySpdxDocumentRef: @@ -24,7 +26,7 @@ def __init__(self, name: str, namespace: str, checksum: str) -> None: self._document_namespace = namespace self._document_checksum = checksum - def generate_external_reference(self) -> ExternalDocumentRef: + def generate_external_reference(self) -> "ExternalDocumentRef": """Generates the external SPDX reference. e.g. @@ -34,6 +36,9 @@ def generate_external_reference(self) -> ExternalDocumentRef: Returns: corresponding reference """ + from spdx.checksum import Algorithm + from spdx.document import ExternalDocumentRef + return ExternalDocumentRef( external_document_id=self._document_name, spdx_document_uri=self._document_namespace, diff --git a/continuous_delivery_scripts/spdx_report/spdx_document.py b/continuous_delivery_scripts/spdx_report/spdx_document.py index 54281c1..1a5aea3 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_document.py +++ b/continuous_delivery_scripts/spdx_report/spdx_document.py @@ -5,19 +5,29 @@ """Definition of an SPDX Document.""" from pathlib import Path -from spdx.creationinfo import Person, Organization, Tool -from spdx.document import Document, License -from spdx.review import Review -from spdx.version import Version -from typing import List, Optional - -from continuous_delivery_scripts.spdx_report.spdx_dependency import DependencySpdxDocumentRef -from continuous_delivery_scripts.spdx_report.spdx_helpers import determine_spdx_value, get_project_namespace -from continuous_delivery_scripts.spdx_report.spdx_package import SpdxPackage, PackageInfo -from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable +from typing import TYPE_CHECKING, List, Optional + +from continuous_delivery_scripts.spdx_report.spdx_dependency import ( + DependencySpdxDocumentRef, +) +from continuous_delivery_scripts.spdx_report.spdx_helpers import ( + determine_spdx_value, + get_project_namespace, +) +from continuous_delivery_scripts.spdx_report.spdx_package import ( + SpdxPackage, + PackageInfo, +) +from continuous_delivery_scripts.utils.configuration import ( + configuration, + ConfigurationVariable, +) from continuous_delivery_scripts.utils.hash_helpers import generate_uuid_based_on_str from continuous_delivery_scripts.utils.python.package_helpers import PackageMetadata +if TYPE_CHECKING: + from spdx.document import Document + TOOL_NAME = "mbed-spdx-generator" @@ -75,7 +85,7 @@ def _generate_namespace(self) -> str: url_base = "http://spdx.org/spdxdocs" uuid = generate_uuid_based_on_str(self.document_name) return f"{url_base}/{self.document_name}-{uuid}" - return get_project_namespace(self._project_config, self.document_name) + return str(get_project_namespace(self._project_config, self.document_name)) @property def name(self) -> str: @@ -84,7 +94,7 @@ def name(self) -> str: Returns: corresponding string """ - return f"{self._package_metadata.name}" + return str(self._package_metadata.name) @property def version(self) -> str: @@ -93,7 +103,7 @@ def version(self) -> str: Returns: package version """ - return self._package_metadata.version + return str(self._package_metadata.version) @property def licence(self) -> str: @@ -102,7 +112,7 @@ def licence(self) -> str: Returns: project's licence """ - return self._package_metadata.licence + return str(self._package_metadata.licence) @property def author(self) -> str: @@ -111,7 +121,7 @@ def author(self) -> str: Returns: document's author """ - return self._package_metadata.author + return str(self._package_metadata.author) @property def author_email(self) -> str: @@ -120,7 +130,7 @@ def author_email(self) -> str: Returns: document author's email """ - return self._package_metadata.author_email + return str(self._package_metadata.author_email) @property def organisation(self) -> str: @@ -199,7 +209,7 @@ def generate_spdx_package(self) -> SpdxPackage: ) return self._spdx_package - def generate_spdx_document(self) -> Document: + def generate_spdx_document(self) -> "Document": """Generates the SPDX document. Example of SPDX document section. @@ -219,6 +229,11 @@ def generate_spdx_document(self) -> Document: Returns: the corresponding document """ + from spdx.creationinfo import Person, Organization, Tool + from spdx.document import Document, License + from spdx.review import Review + from spdx.version import Version + doc = Document() doc.version = Version(1, 2) doc.name = determine_spdx_value(self.document_name) @@ -234,7 +249,12 @@ def generate_spdx_document(self) -> Document: doc.creation_info.add_creator(Tool(self.tool_name)) doc.creation_info.set_created_now() if not self._is_dependency: - review = Review(Person(determine_spdx_value(self.reviewer), determine_spdx_value(self.reviewer_email))) + review = Review( + Person( + determine_spdx_value(self.reviewer), + determine_spdx_value(self.reviewer_email), + ) + ) review.set_review_date_now() doc.add_review(review) diff --git a/continuous_delivery_scripts/spdx_report/spdx_file.py b/continuous_delivery_scripts/spdx_report/spdx_file.py index 4437985..0b39a29 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_file.py +++ b/continuous_delivery_scripts/spdx_report/spdx_file.py @@ -5,10 +5,7 @@ """Definition of an SPDX File.""" from pathlib import Path -from spdx.checksum import Algorithm -from spdx.document import License -from spdx.file import File, FileType -from typing import Optional +from typing import TYPE_CHECKING, Optional from continuous_delivery_scripts.spdx_report.spdx_helpers import ( determine_spdx_value, @@ -16,8 +13,16 @@ determine_file_copyright_text, ) from continuous_delivery_scripts.utils.definitions import UNKNOWN -from continuous_delivery_scripts.utils.hash_helpers import generate_uuid_based_on_str, determine_sha1_hash_of_file -from continuous_delivery_scripts.utils.third_party_licences import cleanse_licence_expression +from continuous_delivery_scripts.utils.hash_helpers import ( + generate_uuid_based_on_str, + determine_sha1_hash_of_file, +) +from continuous_delivery_scripts.utils.third_party_licences import ( + cleanse_licence_expression, +) + +if TYPE_CHECKING: + from spdx.file import File class SpdxFile: @@ -49,7 +54,7 @@ def unix_relative_path(self) -> str: the file path """ if str(self.path) == UNKNOWN: - return UNKNOWN + return str(UNKNOWN) unix_path = str(self.path.relative_to(self._project_root)).replace("\\", "/") return f"./{unix_path}" @@ -70,7 +75,7 @@ def id(self) -> str: a UUID """ # Generates a unique Id based on the name of the file - return generate_uuid_based_on_str(self.unix_relative_path) + return str(generate_uuid_based_on_str(self.unix_relative_path)) @property def sha1_check_sum(self) -> str: @@ -79,7 +84,7 @@ def sha1_check_sum(self) -> str: Returns: corresponding hash """ - return determine_sha1_hash_of_file(self._path) + return str(determine_sha1_hash_of_file(self._path)) @property def licence(self) -> str: @@ -89,7 +94,7 @@ def licence(self) -> str: file's licence """ file_licence = determine_file_licence(self.path) - return cleanse_licence_expression(file_licence) if file_licence else self._package_licence + return str(cleanse_licence_expression(file_licence)) if file_licence else str(self._package_licence) @property def copyright(self) -> Optional[str]: @@ -98,9 +103,10 @@ def copyright(self) -> Optional[str]: Returns: file's copyright text """ - return determine_file_copyright_text(self.path) + copyright_text = determine_file_copyright_text(self.path) + return str(copyright_text) if copyright_text is not None else None - def generate_spdx_file(self) -> File: + def generate_spdx_file(self) -> "File": """Generates the SPDX file. SPDX File example: @@ -113,6 +119,10 @@ def generate_spdx_file(self) -> File: Returns: the corresponding file """ + from spdx.checksum import Algorithm + from spdx.document import License + from spdx.file import File, FileType + source_file = File(determine_spdx_value(self.unix_relative_path)) source_file.type = FileType.SOURCE source_file.comment = determine_spdx_value(None) diff --git a/continuous_delivery_scripts/spdx_report/spdx_helpers.py b/continuous_delivery_scripts/spdx_report/spdx_helpers.py index adbbfca..81bd659 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_helpers.py +++ b/continuous_delivery_scripts/spdx_report/spdx_helpers.py @@ -19,20 +19,49 @@ import logging import toml from pathlib import Path -from spdx.utils import SPDXNone, UnKnown from typing import Union, Optional, Iterator, Any, Tuple -from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration +from continuous_delivery_scripts.utils.configuration import ( + ConfigurationVariable, + configuration, +) from continuous_delivery_scripts.utils.definitions import UNKNOWN from continuous_delivery_scripts.utils.filesystem_helpers import ( scan_file_for_pattern, should_exclude_path, list_all_files, ) -from continuous_delivery_scripts.utils.third_party_licences import simplify_licence_expression +from continuous_delivery_scripts.utils.third_party_licences import ( + simplify_licence_expression, +) logger = logging.getLogger(__name__) + +class UnKnown(object): + """Represent SPDX UNKNOWN value without importing spdx-tools.""" + + def to_value(self) -> str: + """Return the SPDX serialised value.""" + return "UNKNOWN" + + def __str__(self) -> str: + """Return the SPDX serialised value as text.""" + return self.to_value() + + +class SPDXNone(object): + """Represent SPDX NONE value without importing spdx-tools.""" + + def to_value(self) -> str: + """Return the SPDX serialised value.""" + return "NONE" + + def __str__(self) -> str: + """Return the SPDX serialised value as text.""" + return self.to_value() + + # Copyright similar to the regex defined in flake8-copyright COPYRIGHT_PATTERN = r"Copyright.*$" COPYRIGHT_REGEX_PATTERN = re.compile(COPYRIGHT_PATTERN, flags=re.MULTILINE | re.IGNORECASE) @@ -63,7 +92,7 @@ def determine_file_licence(path: Path) -> Optional[str]: if not match: return None licence = match.group(1).strip() - return simplify_licence_expression(licence) + return str(simplify_licence_expression(licence)) except Exception as e: logger.error(f"Could not determine the licence of file [{path}] from identifier '{licence}'. Reason: {e}.") return None @@ -98,8 +127,8 @@ def get_project_namespace(project_config_path: Path, document_name: str) -> str: with open(str(project_config_path), "r", encoding="utf8") as f: config = toml.load(f).get(THIRD_PARTY_CONFIG_NAMESPACE, dict()) protocol = "http://" - path_part = f'{config.get("CreatorWebsite")}/{config.get("PathToSpdx")}' - name_part = f'{document_name}-{config.get("UUID")}' + path_part = f"{config.get('CreatorWebsite')}/{config.get('PathToSpdx')}" + name_part = f"{document_name}-{config.get('UUID')}" return f"{protocol}{path_part}/{name_part}" @@ -109,7 +138,8 @@ def list_project_files_for_licensing(project_root: Path) -> Iterator[Path]: def ignore_path(p: Path) -> bool: return True if p.name.startswith(".") else should_exclude_path(p, PATHS_TO_EXCLUDE) - return list_all_files(project_root, ignore_path) + for path in list_all_files(project_root, ignore_path): + yield path def _convert_list_into_dict(checked_packages: Any) -> dict: diff --git a/continuous_delivery_scripts/spdx_report/spdx_package.py b/continuous_delivery_scripts/spdx_report/spdx_package.py index c372b32..f6caeec 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_package.py +++ b/continuous_delivery_scripts/spdx_report/spdx_package.py @@ -6,15 +6,13 @@ from dataclasses import dataclass from pathlib import Path -from spdx.checksum import Algorithm -from spdx.creationinfo import Person -from spdx.document import License -from spdx.package import Package -from spdx.utils import NoAssert -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from continuous_delivery_scripts.spdx_report.spdx_file import SpdxFile -from continuous_delivery_scripts.spdx_report.spdx_helpers import determine_spdx_value, list_project_files_for_licensing +from continuous_delivery_scripts.spdx_report.spdx_helpers import ( + determine_spdx_value, + list_project_files_for_licensing, +) from continuous_delivery_scripts.utils.definitions import UNKNOWN from continuous_delivery_scripts.utils.python.package_helpers import PackageMetadata from continuous_delivery_scripts.utils.third_party_licences import ( @@ -24,6 +22,9 @@ determine_licence_compound, ) +if TYPE_CHECKING: + from spdx.package import Package + @dataclass(frozen=True, order=True) class PackageInfo: @@ -42,7 +43,7 @@ class PackageInfo: uuid: str -def _set_package_copyright(file: SpdxFile, package: Package) -> None: +def _set_package_copyright(file: SpdxFile, package: "Package") -> None: """Sets the copyright field of a package based on file copyright.""" if file.copyright: package.cr_text = determine_spdx_value(file.copyright) @@ -98,7 +99,7 @@ def name(self) -> str: Returns: corresponding string """ - return self._package_info.metadata.name + return str(self._package_info.metadata.name) @property def version(self) -> str: @@ -107,7 +108,7 @@ def version(self) -> str: Returns: package version """ - return self._package_info.metadata.version + return str(self._package_info.metadata.version) @property def main_licence(self) -> str: @@ -126,7 +127,7 @@ def main_licence(self) -> str: @property def is_main_licence_accepted(self) -> bool: """States whether the main licence of the package is part of the accepted licence list.""" - return is_licence_accepted(self.main_licence) + return bool(is_licence_accepted(self.main_licence)) @property def licence(self) -> str: @@ -147,7 +148,7 @@ def licence(self) -> str: @property def is_licence_accepted(self) -> bool: """States whether the actual package's licence of the package is part of the accepted licence list.""" - return is_licence_accepted(self.licence) + return bool(is_licence_accepted(self.licence)) @property def author(self) -> str: @@ -156,7 +157,7 @@ def author(self) -> str: Returns: document's author """ - return self._package_info.metadata.author + return str(self._package_info.metadata.author) @property def author_email(self) -> str: @@ -165,7 +166,7 @@ def author_email(self) -> str: Returns: document author's email """ - return self._package_info.metadata.author_email + return str(self._package_info.metadata.author_email) @property def url(self) -> str: @@ -174,7 +175,7 @@ def url(self) -> str: Returns: the package homepage """ - return self._package_info.metadata.url + return str(self._package_info.metadata.url) @property def description(self) -> str: @@ -183,7 +184,7 @@ def description(self) -> str: Returns: some description """ - return self._package_info.metadata.description + return str(self._package_info.metadata.description) def get_spdx_files(self) -> Optional[List[SpdxFile]]: """Gets package's files SPDX description. @@ -195,7 +196,7 @@ def get_spdx_files(self) -> Optional[List[SpdxFile]]: return None return [SpdxFile(p, self._package_info.root_dir, self.main_licence) for p in self.files] - def generate_spdx_package(self) -> Package: + def generate_spdx_package(self) -> "Package": """Generates the SPDX package. Example of a SPDX package: @@ -220,6 +221,12 @@ def generate_spdx_package(self) -> Package: Returns: the corresponding package """ + from spdx.checksum import Algorithm + from spdx.creationinfo import Person + from spdx.document import License + from spdx.package import Package + from spdx.utils import NoAssert + package = Package( name=determine_spdx_value(self.name), spdx_id=f"SPDXRef-{self.id}", @@ -227,7 +234,10 @@ def generate_spdx_package(self) -> Package: version=determine_spdx_value(self.version), file_name=determine_spdx_value(self.name), supplier=None, - originator=Person(determine_spdx_value(self.author), determine_spdx_value(self.author_email)), + originator=Person( + determine_spdx_value(self.author), + determine_spdx_value(self.author_email), + ), ) package.check_sum = Algorithm("SHA1", str(NoAssert())) package.cr_text = NoAssert() diff --git a/continuous_delivery_scripts/spdx_report/spdx_project.py b/continuous_delivery_scripts/spdx_report/spdx_project.py index d455520..f5175c3 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_project.py +++ b/continuous_delivery_scripts/spdx_report/spdx_project.py @@ -6,14 +6,19 @@ from pathlib import Path import os -from spdx.writers.tagvalue import write_document from typing import Optional, List, cast, Tuple, Dict -from continuous_delivery_scripts.spdx_report.spdx_dependency import DependencySpdxDocumentRef +from continuous_delivery_scripts.spdx_report.spdx_dependency import ( + DependencySpdxDocumentRef, +) from continuous_delivery_scripts.spdx_report.spdx_document import SpdxDocument from continuous_delivery_scripts.utils.hash_helpers import determine_sha1_hash_of_file -from continuous_delivery_scripts.utils.python.package_helpers import ProjectMetadataFetcher -from continuous_delivery_scripts.spdx_report.spdx_helpers import is_package_licence_manually_checked +from continuous_delivery_scripts.utils.python.package_helpers import ( + ProjectMetadataFetcher, +) +from continuous_delivery_scripts.spdx_report.spdx_helpers import ( + is_package_licence_manually_checked, +) from continuous_delivery_scripts.spdx_report.spdx_summary import SummaryGenerator @@ -72,9 +77,11 @@ def generate_tag_value_file(dir: Path, spdx_doc: SpdxDocument, filename: str = " raise NotADirectoryError(str(dir)) path = dir.joinpath(filename) + from spdx.writers.tagvalue import write_document + with open(str(path), mode="w", encoding="utf-8") as out: write_document(spdx_doc.generate_spdx_document(), out) - return determine_sha1_hash_of_file(path) + return str(determine_sha1_hash_of_file(path)) def generate_licensing_summary(self, dir: Path) -> None: """Generates licensing summary into the specified directory. @@ -83,7 +90,8 @@ def generate_licensing_summary(self, dir: Path) -> None: dir: output directory """ SummaryGenerator( - self.main_document.generate_spdx_package(), [d.generate_spdx_package() for d in self.dependency_documents] + self.main_document.generate_spdx_package(), + [d.generate_spdx_package() for d in self.dependency_documents], ).generate_summary(dir) def generate_tag_value_files(self, dir: Path) -> None: @@ -107,7 +115,9 @@ def generate_tag_value_files(self, dir: Path) -> None: checksum = SpdxProject.generate_tag_value_file(dir, spdx_dependency, file_name) externalRefs.append( DependencySpdxDocumentRef( - name=spdx_dependency.document_name, namespace=spdx_dependency.document_namespace, checksum=checksum + name=spdx_dependency.document_name, + namespace=spdx_dependency.document_namespace, + checksum=checksum, ) ) self.main_document.external_refs = externalRefs @@ -147,7 +157,9 @@ def check_licence_compliance(self) -> None: self._report_issues(issues) -def _check_package_licence(package_document: SpdxDocument) -> Tuple[bool, bool, str, str, str]: +def _check_package_licence( + package_document: SpdxDocument, +) -> Tuple[bool, bool, str, str, str]: package = package_document.generate_spdx_package() return ( package.is_main_licence_accepted, diff --git a/continuous_delivery_scripts/spdx_report/spdx_summary.py b/continuous_delivery_scripts/spdx_report/spdx_summary.py index 7b4ab8f..810a5b7 100644 --- a/continuous_delivery_scripts/spdx_report/spdx_summary.py +++ b/continuous_delivery_scripts/spdx_report/spdx_summary.py @@ -10,34 +10,47 @@ from pathlib import Path from typing import List, Tuple, Optional, Dict, Any -from continuous_delivery_scripts.spdx_report.spdx_helpers import get_package_manual_check +from continuous_delivery_scripts.spdx_report.spdx_helpers import ( + get_package_manual_check, +) from continuous_delivery_scripts.spdx_report.spdx_package import SpdxPackage JINJA_TEMPLATE_SUMMARY_HTML = "third_party_IP_report.html.jinja2" JINJA_TEMPLATE_SUMMARY_CSV = "third_party_IP_report.csv.jinja2" JINJA_TEMPLATE_SUMMARY_TEXT = "third_party_IP_report.txt.jinja2" -JINJA_TEMPLATES = [JINJA_TEMPLATE_SUMMARY_HTML, JINJA_TEMPLATE_SUMMARY_CSV, JINJA_TEMPLATE_SUMMARY_TEXT] +JINJA_TEMPLATES = [ + JINJA_TEMPLATE_SUMMARY_HTML, + JINJA_TEMPLATE_SUMMARY_CSV, + JINJA_TEMPLATE_SUMMARY_TEXT, +] logger = logging.getLogger(__name__) -try: - jinja2_env = jinja2.Environment( - loader=jinja2.PackageLoader("continuous_delivery_scripts.spdx_report.spdx_summary", "templates"), + + +def _get_jinja2_env() -> jinja2.Environment: + return jinja2.Environment( + loader=jinja2.FileSystemLoader(str(Path(__file__).resolve().parent.joinpath("templates"))), autoescape=jinja2.select_autoescape(["html", "xml"]), ) -except ModuleNotFoundError as e: - logger.error(e) def generate_file_based_on_template( - output_dir: Path, template_name: str, template_args: dict, suffix: Optional[str] = None + output_dir: Path, + template_name: str, + template_args: dict, + suffix: Optional[str] = None, ) -> None: """Write file based on template and arguments.""" logger.info("Loading template '%s'.", template_name) - template = jinja2_env.get_template(template_name) + template = _get_jinja2_env().get_template(template_name) filename = Path(template_name.rsplit(".", 1)[0]) if suffix: filename = Path( "{0}_{2}{1}".format( - *(str(filename.name), str(filename.suffix), str(suffix.replace(".", "_").replace("-", "_"))) + *( + str(filename.name), + str(filename.suffix), + str(suffix.replace(".", "_").replace("-", "_")), + ) ) ) output_filename = output_dir.joinpath(filename) @@ -88,7 +101,11 @@ def _generate_packages_description(self) -> Tuple[bool, dict]: if not is_compliant: global_compliance = False description_list[p.name] = self._generate_description_for_one_package( - is_compliant, is_licence_compliant, package_manually_checked, manual_check_details, p + is_compliant, + is_licence_compliant, + package_manually_checked, + manual_check_details, + p, ) return global_compliance, description_list diff --git a/continuous_delivery_scripts/tag_and_release.py b/continuous_delivery_scripts/tag_and_release.py index d4fe719..49cc1c0 100644 --- a/continuous_delivery_scripts/tag_and_release.py +++ b/continuous_delivery_scripts/tag_and_release.py @@ -9,19 +9,31 @@ import logging import sys from pathlib import Path -from typing import Optional, Tuple, Dict +from typing import TYPE_CHECKING, Optional, Tuple, Dict from continuous_delivery_scripts.generate_docs import generate_documentation from continuous_delivery_scripts.generate_news import version_project from continuous_delivery_scripts.language_specifics import get_language_specifics from continuous_delivery_scripts.license_files import insert_licence_header -from continuous_delivery_scripts.report_third_party_ip import generate_spdx_project_reports, SpdxProject -from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable +from continuous_delivery_scripts.report_third_party_ip import ( + generate_spdx_project_reports, +) +from continuous_delivery_scripts.utils.configuration import ( + configuration, + ConfigurationVariable, +) from continuous_delivery_scripts.utils.definitions import CommitType -from continuous_delivery_scripts.utils.git_helpers import ProjectTempClone, LocalProjectRepository, GitWrapper +from continuous_delivery_scripts.utils.git_helpers import ( + ProjectTempClone, + LocalProjectRepository, + GitWrapper, +) from continuous_delivery_scripts.utils.logging import log_exception, set_log_level from continuous_delivery_scripts.utils.versioning import determine_version_shortcuts +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject + SPDX_REPORTS_DIRECTORY = "licensing" logger = logging.getLogger(__name__) @@ -73,7 +85,7 @@ def _update_documentation() -> None: generate_documentation(docs_dir, module_to_document) -def _update_licensing_summary() -> Optional[SpdxProject]: +def _update_licensing_summary() -> Optional["SpdxProject"]: if not get_language_specifics().can_get_project_metadata(): return None @@ -124,7 +136,7 @@ def _clean_repository() -> None: git.clean() -def _generate_spdx_reports(project: SpdxProject) -> None: +def _generate_spdx_reports(project: "SpdxProject") -> None: report_directory = Path(configuration.get_value(ConfigurationVariable.PROJECT_ROOT)).joinpath( SPDX_REPORTS_DIRECTORY ) @@ -159,10 +171,21 @@ def main() -> None: """ parser = argparse.ArgumentParser(description="Releases the project.") parser.add_argument( - "-t", "--release-type", help="type of release to perform", required=True, type=str, choices=CommitType.choices() + "-t", + "--release-type", + help="type of release to perform", + required=True, + type=str, + choices=CommitType.choices(), ) parser.add_argument("-b", "--current-branch", help="Name of the current branch", nargs="?") - parser.add_argument("-v", "--verbose", action="count", default=0, help="Verbosity, by default errors are reported.") + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Verbosity, by default errors are reported.", + ) args = parser.parse_args() set_log_level(args.verbose) try: diff --git a/continuous_delivery_scripts/utils/language_specifics_base.py b/continuous_delivery_scripts/utils/language_specifics_base.py index e435eea..0cd7efb 100644 --- a/continuous_delivery_scripts/utils/language_specifics_base.py +++ b/continuous_delivery_scripts/utils/language_specifics_base.py @@ -7,13 +7,18 @@ import logging from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Dict +from typing import TYPE_CHECKING, Optional, Dict -from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject -from continuous_delivery_scripts.utils.configuration import configuration, ConfigurationVariable +from continuous_delivery_scripts.utils.configuration import ( + configuration, + ConfigurationVariable, +) from continuous_delivery_scripts.utils.definitions import CommitType from continuous_delivery_scripts.utils.git_helpers import GitWrapper +if TYPE_CHECKING: + from continuous_delivery_scripts.spdx_report.spdx_project import SpdxProject + logger = logging.getLogger(__name__) @@ -109,7 +114,7 @@ def release_package_to_repository(self, mode: CommitType, version: str) -> None: pass @abstractmethod - def get_current_spdx_project(self) -> Optional[SpdxProject]: + def get_current_spdx_project(self) -> Optional["SpdxProject"]: """Gets current project SPDX.""" pass diff --git a/continuous_delivery_scripts/utils/package_helpers.py b/continuous_delivery_scripts/utils/package_helpers.py index 77d1996..d9f2006 100644 --- a/continuous_delivery_scripts/utils/package_helpers.py +++ b/continuous_delivery_scripts/utils/package_helpers.py @@ -6,9 +6,12 @@ import logging from abc import ABC, abstractmethod -from typing import List, Optional +from typing import Any, Dict, List, Optional -from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration +from continuous_delivery_scripts.utils.configuration import ( + ConfigurationVariable, + configuration, +) from continuous_delivery_scripts.utils.definitions import UNKNOWN logger = logging.getLogger(__name__) @@ -24,48 +27,48 @@ class PackageMetadata: def __init__(self, data: dict) -> None: """Constructor.""" - self._data = data + self._data: Dict[str, Any] = data @property def name(self) -> str: """Gets package's name.""" - return self._data.get("Name", UNKNOWN) + return str(self._data.get("Name", UNKNOWN)) @property def version(self) -> str: """Gets package's version.""" - return self._data.get("Version", UNKNOWN) + return str(self._data.get("Version", UNKNOWN)) @property def author(self) -> str: """Gets package's author.""" - return self._data.get("Author", UNKNOWN) + return str(self._data.get("Author", UNKNOWN)) @property def author_email(self) -> str: """Gets package's author email.""" - return self._data.get("Author-email", UNKNOWN) + return str(self._data.get("Author-email", UNKNOWN)) @property def licence(self) -> str: """Gets package's licence.""" - return self._data.get("License", UNKNOWN) + return str(self._data.get("License", UNKNOWN)) @property def description(self) -> str: """Gets package's licence.""" - return self._data.get("Summary", UNKNOWN) + return str(self._data.get("Summary", UNKNOWN)) @property def url(self) -> str: """Gets package's URL.""" - home_page = str(self._data.get("Home-page")) + home_page = self._data.get("Home-page") if home_page: - return home_page - url = str(self._data.get("Project-URL")) + return str(home_page) + url = self._data.get("Project-URL") if url: - return url.split(",")[1].strip() - return UNKNOWN + return str(url).split(",")[1].strip() + return str(UNKNOWN) def __str__(self) -> str: """String representation.""" diff --git a/continuous_delivery_scripts/utils/python/package_helpers.py b/continuous_delivery_scripts/utils/python/package_helpers.py index 884de2d..6a909ce 100644 --- a/continuous_delivery_scripts/utils/python/package_helpers.py +++ b/continuous_delivery_scripts/utils/python/package_helpers.py @@ -4,16 +4,25 @@ # """Utilities for retrieving Python's package information.""" +import importlib.metadata as importlib_metadata import logging import re import subprocess import sys -from typing import List, Any, cast +from typing import Iterable, List, Set, Any, cast -import pkg_resources +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name -from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration -from continuous_delivery_scripts.utils.package_helpers import ProjectMetadataFetcher, PackageMetadata, ProjectMetadata +from continuous_delivery_scripts.utils.configuration import ( + ConfigurationVariable, + configuration, +) +from continuous_delivery_scripts.utils.package_helpers import ( + ProjectMetadataFetcher, + PackageMetadata, + ProjectMetadata, +) logger = logging.getLogger(__name__) @@ -55,20 +64,71 @@ def get_package_metadata_lines(package: Any) -> list: the underlying `get_metadata_lines` function used raises an exception if the file does not exist. We hence need to try to catch all exceptions """ - try: - return cast(list, package.get_metadata_lines("METADATA")) - except Exception as e: - logger.warning(e) - try: - return cast(list, package.get_metadata_lines("PKG-INFO")) - except Exception as e: - logger.warning(e) + for filename in ["METADATA", "PKG-INFO"]: + try: + metadata = package.read_text(filename) + if metadata: + return cast(list, metadata.splitlines()) + except Exception as e: + logger.warning(e) return list() +def _get_distribution_name(distribution: importlib_metadata.Distribution) -> str: + distribution_name = getattr(distribution, "name", None) + if distribution_name: + return str(distribution_name) + + metadata = distribution.metadata + if "Name" in metadata: + return str(metadata["Name"]) + + raise importlib_metadata.PackageNotFoundError("Distribution metadata does not define a package name") + + +def _get_distribution(package_name: str) -> importlib_metadata.Distribution: + try: + return importlib_metadata.distribution(package_name) + except importlib_metadata.PackageNotFoundError: + normalised_package_name = package_name.replace("-", "_") + for distribution in importlib_metadata.distributions(): + if canonicalize_name(_get_distribution_name(distribution)) == canonicalize_name(normalised_package_name): + return distribution + raise + + +def _iter_dependency_distributions( + distribution: importlib_metadata.Distribution, seen_packages: Set[str] +) -> Iterable[importlib_metadata.Distribution]: + for requirement_text in distribution.requires or []: + requirement = Requirement(requirement_text) + if requirement.marker and not requirement.marker.evaluate(): + continue + + normalised_name = canonicalize_name(requirement.name) + if normalised_name in seen_packages: + continue + + try: + dependency_distribution = _get_distribution(requirement.name) + except importlib_metadata.PackageNotFoundError as e: + logger.warning(e) + continue + + seen_packages.add(normalised_name) + yield dependency_distribution + yield from _iter_dependency_distributions(dependency_distribution, seen_packages) + + def get_all_packages_metadata_lines(package_name: str) -> List[list]: """Determines the metadata lines for the present package as well as for all its dependencies.""" - return [get_package_metadata_lines(package) for package in pkg_resources.require(package_name)] + distribution = _get_distribution(package_name) + seen_packages = {canonicalize_name(_get_distribution_name(distribution))} + all_distributions = [ + distribution, + *_iter_dependency_distributions(distribution, seen_packages), + ] + return [get_package_metadata_lines(package) for package in all_distributions] def parse_package_metadata_lines(metadata: list) -> PackageMetadata: diff --git a/continuous_delivery_scripts/utils/third_party_licences.py b/continuous_delivery_scripts/utils/third_party_licences.py index 5a6bf91..550e029 100644 --- a/continuous_delivery_scripts/utils/third_party_licences.py +++ b/continuous_delivery_scripts/utils/third_party_licences.py @@ -8,12 +8,18 @@ import json from dataclasses import dataclass +from importlib.util import find_spec from license_expression import Licensing, LicenseExpression, OR -from spdx.config import _licenses +from pathlib import Path from typing import Iterable, cast, Optional, Iterator, List, Pattern, Any -from continuous_delivery_scripts.utils.configuration import ConfigurationVariable, configuration -from continuous_delivery_scripts.utils.string_helpers import determine_similar_string_from_list +from continuous_delivery_scripts.utils.configuration import ( + ConfigurationVariable, + configuration, +) +from continuous_delivery_scripts.utils.string_helpers import ( + determine_similar_string_from_list, +) @dataclass(order=True, frozen=True) @@ -48,6 +54,13 @@ class Licence: LICENCE_NON_ACCEPTED_CHARACTERS = r"[^\w\s\.\:\-()]" +def _get_spdx_licenses_path() -> Path: + spec = find_spec("spdx") + if not spec or not spec.origin: + raise ModuleNotFoundError("No module named 'spdx'") + return Path(spec.origin).resolve().parent.joinpath("licenses.json") + + def _parse_licence_expression(licensing: Licensing, licence_expression: str) -> LicenseExpression: # Removing any unwanted characters so that the expression follows the laws: # > the valid characters are: letters and numbers, underscore, dot, colon or hyphen signs and spaces @@ -115,7 +128,7 @@ def load(self) -> None: return self._licence_store = {UNKNOWN_LICENCE.identifier: UNKNOWN_LICENCE} self._licence_list = [UNKNOWN_LICENCE.identifier] - with open(_licenses, "r", encoding="utf8") as f: + with open(_get_spdx_licenses_path(), "r", encoding="utf8") as f: for licence in iter_licenses(json.load(f)): self._licence_store[licence.identifier] = licence self._licence_list.append(licence.identifier) @@ -174,7 +187,9 @@ def _iter_matching_licences_from_pattern(desc: str) -> Iterable[Licence]: yield from licences -def _retrieve_licences_from_identifier_list(identifiers: Iterable[str]) -> Iterable[Licence]: +def _retrieve_licences_from_identifier_list( + identifiers: Iterable[str], +) -> Iterable[Licence]: for desc in identifiers: if "*" in desc: yield from _iter_matching_licences_from_pattern(desc) @@ -182,7 +197,9 @@ def _retrieve_licences_from_identifier_list(identifiers: Iterable[str]) -> Itera yield from _iter_matching_licences(desc) -def determine_allowed_opensource_licences_from_string(allowed_licences: Any) -> Iterable[Licence]: +def determine_allowed_opensource_licences_from_string( + allowed_licences: Any, +) -> Iterable[Licence]: """Determines all the third party licences allowed as set in the input parameter.""" if isinstance(allowed_licences, str): allowed_licences = allowed_licences.split(", ")