Merge branch 'develop' into 2402-detect-gibberish-copyright

Author: JonoYang

Dockerfile

@@ -37,12 +37,13 @@ WORKDIR /scancode-toolkit
 # Copy sources into docker container
 COPY . /scancode-toolkit
 
-# Initial configuration using ./configure, train-gibberish-model to build the
-# pickle containing the probabilities , scancode-reindex-licenses to build the base
-# license index
+# Initial configuration using ./configure, scancode-reindex-licenses to build
+# the base license index and scancode-reindex-package-patterns to build the
+# package patterns cache
 RUN ./configure \
- && ./venv/bin/scancode-reindex-licenses
- && ./venv/bin/train-gibberish-model
+ && ./venv/bin/scancode-reindex-licenses \
+ && ./venv/bin/scancode-reindex-package-patterns \
+ && ./venv/bin/scancode-train-gibberish-model
 
 # Add scancode to path
 ENV PATH=/scancode-toolkit:$PATH

README.rst

@@ -2,12 +2,15 @@
 ScanCode Toolkit
 ================
 
-ScanCode Toolkit is a set of code scanning tools that detect the origin (copyrights), license and vulnerabilities of code, packages and dependencies in a codebase. ScanCode Toolkit is an `AboutCode project <https://aboutcode.org>`_.
+ScanCode Toolkit is a set of code scanning tools that detect the origin (copyrights), license and vulnerabilities of code,
+packages and dependencies in a codebase. ScanCode Toolkit is an `AboutCode project <https://aboutcode.org>`_.
 
 Why Use ScanCode Toolkit?
 =========================
 
-ScanCode Toolkit is the leading tool in scanning depth and accuracy, used by hundreds of software teams. You can use ScanCode Toolkit as a command line tool or as a library.
+ScanCode Toolkit is the leading tool in scanning depth and accuracy,
+used by hundreds of software teams. You can use ScanCode Toolkit
+as a command line tool or as a library.
 
 Getting Started
 ===============
@@ -84,7 +87,7 @@ Benefits of ScanCode
 Support
 =======
 
-If you have a specific problem, suggestion or bug, please submit a 
+If you have a specific problem, suggestion or bug, please submit a
 `GitHub issue <https://github.com/aboutcode-org/scancode-toolkit/issues>`_.
 
 For quick questions or socializing, join the AboutCode community discussions on `Slack <https://join.slack.com/t/aboutcode-org/shared_invite/zt-3li3bfs78-mmtKG0Qhv~G2dSlNCZW2pA>`_.

azure-pipelines.yml

@@ -145,14 +145,6 @@ jobs:
           test_suites:
               all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: macos13_cpython
-          image_name: macOS-13
-          python_versions: ['3.10', '3.11', '3.12', '3.13']
-          test_suites:
-              all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py --reruns 2
-
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2025_cpython
@@ -220,14 +212,6 @@ jobs:
           test_suites:
               all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
 
-    - template: etc/ci/azure-posix.yml
-      parameters:
-          job_name: macos13_cpython_latest_from_pip
-          image_name: macos-13
-          python_versions: ['3.10', '3.11', '3.12', '3.13']
-          test_suites:
-              all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[testing] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py
-
     - template: etc/ci/azure-win.yml
       parameters:
           job_name: win2019_cpython_latest_from_pip

docs/source/rst-snippets/cli-basic-options.rst

@@ -33,6 +33,12 @@ documenting a program's options. For example:
 --system-package             Scan ``<input>`` for installed system package
                              databases.
 
+--package-in-compiled        Scan compiled executable binaries such as ELF,
+                             WinpE and Mach-O files, looking for structured
+                             package and dependency metadata. Note that looking for
+                             packages in binaries makes package scan slower.
+                             Currently supported compiled binaries: Go, Rust.
+
 --package-only               Faster package scan, scanning ``<input>`` for
                              system and application packages, only for package
                              metadata. This option is skipping

etc/release/scancode-create-pypi-wheel.sh

@@ -19,7 +19,8 @@ set -e
 
 ./configure --dev
 venv/bin/scancode-reindex-licenses
-venv/bin/train-gibberish-model
+venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 
 python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" )
 

etc/release/scancode-create-release-app-linux.sh

@@ -65,7 +65,8 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
-venv/bin/train-gibberish-model
+venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-macos.sh

@@ -63,7 +63,8 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
-venv/bin/train-gibberish-model
+venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-windows.sh

@@ -62,7 +62,8 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
-venv/bin/train-gibberish-model
+venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

requirements.txt

@@ -11,7 +11,7 @@ charset-normalizer==3.4.2
 click==8.3.0;python_version>='3.10'
 click==8.1.7;python_version<'3.10'
 colorama==0.4.6
-commoncode==32.4.0
+commoncode==32.4.2
 construct==2.10.70
 container-inspector==33.0.0
 cryptography==45.0.4
@@ -40,6 +40,7 @@ license-expression==30.4.4
 lxml==5.4.0
 MarkupSafe==3.0.2
 more-itertools==10.7.0
+multiregex==2.0.3
 normality==2.6.1
 packageurl-python==0.17.1
 packaging==25.0
@@ -70,7 +71,7 @@ soupsieve==2.7
 spdx-tools==0.8.2
 text-unidecode==1.3
 tomli==2.3.0
-typecode==30.0.2
+typecode==30.1.0
 typecode-libmagic==5.39.210531
 typing-extensions==4.14.0
 uritools==5.0.0

setup-mini.cfg

@@ -89,6 +89,7 @@ install_requires =
     license_expression >= 30.4.4
     lxml >= 5.4.0
     MarkupSafe >= 2.1.2
+    multiregex >= 2.0.3    
     normality <= 2.6.1
     packageurl_python >= 0.9.0
     packvers >= 21.0.0
@@ -112,7 +113,7 @@ install_requires =
     tomli >= 2; python_version < "3.11"
     urlpy
     xmltodict >= 0.11.0
-    typecode >= 30.0.1
+    typecode >= 30.1.0
 #    typecode[full] >= 30.0.1
 #    extractcode[full] >= 31.0.0
 
@@ -123,7 +124,7 @@ where = src
 
 [options.extras_require]
 full =
-    typecode[full] >= 30.0.0
+    typecode[full] >= 30.1.0
     extractcode[full] >= 31.0.0
 
 dev =
@@ -156,6 +157,7 @@ packages =
 console_scripts =
     scancode = scancode.cli:scancode
     scancode-reindex-licenses = licensedcode.reindex:reindex_licenses
+    scancode-reindex-package-patterns = packagedcode.cache:cache_package_patterns
     scancode-license-data = licensedcode.license_db:dump_scancode_license_data
     regen-package-docs = packagedcode.regen_package_docs:regen_package_docs
     add-required-phrases = licensedcode.required_phrases:add_required_phrases