Merge branch 'develop' into update-licenses

Author: AyanSinhaMahapatra

.github/workflows/docs-ci.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.13]
+        python-version: [3.14]
 
     steps:
       - name: Checkout code

.github/workflows/scancode-release.yml

@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-       pyver: ["3.10", "3.11", "3.12", "3.13"]
+       pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -74,7 +74,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.14"
 
       - name: Install requirements then build main and mini sdist
         run: etc/release/scancode-create-pypi-sdist.sh
@@ -100,7 +100,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-       pyver: ["3.10", "3.11", "3.12", "3.13"]
+       pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -135,7 +135,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-       pyver: ["3.10", "3.11", "3.12", "3.13"]
+       pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -170,7 +170,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-       pyver: ["3.10", "3.11", "3.12", "3.13"]
+       pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -211,7 +211,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
+          python-version: "3.14"
 
       - name: Build source archive with deps
         run: etc/release/scancode-create-release-app-sources.sh
@@ -239,8 +239,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-24.04, ubuntu-24.04, macos-13, macos-14]
-        pyver: ["3.10", "3.11", "3.12", "3.13"]
+        os: [ubuntu-24.04, ubuntu-24.04, macos-14, macos-15]
+        pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -286,7 +286,7 @@ jobs:
       fail-fast: true
       matrix:
        os: [windows-2025, windows-2022]
-       pyver: ["3.10", "3.11", "3.12", "3.13"]
+       pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -331,7 +331,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-24.04, ubuntu-24.04]
-        pyver: ["3.10", "3.11", "3.12", "3.13"]
+        pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -375,8 +375,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [macos-13, macos-14]
-        pyver: ["3.10", "3.11", "3.12", "3.13"]
+        os: [macos-14, macos-15]
+        pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -421,7 +421,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [windows-2025, windows-2022]
-        pyver: ["3.10", "3.11", "3.12", "3.13"]
+        pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     steps:
       - uses: actions/checkout@v4
@@ -493,6 +493,12 @@ jobs:
           name: macos_app_py_3.13
           path: dist
 
+      - name: Download a single artifact macos_app for python 3.14
+        uses: actions/download-artifact@v4
+        with:
+          name: macos_app_py_3.14
+          path: dist
+
       - name: Download a single artifact linux_app for python 3.10
         uses: actions/download-artifact@v4
         with:
@@ -517,6 +523,12 @@ jobs:
           name: linux_app_py_3.13
           path: dist
 
+      - name: Download a single artifact linux_app for python 3.14
+        uses: actions/download-artifact@v4
+        with:
+          name: linux_app_py_3.14
+          path: dist
+
       - name: Download a single artifact windows_app for python 3.10
         uses: actions/download-artifact@v4
         with:
@@ -541,6 +553,12 @@ jobs:
           name: windows_app_py_3.13
           path: dist
 
+      - name: Download a single artifact windows_app for python 3.14
+        uses: actions/download-artifact@v4
+        with:
+          name: windows_app_py_3.14
+          path: dist
+
       - name: Mock GH release
         run: |
           ls -al dist
@@ -561,19 +579,22 @@ jobs:
       - smoke_test_install_and_run_pypi_wheels_on_posix
       - publish_to_gh_release
     runs-on: ubuntu-24.04
+    environment: pypi-publish
+    permissions:
+      id-token: write
     defaults:
       run:
         shell: bash
     strategy:
       fail-fast: false
       matrix:
-        dist_names: ["wheels-3.10", "wheels-3.11", "wheels-3.12", "wheels-3.13", sdists]
+        dist_names: ["wheels-3.10", "wheels-3.11", "wheels-3.12", "wheels-3.13", "wheels-3.14", "sdists"]
 
     steps:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.13
+          python-version: 3.14
 
       - name: Download a single artifact
         uses: actions/download-artifact@v4
@@ -587,5 +608,3 @@ jobs:
 
       - name: Publish distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -116,3 +116,7 @@ selenium
 rpmdb.sqlite-*
 /.ruff_cache/
 .env
+
+# Ignore gibberish detector model, this is trained on configure and build and
+# should not be committed
+gib_model.pki

CHANGELOG.rst

@@ -4,6 +4,42 @@ Changelog
 Next release
 --------------
 
+v3.5.0 - 2026-01-15
+-------------------
+
+- Improve package scan performance by:
+
+  - Skipping binary package detection steps by default,
+    and introducing a new CLI option ``--package-in-compiled``
+    to detect packages in compiled binaries like rust/go binaries
+    Note: previously these were detected in ``--package`` CLI
+    option directly but this is no longer the case, you've to
+    use the new ``--package-in-compiled`` to detect packages
+    from go/rust and other binaries
+
+  - Creating cached regex patterns and multiregex pre-matchers,
+    for a fast package path detection filtering step
+
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4606
+
+- Add gibberish detection to copyright scanning. This is done using a
+  2-character Markov chain. A new CLI command,
+  ``scancode-train-gibberish-model``, has been added to regenerate the model
+  used by the detector.
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4610
+  https://github.com/aboutcode-org/scancode-toolkit/issues/2402
+
+- Use a cython implementation of sequence matcher to improve
+  license detection performance
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4500
+
+- Add python3.14 support in scancode
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4595
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4541
+
+- Always print license references table in html output
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4474
+
 - Enable License References table for HTML Output without requiring
   `--license-references` by implementing a fallback license reference
   collection based on the behavior of v32.0.0.
@@ -13,6 +49,34 @@ Next release
 - Replace unmaintained ``toml`` library with ``tomllib`` / ``tomli``.
   https://github.com/aboutcode-org/scancode-toolkit/issues/4532
 
+- Pin fingerprints and normality to avoid pyicu
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4493
+
+- Fix click compatibility issues and failures
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4572
+
+- Remove deprecated ast module attributes
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4539
+
+- Fix cargo scanning failures
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4581
+
+- Reorg and improve docs user experience
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4629
+
+- Handle is_private strings in npm properly
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4635
+
+- Restructure README docs
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4667
+
+- Add new licenses, license rules and misc license detection
+  improvements
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4562
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4674
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4666
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4622
+
 v32.4.1 - 2025-07-23
 --------------------
 

CONTRIBUTING.rst

@@ -9,7 +9,7 @@ Every little bit helps, and a credit will always be given.
 __ issues_
 
 If you are new to ScanCode and want to find easy tickets to work on,
-check `easy issues <https://github.com/nexB/scancode-toolkit/labels/easy>`_
+check `good first issues <https://github.com/nexB/scancode-toolkit/labels/easy>`_
 
 When contributing to ScanCode (such as code, bugs, documentation, etc.) you
 agree to the Developer `Certificate of Origin <http://developercertificate.org/>`_
@@ -85,7 +85,7 @@ template. Your help and contribution make ScanCode docs better, we love hearing
 
 The ScanCode documentation is hosted at `scancode-toolkit.readthedocs.io <https://scancode-toolkit.readthedocs.io/en/latest/>`_.
 
-If you want to contribute to Scancode Documentation, you'll find `this guide here <https://scancode-toolkit.readthedocs.io/en/latest/contribute/contrib_doc.html>`_ helpful.
+If you want to contribute to Scancode Documentation, you'll find `this guide here https://scancode-toolkit.readthedocs.io/en/latest/getting-started/contribute/contributing-docs.html`_ helpful.
 
 Development
 ===========

Dockerfile

@@ -38,9 +38,13 @@ WORKDIR /scancode-toolkit
 COPY . /scancode-toolkit
 
 # Initial configuration using ./configure, scancode-reindex-licenses to build
-# the base license index
+# the base license index, scancode-reindex-package-patterns to build the package
+# patterns cache, and scancode-train-gibberish-model to train the Markov chain
+# model used for gibberish detection.
 RUN ./configure \
- && ./venv/bin/scancode-reindex-licenses
+ && ./venv/bin/scancode-reindex-licenses \
+ && ./venv/bin/scancode-reindex-package-patterns \
+ && ./venv/bin/scancode-train-gibberish-model
 
 # Add scancode to path
 ENV PATH=/scancode-toolkit:$PATH

INSTALL.rst

@@ -16,7 +16,7 @@ Prerequisites
 -------------
 
 Before installing ScanCode make sure you have  installed these prerequisites.
-The main one is to have Python installed version 3.10, 3.11, 3.12 or 3.13.
+The main one is to have Python installed version 3.10, 3.11, 3.12, 3.13 or 3.14.
 
 - For Linux(Ubuntu):
   ``sudo apt install python3.10-dev bzip2 xz-utils zlib1g libxml2-dev libxslt1-dev``