Merge branch 'main' into license-detection-models

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - name: Checkout code

.github/workflows/generate-sboms.yml

@@ -0,0 +1,39 @@
+name: Generate SBOMS
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "v*.*.*"
+
+env:
+  INPUTS_PATH: scancode-inputs
+
+jobs:
+  generate-sboms:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Ensure INPUTS_PATH directory exists
+        run: mkdir -p "${{ env.INPUTS_PATH }}"
+
+      - name: Build the Docker image from local Dockerfile
+        run: docker build -t local-image .
+
+      - name: Run pip freeze inside the built Docker container
+        run: docker run --rm local-image pip freeze --all --exclude scancodeio > "${{ env.INPUTS_PATH }}/requirements.txt"
+
+      - name: Collect all .ABOUT files in the scancodeio/ directory
+        run: |
+          mkdir -p "${{ env.INPUTS_PATH }}/about-files"
+          find scancodeio/ -type f -name "*.ABOUT" -exec cp {} "${{ env.INPUTS_PATH }}/about-files/" \;
+
+      - name: Resolve the dependencies using ScanCode-action
+        uses: aboutcode-org/scancode-action@main
+        with:
+          pipelines: "resolve_dependencies:DynamicResolver"
+          inputs-path: ${{ env.INPUTS_PATH }}
+          scancodeio-repo-branch: main

.github/workflows/pypi-release-aboutcode-pipeline.yml

@@ -17,13 +17,13 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: 3.13
 
       - name: Install flot
         run: python -m pip install flot --user
 
       - name: Build a binary wheel and a source tarball
-        run:  python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/
+        run: python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/
 
       - name: Publish to PyPI
         if: startsWith(github.ref, 'refs/tags')

.github/workflows/pypi-release.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: 3.13
 
       - name: Install pypa/build
         run: python -m pip install build --user
@@ -38,7 +38,8 @@ jobs:
           path: dist/*
 
       - name: Create a GitHub release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
+          generate_release_notes: true
           draft: false
           files: dist/*

.gitignore

@@ -51,6 +51,7 @@ local
 *.rdb
 *.aof
 .vscode
+.ipynb_checkpoints
 
 # This is only created when packaging for external redistribution
 /thirdparty/

CHANGELOG.rst

@@ -1,7 +1,246 @@
 Changelog
 =========
 
-v34.9.1 (unreleased)
+v35.1.0 (2025-07-02)
+--------------------
+
+- Replace the ``setup.py``/``setup.cfg`` by ``pyproject.toml`` file.
+  https://github.com/aboutcode-org/scancode.io/issues/1608
+
+- Update scancode-toolkit to v32.4.0. See CHANGELOG for updates:
+  https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.4.0
+  Adds a new ``git_sha1`` attribute to the ``CodebaseResource`` model as this
+  is now computed and returned from the ``scancode-toolkit`` ``--info`` plugin.
+  https://github.com/aboutcode-org/scancode.io/pull/1708
+
+- Add a ``--fail-on-vulnerabilities`` option in ``check-compliance`` management command.
+  When this option is enabled, the command will exit with a non-zero status if known
+  vulnerabilities are detected in discovered packages and dependencies.
+  Requires the ``find_vulnerabilities`` pipeline to be executed beforehand.
+  https://github.com/aboutcode-org/scancode.io/pull/1702
+
+- Enable ``--license-references`` scan option in the ``scan_single_package`` pipeline.
+  The ``license_references`` and ``license_rule_references`` attributes will now be
+  available in the scan results, including the details about detected licenses and
+  license rules used during the scan.
+  https://github.com/aboutcode-org/scancode.io/issues/1657
+
+- Add a new step to the ``DeployToDevelop`` pipeline, ``map_python``, to match
+  Cython source files (.pyx) to their compiled binaries.
+  https://github.com/aboutcode-org/scancode.io/pull/1703
+
+v35.0.0 (2025-06-23)
+--------------------
+
+- Add support for Python 3.13.
+  Upgrade the base image in Dockerfile to ``python:3.13-slim``.
+  https://github.com/aboutcode-org/scancode.io/pull/1469/files
+
+- Display matched snippets details in "Resource viewer", including the package,
+  resource, and similarity values.
+  https://github.com/aboutcode-org/scancode.io/issues/1688
+
+- Add filtering by label and pipeline in the ``flush-projects`` management command.
+  Also, a new ``--dry-run`` option is available to test the filters before applying
+  the deletion.
+  https://github.com/aboutcode-org/scancode.io/pull/1690
+
+- Add support for using Package URL (purl) as project input.
+  This implementation is based on ``purl2url.get_download_url``.
+  https://github.com/aboutcode-org/scancode.io/issues/1383
+
+- Raise a ``MatchCodeIOException`` when the response from the MatchCode.io service is
+  not valid in ``send_project_json_to_matchcode``.
+  This generally means an issue on the MatchCode.io server side.
+  https://github.com/aboutcode-org/scancode.io/issues/1665
+
+- Upgrade Bulma CSS and Ace JS libraries to latest versions.
+  Refine the CSS for the Resource viewer.
+  https://github.com/aboutcode-org/scancode.io/pull/1692
+
+- Add "(No value detected)" for Copyright and Holder charts.
+  https://github.com/aboutcode-org/scancode.io/issues/1697
+
+- Add "Package Compliance Alert" chart in the Policies section.
+  https://github.com/aboutcode-org/scancode.io/pull/1699
+
+- Update univers to v31.0.0, catch ``NotImplementedError`` in
+  ``get_unique_unresolved_purls``, and properly log error in project.
+  https://github.com/aboutcode-org/scancode.io/pull/1700
+  https://github.com/aboutcode-org/scancode.io/pull/1701
+
+v34.11.0 (2025-05-02)
+---------------------
+
+- Add a ``UUID`` field on the DiscoveredDependency model.
+  Use the UUID for the DiscoveredDependency spdx_id for better SPDX compatibility.
+  https://github.com/aboutcode-org/scancode.io/issues/1651
+
+- Add MatchCode-specific functions to compute fingerprints from stemmed code
+  files. Update CodebaseResource file content view to display snippet matches,
+  if available, when the codebase has been sent for matching to MatchCode.
+  https://github.com/aboutcode-org/scancode.io/pull/1656
+
+- Add the ability to export filtered QuerySet of a FilterView into the JSON format.
+  https://github.com/aboutcode-org/scancode.io/pull/1572
+
+- Include ``ProjectMessage`` records in the JSON output ``headers`` section.
+  https://github.com/aboutcode-org/scancode.io/issues/1659
+
+v34.10.1 (2025-03-26)
+---------------------
+
+- Convert the ``declared_license`` field value return by ``python-inspector`` in
+  ``resolve_pypi_packages``.
+  Resolving requirements.txt files will now return proper license data.
+  https://github.com/aboutcode-org/scancode.io/issues/1598
+
+- Add support for installing on Apple Silicon (macOS ARM64) in dev mode.
+  https://github.com/aboutcode-org/scancode.io/pull/1646
+
+v34.10.0 (2025-03-21)
+---------------------
+
+- Rename the ``docker``, ``docker_windows``, and ``root_filesystem`` modules to
+  ``analyze_docker``, ``analyze_docker_windows``, and ``analyze_root_filesystem``
+  for consistency.
+
+- Refine and document the Webhook system
+  https://github.com/aboutcode-org/scancode.io/issues/1587
+  * Add UI to add/delete Webhooks from the project settings
+  * Add a new ``add-webhook`` management command
+  * Add a ``add_webhook`` REST API action
+  * Add a new ``SCANCODEIO_GLOBAL_WEBHOOK`` setting
+  * Add a new chapter dedicated to Webhooks management in the documentation
+  * Add support for custom payload dedicated to Slack webhooks
+
+- Upgrade Bulma CSS library to version 1.0.2
+  https://github.com/aboutcode-org/scancode.io/pull/1268
+
+- Disable the creation of the global webhook in the ``batch-create`` command by default.
+  The global webhook can be created by providing the ``--create-global-webhook`` option.
+  A ``--no-global-webhook`` option was also added to the ``create-project`` command to
+  provide the ability to skip the global webhook creation.
+  https://github.com/aboutcode-org/scancode.io/pull/1629
+
+- Add support for "Permission denied" file access in make_codebase_resource.
+  https://github.com/aboutcode-org/scancode.io/issues/1630
+
+- Refine the ``scan_single_package`` pipeline to work on git fetched inputs.
+  https://github.com/aboutcode-org/scancode.io/issues/1376
+
+v34.9.5 (2025-02-19)
+--------------------
+
+- Add support for the XLSX report in REST API.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
+- Add options to the Project reset action.
+  Also, the Project labels are kept during reset.
+  https://github.com/aboutcode-org/scancode.io/issues/1568
+
+- Add aboutcode.pipeline as an install_requires external dependency to prevent conflicts
+  with other aboutcode submodules.
+  https://github.com/aboutcode-org/scancode.io/issues/1423
+
+- Add a ``add-webhook`` management command that allows to add webhook subscription on
+  a project.
+  https://github.com/aboutcode-org/scancode.io/issues/1587
+
+- Add proper progress logging for the ``assemble`` section of the
+  ``scan_for_application_packages``.
+  https://github.com/aboutcode-org/scancode.io/issues/1601
+
+v34.9.4 (2025-01-21)
+--------------------
+
+- Improve Project list page navigation.
+  A top previous/next page navigation was added in the header for consistency with other
+  list views.
+  Any paginated view can now be navigated using the left/right keyboard keys.
+  https://github.com/aboutcode-org/scancode.io/issues/1200
+
+- Add support for importing the ``extra_data`` value from the JSON input with the
+  ``load_inventory`` pipeline.
+  When multiple JSON files are provided as inputs, the ``extra`` is prefixed with
+  the input filename.
+  https://github.com/aboutcode-org/scancode.io/issues/926
+
+- Disable CycloneDX document strict validation, which halts the entire loading process,
+  and let the data loading process handle the data issues.
+  https://github.com/aboutcode-org/scancode.io/issues/1515
+
+- Add a report action on project list to export XLSX containing packages from selected
+  projects.
+  https://github.com/aboutcode-org/scancode.io/issues/1437
+
+- Add a download action on project list to enable bulk download of Project output files.
+  https://github.com/aboutcode-org/scancode.io/issues/1518
+
+- Add labels to Project level search.
+  The labels are now always presented in alphabetical order for consistency.
+  https://github.com/aboutcode-org/scancode.io/issues/1520
+
+- Add a ``batch-create`` management command that allows to create multiple projects
+  at once from a directory containing input files.
+  https://github.com/aboutcode-org/scancode.io/issues/1437
+
+- Do not download input_urls in management commands. The fetch/download is delegated to
+  the pipeline execution.
+  https://github.com/aboutcode-org/scancode.io/issues/1437
+
+- Add a "TODOS" sheet containing on REQUIRES_REVIEW resources in XLSX.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
+- Improve XLSX output for Vulnerabilities.
+  Replace the ``affected_by_vulnerabilities`` field in the PACKAGES and DEPENDENCIES
+  sheets with a dedicated VULNERABILITIES sheet.
+  https://github.com/aboutcode-org/scancode.io/issues/1519
+
+- Keep the InputSource objects when using ``reset`` on Projects.
+  https://github.com/aboutcode-org/scancode.io/issues/1536
+
+- Add a ``report`` management command that allows to generate XLSX reports for
+  multiple projects at once using labels and searching by project name.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
+- Add the ability to "select across" in Projects list when using the "select all"
+  checkbox on paginated list.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
+- Update scancode-toolkit to v32.3.2. See CHANGELOG for updates:
+  https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.3.2
+  https://github.com/aboutcode-org/scancode-toolkit/releases/tag/v32.3.1
+
+- Adds  a project settings ``scan_max_file_size`` and a scancode.io settings field
+  ``SCANCODEIO_SCAN_MAX_FILE_SIZE`` to skip scanning files above a certain
+  file size (in bytes) as a temporary fix for large memory spikes while
+  scanning for licenses in certain large files.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/3711
+
+v34.9.3 (2024-12-31)
+--------------------
+
+- Refine the available settings for RQ_QUEUES:
+  * Rename the RQ_QUEUES sub-settings to SCANCODEIO_RQ_REDIS_*
+  * Add SCANCODEIO_RQ_REDIS_SSL setting to enable SSL.
+  https://github.com/aboutcode-org/scancode.io/issues/1465
+
+- Add support to map binaries to source files using symbols
+  for rust binaries and source files. This adds also using
+  ``rust-inspector`` to extract symbols from rust binaries.
+  This is a new optional ``Rust`` step in the
+  ``map_deploy_to_develop`` pipeline.
+  https://github.com/aboutcode-org/scancode.io/issues/1435
+
+v34.9.2 (2024-12-10)
+--------------------
+
+- Fix an issue with the ``scan_rootfs_for_system_packages`` pipe when a namespace is
+  missing for the discovered packages.
+  https://github.com/aboutcode-org/scancode.io/issues/1462
+
+v34.9.1 (2024-12-09)
 --------------------
 
 - Add the ability to filter on Project endpoint API actions.

Dockerfile

@@ -20,7 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-FROM python:3.12-slim
+FROM python:3.13-slim
 
 LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/scancode.io"
 LABEL org.opencontainers.image.description="ScanCode.io"
@@ -87,7 +87,7 @@ RUN mkdir -p /var/$APP_NAME/static/ \
  && mkdir -p /var/$APP_NAME/workspace/
 
 # Install the dependencies before the codebase COPY for proper Docker layer caching
-COPY --chown=$APP_USER:$APP_USER setup.cfg setup.py $APP_DIR/
+COPY --chown=$APP_USER:$APP_USER pyproject.toml $APP_DIR/
 RUN pip install --no-cache-dir .
 
 # Copy the codebase and set the proper permissions for the APP_USER