Merge branch 'main' into 1145-load-spdx-dependencies

Author: tdruez

.dockerignore

@@ -1,3 +1,4 @@
+**/.*
 **/.git
 **/.gitignore
 **/.github
@@ -9,6 +10,7 @@
 **/.DS_Store
 **/.aof
 **/venv
+**/.venv
 **/env
 **/bin
 **/build
@@ -18,11 +20,14 @@
 **/lib
 **/var
 **/*.egg-info
+**/.*cache
 .dockerignore
 .readthedocs.yaml
 docker.env
 .env
 CHANGELOG.rst
+MANIFEST.in
+Makefile
 Dockerfile
 README.rst
 docker-compose.yml

.github/workflows/pypi-release-aboutcode-pipeline.yml

@@ -0,0 +1,38 @@
+name: Build aboutcode.pipeline Python distributions and publish on PyPI
+
+on:
+  workflow_dispatch:
+  push:
+   tags:
+     - "aboutcode.pipeline/*"
+
+jobs:
+  build-and-publish:
+    name: Build and publish library to PyPI
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install flot
+        run: python -m pip install flot --user
+
+      - name: Build a binary wheel and a source tarball
+        run:  python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/
+
+      - name: Publish to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN_ABOUTCODE_PIPELINE }}
+
+      - name: Upload built archives
+        uses: actions/upload-artifact@v4
+        with:
+          name: pypi_archives
+          path: dist/*

CHANGELOG.rst

@@ -26,6 +26,31 @@ v34.7.2 (unreleased)
 - Add a new Dependency view that renders the project dependencies as a tree.
   https://github.com/nexB/scancode.io/issues/1145
 
+- The ``purldb-scan-worker`` command has been updated to send project results
+  back using the Project webhook subscriptions. This allows us to not have the
+  main task loop to monitor a single project run for completion in order to
+  return data, and allows us to have multiple scan projects active at once while
+  we use ``purldb-scan-worker``. A new option ``--max-concurrent-projects`` has
+  been added to set the number of purldb packages that can be requested and
+  processed at once.
+  https://github.com/nexB/scancode.io/issues/1287
+
+- Add notes field on the DiscoveredPackage model.
+  https://github.com/nexB/scancode.io/issues/1342
+
+- Fix an issue with conflicting groups checkbox id in the Add pipeline modal.
+  https://github.com/nexB/scancode.io/issues/1353
+
+- Move the BasePipeline class to a new `aboutcode.pipeline` module.
+  https://github.com/nexB/scancode.io/issues/1351
+
+- Update link references of ownership from nexB to aboutcode-org
+  https://github.com/aboutcode-org/scancode.io/issues/1350
+
+- Add a new ``check-compliance`` management command to check for compliance issues in
+  a project.
+  https://github.com/nexB/scancode.io/issues/1182
+
 v34.7.1 (2024-07-15)
 --------------------
 

Dockerfile

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
-# http://nexb.com and https://github.com/nexB/scancode.io
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
 # The ScanCode.io software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode.io is provided as-is without warranties.
 # ScanCode is a trademark of nexB Inc.
@@ -18,11 +18,11 @@
 # for any legal advice.
 #
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
-# Visit https://github.com/nexB/scancode.io for support and download.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-FROM --platform=linux/amd64 python:3.12-slim
+FROM python:3.12-slim
 
-LABEL org.opencontainers.image.source="https://github.com/nexB/scancode.io"
+LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/scancode.io"
 LABEL org.opencontainers.image.description="ScanCode.io"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 

Makefile

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 #
-# http://nexb.com and https://github.com/nexB/scancode.io
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
 # The ScanCode.io software is licensed under the Apache License version 2.0.
 # Data generated with ScanCode.io is provided as-is without warranties.
 # ScanCode is a trademark of nexB Inc.
@@ -18,7 +18,7 @@
 # for any legal advice.
 #
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
-# Visit https://github.com/nexB/scancode.io for support and download.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
 # Python version can be specified with `$ PYTHON_EXE=python3.x make conf`
 PYTHON_EXE?=python3
@@ -86,7 +86,7 @@ check-deploy:
 
 clean:
 	@echo "-> Clean the Python env"
-	rm -rf .venv/ .*_cache/ *.egg-info/ build/ dist/
+	rm -rf .venv/ .*cache/ *.egg-info/ build/ dist/
 	find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 
 migrate:
@@ -125,6 +125,10 @@ test:
 	@echo "-> Run the test suite"
 	${MANAGE} test --noinput
 
+fasttest:
+	@echo "-> Run the test suite without the PipelinesIntegrationTest"
+	${MANAGE} test --noinput --exclude-tag slow
+
 worker:
 	${MANAGE} rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 2
 
@@ -152,4 +156,4 @@ offline-package: docker-images
 	@mkdir -p dist/
 	@tar -cf dist/scancodeio-offline-package-`git describe --tags`.tar build/
 
-.PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate upgrade postgresdb sqlitedb backupdb run test docs bump docker-images offline-package
+.PHONY: virtualenv conf dev envfile install doc8 check valid check-deploy clean migrate upgrade postgresdb sqlitedb backupdb run test fasttest docs bump docker-images offline-package

NOTICE

@@ -1,6 +1,6 @@
 SPDX-License-Identifier: Apache-2.0
 
-http://nexb.com and https://github.com/nexB/scancode.io
+http://nexb.com and https://github.com/aboutcode-org/scancode.io
 The ScanCode.io software is licensed under the Apache License version 2.0.
 Data generated with ScanCode.io is provided as-is without warranties.
 ScanCode is a trademark of nexB Inc.
@@ -18,4 +18,4 @@ ScanCode.io should be considered or used as legal advice. Consult an Attorney
 for any legal advice.
 
 ScanCode.io is a free software code scanning tool from nexB Inc. and others.
-Visit https://github.com/nexB/scancode.io for support and download.
+Visit https://github.com/aboutcode-org/scancode.io for support and download.

README.rst

@@ -14,13 +14,13 @@ The ScanCode.io documentation is available here: https://scancodeio.readthedocs.
 If you have questions that are not covered by our
 `Documentation <https://scancodeio.readthedocs.io/en/latest/faq.html>`_ or
 `FAQs <https://scancodeio.readthedocs.io/en/latest/faq.html>`_,
-please ask them in `Discussions <https://github.com/nexB/scancode.io/discussions>`_.
+please ask them in `Discussions <https://github.com/aboutcode-org/scancode.io/discussions>`_.
 
 If you want to contribute to ScanCode.io, start with our
 `Contributing <https://scancodeio.readthedocs.io/en/latest/contributing.html>`_ page.
 
 A new GitHub action is now available at
-`scancode-action <https://github.com/nexB/scancode-action>`_
+`scancode-action <https://github.com/aboutcode-org/scancode-action>`_
 to run ScanCode.io pipelines from your GitHub Workflows.
 Visit https://scancodeio.readthedocs.io/en/latest/automation.html to learn more
 about automation.
@@ -56,8 +56,8 @@ ScanCode.io should be considered or used as legal advice. Consult an Attorney
 for any legal advice.
 
 
-.. |ci-tests| image:: https://github.com/nexB/scancode.io/actions/workflows/ci.yml/badge.svg?branch=main
-    :target: https://github.com/nexB/scancode.io/actions/workflows/ci.yml
+.. |ci-tests| image:: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml/badge.svg?branch=main
+    :target: https://github.com/aboutcode-org/scancode.io/actions/workflows/ci.yml
     :alt: CI Tests Status
 
 .. |docs-rtd| image:: https://readthedocs.org/projects/scancodeio/badge/?version=latest

aboutcode/pipeline/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## Release 0.1.0 (August 9, 2023)
+
+* Initial release of the `aboutcode.pipeline` library. [Issue #1351](https://github.com/nexB/scancode.io/issues/1351)

aboutcode/pipeline/README.md

@@ -0,0 +1,53 @@
+# `aboutcode.pipeline`
+
+Define and run pipelines.
+
+### Install
+
+```bash
+pip install aboutcode.pipeline
+```
+
+### Define and execute a pipeline
+
+```python
+from aboutcode.pipeline import BasePipeline
+
+class PrintMessages(BasePipeline):
+    @classmethod
+    def steps(cls):
+        return (cls.step1,)
+
+    def step1(self):
+        print("Message from step1")
+
+PrintMessages().execute()
+```
+
+### Groups and steps selection
+
+```python
+from aboutcode.pipeline import BasePipeline
+from aboutcode.pipeline import group
+
+class PrintMessages(BasePipeline):
+    @classmethod
+    def steps(cls):
+        return (cls.step1, cls.step2)
+
+    def step1(self):
+        print("Message from step1")
+
+    @group("foo")
+    def step2(self):
+        print("Message from step2")
+
+
+# Execute pipeline with group selection
+run = PrintMessages(selected_groups=["foo"])
+exitcode, error = run.execute()
+
+# Execute pipeline with steps selection
+run = PrintMessages(selected_steps=["step1"])
+exitcode, error = run.execute()
+```

aboutcode/pipeline/RELEASE.md

@@ -0,0 +1,29 @@
+# Release instructions for `aboutcode.pipeline`
+
+### Automated release workflow
+
+- Create a new `aboutcode.pipeline-release-x.x.x` branch
+- Update the version in:
+  - `pipeline-pyproject.toml`
+  - `aboutcode/pipeline/__init__.py`
+- Commit and push this branch
+- Create a PR and merge once approved
+- Tag and push to trigger the `pypi-release-aboutcode-pipeline.yml` workflow that 
+  takes care of building the distribution archives and upload those to pypi::
+  ```
+  VERSION=x.x.x  # <- Set the new version here
+  TAG=aboutcode.pipeline/$VERSION
+  git tag -a $TAG -m ""
+  git push origin $TAG
+  ```
+
+### Manual build
+
+```
+cd scancode.io
+source .venv/bin/activate
+pip install flot
+flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/
+```
+
+The distribution archives will be available in the local `dist/` directory.