Merge pull request #4610 from aboutcode-org/2402-detect-gibberish-copyright

Detect gibberish copyright #2402

Author: JonoYang

.gitignore

@@ -116,3 +116,7 @@ selenium
 rpmdb.sqlite-*
 /.ruff_cache/
 .env
+
+# Ignore gibberish detector model, this is trained on configure and build and
+# should not be committed
+gib_model.pki

CHANGELOG.rst

@@ -13,6 +13,14 @@ Next release
 - Replace unmaintained ``toml`` library with ``tomllib`` / ``tomli``.
   https://github.com/aboutcode-org/scancode-toolkit/issues/4532
 
+- Add gibberish detection to copyright scanning. This is done using a
+  2-character Markov chain. A new CLI command,
+  ``scancode-train-gibberish-model``, has been added to regenerate the model
+  used by the detector.
+  https://github.com/aboutcode-org/scancode-toolkit/pull/4610
+  https://github.com/aboutcode-org/scancode-toolkit/issues/2402
+
+
 v32.4.1 - 2025-07-23
 --------------------
 

Dockerfile

@@ -38,11 +38,13 @@ WORKDIR /scancode-toolkit
 COPY . /scancode-toolkit
 
 # Initial configuration using ./configure, scancode-reindex-licenses to build
-# the base license index and scancode-reindex-package-patterns to build the 
-# package patterns cache
+# the base license index, scancode-reindex-package-patterns to build the package
+# patterns cache, and scancode-train-gibberish-model to train the Markov chain
+# model used for gibberish detection.
 RUN ./configure \
  && ./venv/bin/scancode-reindex-licenses \
- && ./venv/bin/scancode-reindex-package-patterns
+ && ./venv/bin/scancode-reindex-package-patterns \
+ && ./venv/bin/scancode-train-gibberish-model
 
 # Add scancode to path
 ENV PATH=/scancode-toolkit:$PATH

configure

@@ -319,6 +319,6 @@ find_python
 create_virtualenv "$VIRTUALENV_DIR"
 install_packages "$CFG_REQUIREMENTS"
 . "$CFG_BIN_DIR/activate"
-
+"$CFG_BIN_DIR/scancode-train-gibberish-model"
 
 set +e

configure.bat

@@ -161,7 +161,7 @@ if %ERRORLEVEL% neq 0 (
     %CFG_QUIET% ^
     %PIP_EXTRA_ARGS% ^
     %CFG_REQUIREMENTS%
-
+"%CFG_BIN_DIR%\scancode-train-gibberish-model"
 
 @rem ################################
 :create_bin_junction

docs/source/reference/scancode-cli/cli-scancode-train-gibberish-model.rst

@@ -0,0 +1,65 @@
+.. _cli-scancode-train-gibberish-model:
+
+ScanCode train gibberish model
+==============================
+
+ScanCode uses a 2-character Markov chain to perform gibberish detection on text.
+At a high level, it detects gibberish strings by seeing if a sequence of letters
+is part or a whole word, two letters at a time. It does this by checking how
+likely it is to go from one letter to another. The probabilities of going from
+one letter to another are determined by a model that has been trained on a large
+set of valid text, where it counts each transition between letters and computes
+a probability based off of that. These probabilities and thresholds are stored
+in a model that is saved to a Python pickle.
+
+The training corpus for the gibberish detector can be found in
+``src/textcode/data/gibberish/``.
+
+``big.txt`` contains the main source of valid words that the gibberish detector
+model is trained on.
+
+``good.txt`` and ``bad.txt`` are used to determine the average threshold, where
+any letter transition whose average transition probability falls below this
+threshold is classified as gibberish.
+
+
+Usage: ``scancode-train-gibberish-model [OPTIONS]``
+
+Quick Reference
+---------------
+
+  --big FILE   Text file containing main training corpus for the gibberish
+               detector
+  --good FILE  Text file containing text considered to be not gibberish (good)
+  --bad FILE   Text file containing text considered to be gibberish (bad)
+  -h, --help   Show this message and exit.
+
+----
+
+.. _cli-scancode-train-gibberish-model-big-option:
+
+``--big`` option
+^^^^^^^^^^^^^^^^
+
+The ``--big`` option allows the user to use a different text file to train the
+gibberish detector model.
+
+.. _cli-scancode-train-gibberish-model-good-option:
+
+``--good`` option
+^^^^^^^^^^^^^^^^^
+
+The ``--good`` option allows the user to use a different text file containing
+strings considered to be valid copyrights. This option is used to adjust the
+average transition probability threshold that determines whether or not a string
+is gibberish.
+
+.. _cli-scancode-train-gibberish-model-bad-option:
+
+``--bad`` option
+^^^^^^^^^^^^^^^^
+
+The ``--bad`` option allows the user to use a different text file containing
+strings considered to be invalid copyrights. This option is used to adjust the
+average transition probability threshold that determines whether or not a string
+is gibberish.

docs/source/reference/scancode-cli/index.rst

@@ -88,3 +88,4 @@ For more details into the post-scan CLI options, see :ref:`cli-post-scan-options
    cli-extractcode
    cli-scancode-reindex-licenses
    cli-scancode-license-data
+   cli-scancode-train-gibberish-model

etc/release/scancode-create-pypi-wheel.sh

@@ -20,6 +20,7 @@ set -e
 ./configure --dev
 venv/bin/scancode-reindex-licenses
 venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 
 python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" )
 

etc/release/scancode-create-release-app-linux.sh

@@ -66,6 +66,7 @@ cp -r etc/thirdparty $release_dir/etc
 ./configure --dev
 venv/bin/scancode-reindex-licenses
 venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-macos.sh

@@ -64,6 +64,7 @@ cp -r etc/thirdparty $release_dir/etc
 ./configure --dev
 venv/bin/scancode-reindex-licenses
 venv/bin/scancode-reindex-package-patterns
+venv/bin/scancode-train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \