Run train-gibberish-model during builds #2402

Signed-off-by: Jono Yang <jyang@nexb.com>

Author: JonoYang

Dockerfile

@@ -37,10 +37,12 @@ WORKDIR /scancode-toolkit
 # Copy sources into docker container
 COPY . /scancode-toolkit
 
-# Initial configuration using ./configure, scancode-reindex-licenses to build
-# the base license index
+# Initial configuration using ./configure, train-gibberish-model to build the
+# pickle containing the probabilities , scancode-reindex-licenses to build the base
+# license index
 RUN ./configure \
  && ./venv/bin/scancode-reindex-licenses
+ && ./venv/bin/train-gibberish-model
 
 # Add scancode to path
 ENV PATH=/scancode-toolkit:$PATH

etc/release/scancode-create-pypi-wheel.sh

@@ -19,6 +19,7 @@ set -e
 
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/train-gibberish-model
 
 python_tag=$( python -c "import platform;print(f\"cp{''.join(platform.python_version_tuple()[:2])}\")" )
 

etc/release/scancode-create-release-app-linux.sh

@@ -65,6 +65,7 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-macos.sh

@@ -63,6 +63,7 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \

etc/release/scancode-create-release-app-windows.sh

@@ -62,6 +62,7 @@ cp -r etc/thirdparty $release_dir/etc
 # Build the wheel
 ./configure --dev
 venv/bin/scancode-reindex-licenses
+venv/bin/train-gibberish-model
 venv/bin/python setup.py --quiet bdist_wheel --python-tag cp$python_version
 
 cp -r \