Added waveform & spectrogram verification to integration tests (#209)

* Added waveform & spectrogram verification to integration tests

* Raised minimum similarity threshold for waveform/spectrogram image comparison in integration tests, except for demucs which is pretty inconsistent with itself

* Updated dependencies, moved to using SSIM for wave/spect comparison in integration test

* Attempt to fix integration test volatility with demucs by fixing Y axis for waveform images, but spectrograms are still all over the place

* Added integration test verifying the output dir and custom output names work as expected

* Split waveform and spec thresholds, lowered all, added images to test artifacts for CI

* Split DEFAULT_SIMILARITY_THRESHOLD to override spectrogram threshold

* Only run integration tests in integration workflow...

Author: beveradb

.github/workflows/run-integration-tests.yaml

@@ -59,7 +59,7 @@ jobs:
           ls -la $AUDIO_SEPARATOR_MODEL_DIR || echo "Directory empty or doesn't exist"
 
       - name: Run integration tests
-        run: poetry run pytest tests -v --cov=audio_separator --cov-report=xml
+        run: poetry run pytest -sv --cov=audio_separator --cov-report=xml tests/integration
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
@@ -74,3 +74,5 @@ jobs:
           path: |
             *.flac
             tests/*.flac
+            **/temp_images/**/*.png
+            tests/**/temp_images/**/*.png

.gitignore

@@ -13,6 +13,7 @@
 *.mp3
 tests/model-metrics/results
 tests/model-metrics/datasets
+temp_images
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

poetry.lock

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "audio-separator"
-version = "0.31.3"
+version = "0.32.0"
 description = "Easy to use audio stem separation, using various models from UVR trained primarily by @Anjok07"
 authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
 license = "MIT"
@@ -65,7 +65,10 @@ audio-separator = 'audio_separator.utils.cli:main'
 [tool.poetry.group.dev.dependencies]
 black = ">=23"
 pytest = "*"
-pytest-cov = "^4.1.0"
+pytest-cov = ">=4.1.0"
+matplotlib = ">=3.8.0"
+pillow = ">=10.1.0"
+scikit-image = ">=0.22.0"
 
 [tool.black]
 line-length = 140