Rust extractor

.github/workflows/code-quality.yml

@@ -14,6 +14,8 @@ on:
       - "**/pyproject.toml"
       - "**/uv.lock"
       - "**/Dockerfile"
+      - "extractor/rustextractor/**/*.rs"
+      - "extractor/rustextractor/**/Cargo.toml"
       - ".github/workflows/code-quality.yml"
       - ".pre-commit-config.yaml"
   pull_request:
@@ -24,6 +26,8 @@ on:
       - "**/pyproject.toml"
       - "**/uv.lock"
       - "**/Dockerfile"
+      - "extractor/rustextractor/**/*.rs"
+      - "extractor/rustextractor/**/Cargo.toml"
       - ".github/workflows/code-quality.yml"
       - ".pre-commit-config.yaml"
 
@@ -76,6 +80,24 @@ jobs:
         with:
           hadolint: latest
 
+      - name: 🦀 Setup Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+
+      - name: 💾 Cache Rust dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            extractor/rustextractor/target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
+
       - name: 📦 Install dependencies
         run: |
           uv sync --all-extras --frozen

.github/workflows/list-sub-projects.yml

@@ -33,7 +33,8 @@ jobs:
           [
             {"name": "dashboard", "use_cache": true},
             {"name": "discovery", "use_cache": false},
-            {"name": "extractor", "use_cache": true},
+            {"name": "extractor/pyextractor", "use_cache": true},
+            {"name": "extractor/rustextractor", "use_cache": true},
             {"name": "graphinator", "use_cache": true},
             {"name": "tableinator", "use_cache": true}
           ]

.github/workflows/test.yml

@@ -14,6 +14,8 @@ on:
       - "**/*.py"
       - "**/pyproject.toml"
       - "**/uv.lock"
+      - "extractor/rustextractor/**/*.rs"
+      - "extractor/rustextractor/**/Cargo.toml"
       - ".github/workflows/test.yml"
   pull_request:
     branches:
@@ -22,6 +24,8 @@ on:
       - "**/*.py"
       - "**/pyproject.toml"
       - "**/uv.lock"
+      - "extractor/rustextractor/**/*.rs"
+      - "extractor/rustextractor/**/Cargo.toml"
       - ".github/workflows/test.yml"
 
 env:
@@ -32,9 +36,9 @@ permissions:
   contents: read
 
 jobs:
-  test:
+  python-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20
 
     steps:
       - name: 🔀 Checkout repository
@@ -94,7 +98,7 @@ jobs:
         run: |
           # Run tests in groups to avoid async conflicts
           echo "Running core tests..."
-          uv run pytest tests/test_config.py tests/test_integration.py tests/extractor/ -v
+          PYTHONPATH="${GITHUB_WORKSPACE}/extractor:${PYTHONPATH}" uv run pytest tests/test_config.py tests/test_integration.py tests/extractor/ -v
 
           echo "Running dashboard tests..."
           uv run pytest tests/dashboard/ -v -m "not e2e"
@@ -110,4 +114,78 @@ jobs:
 
           # Generate combined coverage report
           echo "Generating coverage report..."
-          uv run pytest --cov --cov-append --cov-report=term --cov-report=xml -m "not e2e" --no-header -q
+          PYTHONPATH="${GITHUB_WORKSPACE}/extractor:${PYTHONPATH}" uv run pytest --cov --cov-append --cov-report=term --cov-report=xml -m "not e2e" --no-header -q
+
+      - name: 📊 Upload Python coverage reports
+        if: github.event_name != 'pull_request' || steps.check-tests.outputs.needed == 'true'
+        uses: codecov/codecov-action@v5
+        with:
+          files: coverage.xml
+          flags: python
+          name: python-coverage
+          fail_ci_if_error: false
+
+  rust-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: 🔀 Checkout repository
+        uses: actions/checkout@v5
+
+      - name: 🔍 Check if Rust extractor exists
+        id: check-rust
+        run: |
+          if [[ -f "extractor/rustextractor/Cargo.toml" ]]; then
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+            echo "✅ Rust extractor found - tests will run"
+          else
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+            echo "⏭️ No Rust extractor found - skipping Rust tests"
+          fi
+
+      - name: ⏭️ Skip remaining steps if Rust extractor doesn't exist
+        if: steps.check-rust.outputs.exists == 'false'
+        run: |
+          echo "::notice title=Rust Tests Skipped::No Rust extractor found in this branch"
+          exit 0
+
+      - name: 🦀 Setup Rust toolchain
+        if: steps.check-rust.outputs.exists == 'true'
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: llvm-tools-preview
+
+      - name: 📦 Install cargo-llvm-cov
+        if: steps.check-rust.outputs.exists == 'true'
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      - name: 💾 Cache Rust dependencies
+        if: steps.check-rust.outputs.exists == 'true'
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            extractor/rustextractor/target/
+          key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-test-
+            ${{ runner.os }}-cargo-
+
+      - name: 🧪 Run Rust tests with coverage
+        if: steps.check-rust.outputs.exists == 'true'
+        run: |
+          cd extractor/rustextractor
+          cargo llvm-cov test --verbose --lcov --output-path lcov.info
+
+      - name: 📊 Upload Rust coverage reports
+        if: steps.check-rust.outputs.exists == 'true'
+        uses: codecov/codecov-action@v5
+        with:
+          files: extractor/rustextractor/lcov.info
+          flags: rust
+          name: rust-coverage
+          fail_ci_if_error: false

.gitignore

@@ -201,6 +201,43 @@ static/build/
 backups/
 uv.lock.backup
 
+###
+# Rust
+###
+# Generated by Cargo
+# will have compiled files and executables
+target/
+debug/
+release/
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+# Rust lockfile is committed for applications, not libraries
+# Cargo.lock
+
+# Cargo configuration
+.cargo/config.toml
+.cargo/registry
+
+# IDE specific files for Rust
+*.iml
+*.ipr
+
+# Rust profiling
+*.profdata
+*.profraw
+
+# Criterion.rs benchmark results
+target/criterion/
+
+# Tarpaulin code coverage
+cobertura.xml
+tarpaulin-report.html
+
 ###
 # macOS
 ###

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b  # frozen: v5.0.0
+    rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c  # frozen: v6.0.0
     hooks:
       - id: check-added-large-files
       - id: check-executables-have-shebangs
@@ -22,7 +22,7 @@ repos:
         args: [--autofix, --indent=2]
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 54da05914997e6b04e4db33ed6757d744984c68b  # frozen: 0.33.2
+    rev: 9b9df9f1d3bd9a5e7a52d8f61a3be3e29e73d127  # frozen: 0.33.3
     hooks:
       - id: check-github-workflows
       - id: check-github-actions
@@ -35,16 +35,17 @@ repos:
           - mdformat-black
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 4cbc74d53fe5634e58e0e65db7d28939c9cec3f7  # frozen: v0.12.7
+    rev: 0acff885bcb16381b67930fefb91e460202f172c  # frozen: v0.12.10
     hooks:
       - id: ruff
         args: [--fix]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 850d8bf806620ef89a99381c5cf5ea2c1ea826dd  # frozen: v1.17.0
+    rev: 412de98d50e846f31ea6f4b0ad036f2c24a7a024  # frozen: v1.17.1
     hooks:
       - id: mypy
+        files: ^(common|dashboard|discovery|extractor|graphinator|tableinator)/.*\.py$
         additional_dependencies:
           - types-tqdm
           - types-xmltodict
@@ -84,7 +85,7 @@ repos:
         args: [--strict, -c=.yamllint]
 
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: a23f6b85d0fdd5bb9d564e2579e678033debbdff  # frozen: v0.10.0.1
+    rev: 745eface02aef23e168a8afb6b5737818efbea95  # frozen: v0.11.0.1
     hooks:
       - id: shellcheck
         args: ["--severity=warning"]  # Show warnings and errors
@@ -94,3 +95,19 @@ repos:
     hooks:
       - id: shfmt
         args: ["-i", "2", "-ci", "-bn", "-sr"]  # 2 space indent, indent case, binary next line, redirect operators
+
+  # Rust checks
+  - repo: local
+    hooks:
+      - id: cargo-fmt
+        name: cargo fmt
+        entry: bash -c 'cd extractor/rustextractor && (rustup component add rustfmt 2>/dev/null || true) && cargo fmt --check'
+        language: system
+        files: '^extractor/rustextractor/.*\.rs$'
+        pass_filenames: false
+      - id: cargo-clippy
+        name: cargo clippy
+        entry: bash -c 'cd extractor/rustextractor && (rustup component add clippy 2>/dev/null || true) && cargo clippy -- -D warnings'
+        language: system
+        files: '^extractor/rustextractor/.*\.rs$'
+        pass_filenames: false

README.md

@@ -38,7 +38,8 @@ Perfect for music researchers, data scientists, developers, and music enthusiast
 
 | Service | Purpose | Key Technologies |
 |---------|---------|------------------|
-| **[📥](docs/emoji-guide.md#service-identifiers) Extractor** | Downloads & processes Discogs XML dumps | `asyncio`, `orjson`, `aio-pika` |
+| **[📥](docs/emoji-guide.md#service-identifiers) Extractor** | Downloads & processes Discogs XML dumps (Python) | `asyncio`, `orjson`, `aio-pika` |
+| **[⚡](docs/emoji-guide.md#service-identifiers) Distiller** | High-performance Rust-based extractor | `tokio`, `quick-xml`, `lapin` |
 | **[🔗](docs/emoji-guide.md#service-identifiers) Graphinator** | Builds Neo4j knowledge graphs | `neo4j-driver`, graph algorithms |
 | **[🐘](docs/emoji-guide.md#service-identifiers) Tableinator** | Creates PostgreSQL analytics tables | `psycopg3`, JSONB, full-text search |
 | **[🎵](docs/emoji-guide.md#service-identifiers) Discovery** | AI-powered music intelligence | `sentence-transformers`, `plotly`, `networkx` |
@@ -176,9 +177,13 @@ cd discogsography
 # 2. Copy environment template (optional - has sensible defaults)
 cp .env.example .env
 
-# 3. Start all services
+# 3. Start all services (default: Python extractor)
 docker-compose up -d
 
+# 3b. (Optional) Use high-performance Rust extractor instead
+./scripts/switch-extractor.sh rust
+# To switch back: ./scripts/switch-extractor.sh python
+
 # 4. Watch the magic happen!
 docker-compose logs -f
 
@@ -642,6 +647,7 @@ Typical processing rates on modern hardware:
 | Service | Records/Second | Bottleneck |
 |---------|----------------|------------|
 | 📥 **Extractor** | 5,000-10,000 | XML parsing, I/O |
+| ⚡ **Distiller** | 20,000-400,000+ | Network I/O (Rust-based) |
 | 🔗 **Graphinator** | 1,000-2,000 | Neo4j transactions |
 | 🐘 **Tableinator** | 3,000-5,000 | PostgreSQL inserts |
 

dashboard/Dockerfile

@@ -8,7 +8,7 @@ ARG GID=1000
 FROM python:${PYTHON_VERSION}-slim AS builder
 
 # Install uv
-COPY --from=ghcr.io/astral-sh/uv:0.8.5 /uv /bin/uv
+COPY --from=ghcr.io/astral-sh/uv:0.8.13 /uv /bin/uv
 
 # Set environment for build
 ENV UV_SYSTEM_PYTHON=1 \

discovery/Dockerfile

@@ -8,7 +8,7 @@ ARG GID=1000
 FROM python:${PYTHON_VERSION}-slim AS builder
 
 # Install uv
-COPY --from=ghcr.io/astral-sh/uv:0.8.5 /uv /bin/uv
+COPY --from=ghcr.io/astral-sh/uv:0.8.13 /uv /bin/uv
 
 # Set environment for build
 ENV UV_SYSTEM_PYTHON=1 \

docker-compose.yml

@@ -112,11 +112,48 @@ services:
     security_opt:
       - no-new-privileges:true
 
-  # Extractor service
+  # Rust-based extractor service
+  rustextractor:
+    build:
+      context: .
+      dockerfile: extractor/rustextractor/Dockerfile
+    image: discogsography/rustextractor:latest
+    container_name: discogsography-rustextractor
+    hostname: rustextractor
+    user: "1000:1000"
+    environment:
+      AMQP_CONNECTION: amqp://discogsography:discogsography@rabbitmq:5672/
+      DISCOGS_ROOT: /discogs-data
+      RUST_LOG: rust_extractor=info,lapin=warn
+      STARTUP_DELAY: "10"
+    volumes:
+      - discogs_data:/discogs-data
+      - rust_extractor_logs:/logs
+    depends_on:
+      rabbitmq:
+        condition: service_healthy
+    networks:
+      - discogsography
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s
+    restart: on-failure
+    logging: *default-logging
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    profiles:
+      - rustextractor
+
+  # Python-based extractor service (default)
   extractor:
     build:
       context: .
-      dockerfile: extractor/Dockerfile
+      dockerfile: extractor/pyextractor/Dockerfile
       args:
         PYTHON_VERSION: ${PYTHON_VERSION:-3.13}
         UID: ${UID:-1000}
@@ -370,3 +407,4 @@ volumes:
   neo4j_plugins:
   discogs_data:
   discovery_models:
+  rust_extractor_logs: