Add parallel test execution infrastructure

.github/workflows/ci-test.yml

@@ -47,16 +47,17 @@ jobs:
       CACHIER_TEST_VS_DOCKERIZED_REDIS: "true"
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
 
-      - name: Install package & dependencies
-        run: python -m pip install -e . -r tests/requirements.txt
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e . -r tests/requirements.txt
 
       - name: Lower inotify instance limit (Linux only, for Issue #24 test)
         if: runner.os == 'Linux' && matrix.backend == 'local'
@@ -75,17 +76,14 @@ jobs:
           # For testcontainers to find the Colima socket
           sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
 
-      - name: Install other test dependencies
-        if: matrix.backend != 'local'
-        run: python -m pip install -e . -r tests/requirements_${{ matrix.backend }}.txt
-
       # ToDo: find a way to cache docker images
       #- name: Cache Container Images
       #  if: matrix.backend == 'mongodb'
       #  uses: borda/cache-container-images-action@b32a5e804cb39af3c3d134fc03ab76eac0bfcfa9
       #  with:
       #    prefix-key: "mongo-db"
       #    images: mongo:latest
+
       - name: Start MongoDB in docker
         if: matrix.backend == 'mongodb'
         run: |
@@ -96,6 +94,11 @@ jobs:
           # show running containers
           docker ps -a
 
+      - name: Install MongoDB core test dependencies
+        if: matrix.backend == 'mongodb'
+        run: |
+          python -m pip install -e . -r tests/mongodb_requirements.txt
+
       - name: Unit tests (DB)
         if: matrix.backend == 'mongodb'
         run: pytest -m "mongo" --cov=cachier --cov-report=term --cov-report=xml:cov.xml
@@ -115,10 +118,15 @@ jobs:
           sleep 10
           docker ps -a
 
+      - name: Install SQL core test dependencies (SQL/Postgres)
+        if: matrix.backend == 'postgres'
+        run: |
+          python -m pip install -e . -r tests/sql_requirements.txt
+
       - name: Unit tests (SQL/Postgres)
         if: matrix.backend == 'postgres'
         env:
-          SQLALCHEMY_DATABASE_URL: postgresql+psycopg://testuser:testpass@localhost:5432/testdb
+          SQLALCHEMY_DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb
         run: pytest -m sql --cov=cachier --cov-report=term --cov-report=xml:cov.xml
 
       - name: Start Redis in docker
@@ -131,6 +139,11 @@ jobs:
           sleep 5
           docker ps -a
 
+      - name: Install Redis core test dependencies
+        if: matrix.backend == 'redis'
+        run: |
+          python -m pip install -e . -r tests/redis_requirements.txt
+
       - name: Unit tests (Redis)
         if: matrix.backend == 'redis'
         run: pytest -m redis --cov=cachier --cov-report=term --cov-report=xml:cov.xml

.github/workflows/release.yml

@@ -22,8 +22,8 @@ jobs:
     env:
       RELEASING_PROCESS: "1"
     steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Install dependencies
@@ -43,7 +43,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload dist artifact for the publish job
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: cachier_dist
           path: dist
@@ -56,11 +56,11 @@ jobs:
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
     steps:
-      - uses: actions/setup-python@v6
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
       - name: Download dist artifact
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v4
         with:
           name: cachier_dist
           path: dist
@@ -77,12 +77,12 @@ jobs:
     environment: pypi_publish
     steps:
       - name: Download dist artifact
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v4
         with:
           name: cachier_dist
           path: dist
       - name: Publish distribution 📦 to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.13.0
+        uses: pypa/gh-action-pypi-publish@v1.12.4
         with:
           user: __token__
           password: ${{ secrets.pypi_password }}

Makefile

@@ -52,9 +52,9 @@ install-dev:
 install-all:
 	pip install -e .[all]
 	pip install -r tests/requirements.txt
-	pip install -r tests/requirements_mongodb.txt
-	pip install -r tests/requirements_redis.txt
-	pip install -r tests/requirements_postgres.txt
+	pip install -r tests/mongodb_requirements.txt
+	pip install -r tests/redis_requirements.txt
+	pip install -r tests/sql_requirements.txt
 
 # Testing targets
 test:

pyproject.toml

@@ -179,6 +179,7 @@ markers = [
   "sql: test the SQL core",
   "maxage: test the max_age functionality",
   "asyncio: marks tests as async",
+  "seriallocal: tests that must run serially for local backends",
 ]
 
 [tool.coverage.report]

scripts/README-local-testing.md

@@ -41,6 +41,8 @@ This guide explains how to run cachier tests locally with Docker containers for
 - `-k, --keep-running` - Keep Docker containers running after tests
 - `-h, --html-coverage` - Generate HTML coverage report
 - `-f, --files` - Specify test files to run (can be used multiple times)
+- `-p, --parallel` - Run tests in parallel using pytest-xdist
+- `-w, --workers` - Number of parallel workers (default: auto)
 - `--help` - Show help message
 
 ## Examples
@@ -96,6 +98,18 @@ CACHIER_TEST_CORES="mongo redis" ./scripts/test-local.sh
 
 # Combine file selection with other options
 ./scripts/test-local.sh redis sql -f tests/test_sql_core.py -v -k
+
+# Run tests in parallel with automatic worker detection
+./scripts/test-local.sh all -p
+
+# Run tests in parallel with 4 workers
+./scripts/test-local.sh external -p -w 4
+
+# Run local tests in parallel (memory and pickle)
+./scripts/test-local.sh memory pickle -p
+
+# Combine parallel testing with other options
+./scripts/test-local.sh mongo redis -p -v -k
 ```
 
 ### Docker Compose
@@ -107,7 +121,7 @@ make services-start
 # Run tests manually
 CACHIER_TEST_HOST=localhost CACHIER_TEST_PORT=27017 CACHIER_TEST_VS_DOCKERIZED_MONGO=true \
 CACHIER_TEST_REDIS_HOST=localhost CACHIER_TEST_REDIS_PORT=6379 CACHIER_TEST_VS_DOCKERIZED_REDIS=true \
-SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" \
+SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" \
 pytest -m "mongo or redis or sql"
 
 # Stop all services
@@ -146,17 +160,17 @@ The script automatically sets the required environment variables:
 
 ### SQL/PostgreSQL
 
-- `SQLALCHEMY_DATABASE_URL=postgresql+psycopg://testuser:testpass@localhost:5432/testdb`
+- `SQLALCHEMY_DATABASE_URL=postgresql://testuser:testpass@localhost:5432/testdb`
 
 ## Prerequisites
 
 1. **Docker** - Required for external backends (mongo, redis, sql)
 2. **Python dependencies** - Install test requirements:
    ```bash
    pip install -r tests/requirements.txt
-   pip install -r tests/requirements_mongodb.txt  # For MongoDB tests
-   pip install -r tests/requirements_redis.txt    # For Redis tests
-   pip install -r tests/requirements_postgres.txt      # For SQL tests
+   pip install -r tests/mongodb_requirements.txt  # For MongoDB tests
+   pip install -r tests/redis_requirements.txt    # For Redis tests
+   pip install -r tests/sql_requirements.txt      # For SQL tests
    ```
 
 ## Troubleshooting
@@ -193,10 +207,12 @@ The script automatically sets the required environment variables:
 2. **For quick iteration**: Use memory and pickle tests (no Docker required)
 3. **For debugging**: Use `-k` to keep containers running and inspect them
 4. **For CI parity**: Test with the same backends that CI uses
+5. **For faster test runs**: Use `-p` to run tests in parallel, especially when testing multiple backends
+6. **For parallel testing**: The script automatically installs pytest-xdist when needed
+7. **Worker count**: Use `-w auto` (default) to let pytest-xdist determine optimal workers, or specify a number based on your CPU cores
 
 ## Future Enhancements
 
 - Add MySQL/MariaDB support
 - Add Elasticsearch support
 - Add performance benchmarking mode
-- Add parallel test execution for multiple backends

scripts/docker-compose.all-cores.yml

@@ -76,6 +76,6 @@ networks:
 #   docker-compose -f scripts/docker-compose.all-cores.yml up -d
 #   CACHIER_TEST_HOST=localhost CACHIER_TEST_PORT=27017 CACHIER_TEST_VS_DOCKERIZED_MONGO=true \
 #   CACHIER_TEST_REDIS_HOST=localhost CACHIER_TEST_REDIS_PORT=6379 CACHIER_TEST_VS_DOCKERIZED_REDIS=true \
-#   SQLALCHEMY_DATABASE_URL="postgresql+psycopg://testuser:testpass@localhost:5432/testdb" \
+#   SQLALCHEMY_DATABASE_URL="postgresql://testuser:testpass@localhost:5432/testdb" \
 #   pytest -m "mongo or redis or sql"
 #   docker-compose -f scripts/docker-compose.all-cores.yml down