Parallel testing for faster local testing (up to 4a4f2ca)

Author: shaypal5

pyproject.toml

@@ -90,6 +90,9 @@ namespaces = false # to disable scanning PEP 420 namespaces (true by default)
 
 # === Linting & Formatting ===
 
+[tool.black]
+line-length = 120
+
 # --- ruff ---
 
 [tool.ruff]
@@ -146,6 +149,7 @@ lint.per-file-ignores."tests/**" = [
   "D401",
   "S101",
   "S105",
+  "S110",
   "S311",
   "S603",
 ]
@@ -188,6 +192,7 @@ addopts = [
   "-v",
   "-s",
   "-W error",
+  # Note: parallel execution is opt-in via -n/--numprocesses (pytest-xdist)
 ]
 markers = [
   "mongo: test the MongoDB core",
@@ -199,6 +204,8 @@ markers = [
   "maxage: test the max_age functionality",
   "asyncio: marks tests as async",
   "smoke: fast smoke tests with no external service dependencies",
+  "flaky: tests that are known to be flaky and should be retried",
+  "seriallocal: local core tests that should run serially",
 ]
 
 [tool.coverage.report]
@@ -209,10 +216,17 @@ exclude_lines = [
   "raise NotImplementedError", # Don't complain if tests don't hit defensive assertion code:
   "if TYPE_CHECKING:",         # Is only true when running mypy, not tests
 ]
+# Parallel test execution configuration
+# Use: pytest -n auto (for automatic worker detection)
+# Or: pytest -n 4 (for specific number of workers)
+# Memory tests are safe to run in parallel by default
+# Pickle tests require isolation (handled by conftest.py fixture)
+
 # --- coverage ---
 
 [tool.coverage.run]
 branch = true
+parallel = true
 # dynamic_context = "test_function"
 omit = [
   "tests/*",

scripts/test-local.sh

@@ -26,6 +26,8 @@ KEEP_RUNNING=false
 SELECTED_CORES=""
 INCLUDE_LOCAL_CORES=false
 TEST_FILES=""
+PARALLEL=false
+PARALLEL_WORKERS="auto"
 
 # Function to print colored messages
 print_message() {
@@ -57,6 +59,8 @@ OPTIONS:
     -k, --keep-running  Keep containers running after tests
     -h, --html-coverage Generate HTML coverage report
     -f, --files         Specify test files to run (can be used multiple times)
+    -p, --parallel      Run tests in parallel using pytest-xdist
+    -w, --workers       Number of parallel workers (default: auto)
     --help              Show this help message
 
 EXAMPLES:
@@ -66,6 +70,8 @@ EXAMPLES:
     $0 external -k              # Run external backends, keep containers
     $0 mongo memory -v          # Run MongoDB and memory tests verbosely
     $0 all -f tests/test_main.py -f tests/test_redis_core_coverage.py  # Run specific test files
+    $0 memory pickle -p         # Run local tests in parallel
+    $0 all -p -w 4             # Run all tests with 4 parallel workers
 
 ENVIRONMENT:
     You can also set cores via CACHIER_TEST_CORES environment variable:
@@ -103,6 +109,20 @@ while [[ $# -gt 0 ]]; do
             usage
             exit 0
             ;;
+        -p|--parallel)
+            PARALLEL=true
+            shift
+            ;;
+        -w|--workers)
+            shift
+            if [[ $# -eq 0 ]] || [[ "$1" == -* ]]; then
+                print_message $RED "Error: -w/--workers requires a number argument"
+                usage
+                exit 1
+            fi
+            PARALLEL_WORKERS="$1"
+            shift
+            ;;
         -*)
             print_message $RED "Unknown option: $1"
             usage
@@ -234,6 +254,17 @@ check_dependencies() {
         }
     fi
 
+    # Check for pytest-xdist if parallel testing is requested
+    if [ "$PARALLEL" = true ]; then
+        if ! python -c "import xdist" 2>/dev/null; then
+            print_message $YELLOW "Installing pytest-xdist for parallel testing..."
+            pip install pytest-xdist || {
+                print_message $RED "Failed to install pytest-xdist"
+                exit 1
+            }
+        fi
+    fi
+
     # Check MongoDB dependencies if testing MongoDB
     if echo "$SELECTED_CORES" | grep -qw "mongo"; then
         if ! python -c "import pymongo" 2>/dev/null; then
@@ -423,14 +454,20 @@ main() {
     # Check and install dependencies
     check_dependencies
 
-    # Check if we need Docker
+    # Check if we need Docker, and if we should run serial pickle tests
     needs_docker=false
+    run_serial_local_tests=false
     for core in $SELECTED_CORES; do
         case $core in
             mongo|redis|sql)
                 needs_docker=true
                 ;;
         esac
+        case $core in
+            pickle|all)
+                run_serial_local_tests=true
+                ;;
+        esac
     done
 
     if [ "$needs_docker" = true ]; then
@@ -497,15 +534,31 @@ main() {
             sql) test_sql ;;
         esac
     done
+    if [ -n "$pytest_markers" ]; then
+        pytest_markers="($pytest_markers) and not seriallocal"
+    else
+        pytest_markers="not seriallocal"
+    fi
 
     # Run pytest
     # Build pytest command
     PYTEST_CMD="pytest"
+    # and the specific pytest command for running serial pickle tests
+    SERIAL_PYTEST_CMD="pytest -m seriallocal"
+    # Only add -n0 if pytest-xdist is available; otherwise, plain pytest is already serial
+    if python - << 'EOF' >/dev/null 2>&1
+import xdist  # noqa: F401
+EOF
+    then
+        SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD -n0"
+    fi
 
     # Add test files if specified
     if [ -n "$TEST_FILES" ]; then
         PYTEST_CMD="$PYTEST_CMD $TEST_FILES"
         print_message $BLUE "Test files specified: $TEST_FILES"
+        # and turn off serial local tests, so we run only selected files
+        run_serial_local_tests=false
     fi
 
     # Add markers if needed (only if no specific test files were given)
@@ -517,6 +570,10 @@ main() {
 
         if [ "$selected_sorted" != "$all_sorted" ]; then
             PYTEST_CMD="$PYTEST_CMD -m \"$pytest_markers\""
+        else
+            print_message $BLUE "Running all tests without markers since all cores are selected"
+            PYTEST_CMD="$PYTEST_CMD -m \"not seriallocal\""
+            run_serial_local_tests=true
         fi
     else
         # When test files are specified, still apply markers if not running all cores
@@ -532,15 +589,41 @@ main() {
     # Add verbose flag if needed
     if [ "$VERBOSE" = true ]; then
         PYTEST_CMD="$PYTEST_CMD -v"
+        SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD -v"
+    fi
+
+    # Add parallel testing options if requested
+    if [ "$PARALLEL" = true ]; then
+        PYTEST_CMD="$PYTEST_CMD -n $PARALLEL_WORKERS"
+
+        # Show parallel testing info
+        if [ "$PARALLEL_WORKERS" = "auto" ]; then
+            print_message $BLUE "Running tests in parallel with automatic worker detection"
+        else
+            print_message $BLUE "Running tests in parallel with $PARALLEL_WORKERS workers"
+        fi
+
+        # Special note for pickle tests
+        if echo "$SELECTED_CORES" | grep -qw "pickle"; then
+            print_message $YELLOW "Note: Pickle tests will use isolated cache directories for parallel safety"
+        fi
     fi
 
     # Add coverage options
     PYTEST_CMD="$PYTEST_CMD --cov=cachier --cov-report=$COVERAGE_REPORT"
+    SERIAL_PYTEST_CMD="$SERIAL_PYTEST_CMD --cov=cachier --cov-report=$COVERAGE_REPORT --cov-append"
 
     # Print and run the command
     print_message $BLUE "Running: $PYTEST_CMD"
     eval $PYTEST_CMD
 
+    if [ "$run_serial_local_tests" = true ]; then
+        print_message $BLUE "Running serial local tests (pickle, memory) with: $SERIAL_PYTEST_CMD"
+        eval $SERIAL_PYTEST_CMD
+    else
+        print_message $BLUE "Skipping serial local tests (pickle, memory) since not requested"
+    fi
+
     TEST_EXIT_CODE=$?
 
     if [ $TEST_EXIT_CODE -eq 0 ]; then