Merge pull request #130 from datajoint/pre/v2.1

Author: MilagrosMarin

docker-compose.yaml

@@ -2,7 +2,8 @@
 #
 # MODE="LIVE" docker compose up --build     # Live dev server at http://localhost:8000/
 # MODE="BUILD" docker compose up --build    # Build static site
-# docker compose run docs jupyter nbconvert --execute ...  # Execute notebooks
+# MODE="EXECUTE" docker compose up --build  # Execute notebooks against MySQL
+# MODE="EXECUTE_PG" docker compose up --build  # Execute notebooks against PostgreSQL
 #
 services:
   mysql:
@@ -19,6 +20,22 @@ services:
       timeout: 5s
       retries: 10
 
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: tutorial
+      POSTGRES_DB: datajoint
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
   minio:
     image: minio/minio:latest
     environment:
@@ -42,9 +59,10 @@ services:
       dockerfile: Dockerfile
     environment:
       - MODE
-      - DJ_HOST=mysql
-      - DJ_USER=root
-      - DJ_PASS=tutorial
+      - DJ_HOST=${DJ_HOST:-mysql}
+      - DJ_USER=${DJ_USER:-root}
+      - DJ_PASS=${DJ_PASS:-tutorial}
+      - DJ_BACKEND=${DJ_BACKEND:-mysql}
     volumes:
       - .:/main
       - ${DJ_PYTHON_PATH:-../datajoint-python}:/datajoint-python
@@ -53,6 +71,8 @@ services:
     depends_on:
       mysql:
         condition: service_healthy
+      postgres:
+        condition: service_healthy
     command:
       - sh
       - -c
@@ -70,14 +90,28 @@ services:
             # Generate llms-full.txt with current git info
             python scripts/gen_llms_full.py
             mkdocs build --config-file ./mkdocs.yaml
+        elif echo "$${MODE}" | grep -i execute_pg &>/dev/null; then
+            # EXECUTE_PG mode: execute notebooks against PostgreSQL
+            pip install -e "/datajoint-python[postgres]"
+            pip install scikit-image pooch nbconvert
+            echo "Executing notebooks against PostgreSQL..."
+            python scripts/execute_notebooks.py --backend postgresql
+        elif echo "$${MODE}" | grep -i execute &>/dev/null; then
+            # EXECUTE mode: execute notebooks against MySQL (default)
+            pip install -e /datajoint-python
+            pip install scikit-image pooch nbconvert
+            echo "Executing notebooks against MySQL..."
+            python scripts/execute_notebooks.py --backend mysql
         else
-            echo "Unexpected mode..."
+            echo "Unexpected mode. Use: LIVE, BUILD, EXECUTE, or EXECUTE_PG"
             exit 1
         fi
 
 volumes:
   mysql_data:
     # Persistent storage for tutorial databases
     # Use `docker compose down -v` to reset
+  postgres_data:
+    # Persistent storage for PostgreSQL databases
   minio_data:
     # Persistent storage for external objects

mkdocs.yaml

@@ -87,6 +87,7 @@ nav:
       - reference/index.md
       - Specifications:
           - reference/specs/index.md
+          - Database Backends: reference/specs/database-backends.md
           - Schema Definition:
               - Table Declaration: reference/specs/table-declaration.md
               - Master-Part: reference/specs/master-part.md
@@ -213,7 +214,7 @@ markdown_extensions:
       generic: true
 extra:
   generator: false # Disable watermark
-  datajoint_version: "2.0"  # DataJoint Python version this documentation covers
+  datajoint_version: "2.1"  # DataJoint Python version this documentation covers
   social:
     - icon: main/company-logo
       link: https://www.datajoint.com

scripts/drop_tutorial_schemas.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""
+Drop all tutorial schemas from the database.
+
+Usage:
+    python drop_tutorial_schemas.py --backend mysql
+    python drop_tutorial_schemas.py --backend postgresql
+
+This script drops all schemas matching the 'tutorial_%' pattern,
+preparing for a fresh notebook execution run.
+"""
+
+import argparse
+import os
+import sys
+
+
+def get_connection(backend: str):
+    """
+    Create a database connection for the specified backend.
+
+    Parameters
+    ----------
+    backend : str
+        Either 'mysql' or 'postgresql'
+
+    Returns
+    -------
+    connection
+        Database connection object
+    """
+    if backend == "postgresql":
+        import psycopg2
+
+        return psycopg2.connect(
+            host=os.environ.get("DJ_HOST", "127.0.0.1"),
+            port=int(os.environ.get("DJ_PORT", "5432")),
+            user=os.environ.get("DJ_USER", "postgres"),
+            password=os.environ.get("DJ_PASS", "tutorial"),
+            dbname="postgres",
+        )
+    else:  # mysql
+        import pymysql
+
+        return pymysql.connect(
+            host=os.environ.get("DJ_HOST", "127.0.0.1"),
+            port=int(os.environ.get("DJ_PORT", "3306")),
+            user=os.environ.get("DJ_USER", "root"),
+            password=os.environ.get("DJ_PASS", "tutorial"),
+        )
+
+
+def drop_tutorial_schemas(backend: str, dry_run: bool = False) -> list[str]:
+    """
+    Drop all tutorial schemas from the database.
+
+    Parameters
+    ----------
+    backend : str
+        Either 'mysql' or 'postgresql'
+    dry_run : bool
+        If True, only list schemas without dropping
+
+    Returns
+    -------
+    list[str]
+        List of dropped schema names
+    """
+    conn = get_connection(backend)
+    cursor = conn.cursor()
+
+    # Find tutorial schemas
+    if backend == "postgresql":
+        cursor.execute(
+            """
+            SELECT schema_name FROM information_schema.schemata
+            WHERE schema_name LIKE 'tutorial_%'
+            ORDER BY schema_name
+            """
+        )
+    else:  # mysql
+        cursor.execute(
+            """
+            SELECT schema_name FROM information_schema.schemata
+            WHERE schema_name LIKE 'tutorial_%'
+            ORDER BY schema_name
+            """
+        )
+
+    schemas = [row[0] for row in cursor.fetchall()]
+
+    if not schemas:
+        print("No tutorial schemas found.")
+        return []
+
+    print(f"Found {len(schemas)} tutorial schema(s):")
+    for schema in schemas:
+        print(f"  - {schema}")
+
+    if dry_run:
+        print("\nDry run - no schemas dropped.")
+        return schemas
+
+    print("\nDropping schemas...")
+
+    for schema in schemas:
+        if backend == "postgresql":
+            # PostgreSQL uses double quotes for identifiers
+            cursor.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')
+        else:  # mysql
+            # MySQL uses backticks for identifiers
+            cursor.execute(f"DROP DATABASE IF EXISTS `{schema}`")
+        print(f"  Dropped: {schema}")
+
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+    print(f"\nDropped {len(schemas)} schema(s).")
+    return schemas
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Drop all tutorial schemas from the database"
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["mysql", "postgresql"],
+        default="mysql",
+        help="Database backend (default: mysql)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="List schemas without dropping them",
+    )
+
+    args = parser.parse_args()
+
+    print(f"{'=' * 60}")
+    print(f"Drop Tutorial Schemas ({args.backend.upper()})")
+    print(f"{'=' * 60}\n")
+
+    try:
+        drop_tutorial_schemas(args.backend, args.dry_run)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()