Skip to content

Commit e166326

Browse files
Merge pre/v2.1 into docs/singleton-tables
Combines: - Singleton tables documentation - PostgreSQL compatibility fixes - Decimal types for human-entered fields - Secondary attributes terminology standardization - Notebook execution script Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2 parents 4c01f17 + e3376bd commit e166326

64 files changed

Lines changed: 4777 additions & 3589 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,6 @@ temp*
88

99
# Generated documentation files
1010
src/llms-full.txt
11-
site/llms-full.txt
11+
site/llms-full.txt
12+
dj_local_conf.json
13+
__pycache__/

docker-compose.yaml

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
#
33
# MODE="LIVE" docker compose up --build # Live dev server at http://localhost:8000/
44
# MODE="BUILD" docker compose up --build # Build static site
5-
# docker compose run docs jupyter nbconvert --execute ... # Execute notebooks
5+
# MODE="EXECUTE" docker compose up --build # Execute notebooks against MySQL
6+
# MODE="EXECUTE_PG" docker compose up --build # Execute notebooks against PostgreSQL
67
#
78
services:
89
mysql:
@@ -22,14 +23,15 @@ services:
2223
postgres:
2324
image: postgres:15
2425
environment:
25-
POSTGRES_USER: datajoint
26+
POSTGRES_USER: postgres
2627
POSTGRES_PASSWORD: tutorial
28+
POSTGRES_DB: datajoint
2729
ports:
2830
- "5432:5432"
2931
volumes:
3032
- postgres_data:/var/lib/postgresql/data
3133
healthcheck:
32-
test: ["CMD-SHELL", "pg_isready -U datajoint"]
34+
test: ["CMD-SHELL", "pg_isready -U postgres"]
3335
interval: 5s
3436
timeout: 5s
3537
retries: 10
@@ -57,9 +59,10 @@ services:
5759
dockerfile: Dockerfile
5860
environment:
5961
- MODE
60-
- DJ_HOST=mysql
61-
- DJ_USER=root
62-
- DJ_PASS=tutorial
62+
- DJ_HOST=${DJ_HOST:-mysql}
63+
- DJ_USER=${DJ_USER:-root}
64+
- DJ_PASS=${DJ_PASS:-tutorial}
65+
- DJ_BACKEND=${DJ_BACKEND:-mysql}
6366
volumes:
6467
- .:/main
6568
- ${DJ_PYTHON_PATH:-../datajoint-python}:/datajoint-python
@@ -68,6 +71,8 @@ services:
6871
depends_on:
6972
mysql:
7073
condition: service_healthy
74+
postgres:
75+
condition: service_healthy
7176
command:
7277
- sh
7378
- -c
@@ -85,8 +90,20 @@ services:
8590
# Generate llms-full.txt with current git info
8691
python scripts/gen_llms_full.py
8792
mkdocs build --config-file ./mkdocs.yaml
93+
elif echo "$${MODE}" | grep -i execute_pg &>/dev/null; then
94+
# EXECUTE_PG mode: execute notebooks against PostgreSQL
95+
pip install -e "/datajoint-python[postgres]"
96+
pip install scikit-image pooch nbconvert
97+
echo "Executing notebooks against PostgreSQL..."
98+
python scripts/execute_notebooks.py --backend postgresql
99+
elif echo "$${MODE}" | grep -i execute &>/dev/null; then
100+
# EXECUTE mode: execute notebooks against MySQL (default)
101+
pip install -e /datajoint-python
102+
pip install scikit-image pooch nbconvert
103+
echo "Executing notebooks against MySQL..."
104+
python scripts/execute_notebooks.py --backend mysql
88105
else
89-
echo "Unexpected mode..."
106+
echo "Unexpected mode. Use: LIVE, BUILD, EXECUTE, or EXECUTE_PG"
90107
exit 1
91108
fi
92109

mkdocs.yaml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ nav:
1919
- Computation Model: explanation/computation-model.md
2020
- Queries:
2121
- Query Algebra: explanation/query-algebra.md
22+
- Semantic Matching: explanation/semantic-matching.md
2223
- Storage:
2324
- Type System: explanation/type-system.md
2425
- Custom Codecs: explanation/custom-codecs.md
@@ -51,6 +52,7 @@ nav:
5152
- how-to/index.md
5253
- Setup:
5354
- Installation: how-to/installation.md
55+
- Manage Secrets: how-to/manage-secrets.md
5456
- Configure Database: how-to/configure-database.md
5557
- Configure Object Storage: how-to/configure-storage.md
5658
- Command-Line Interface: how-to/use-cli.md
@@ -72,6 +74,8 @@ nav:
7274
- Handle Errors: how-to/handle-errors.md
7375
- Monitor Progress: how-to/monitor-progress.md
7476
- Object Storage:
77+
- Overview: how-to/object-storage-overview.md
78+
- Choose Storage Type: how-to/choose-storage-type.md
7579
- Use Object Storage: how-to/use-object-storage.md
7680
- Use NPY Codec: how-to/use-npy-codec.md
7781
- Use Plugin Codecs: how-to/use-plugin-codecs.md
@@ -86,6 +90,7 @@ nav:
8690
- reference/index.md
8791
- Specifications:
8892
- reference/specs/index.md
93+
- Database Backends: reference/specs/database-backends.md
8994
- Schema Definition:
9095
- Table Declaration: reference/specs/table-declaration.md
9196
- Master-Part: reference/specs/master-part.md
@@ -212,7 +217,7 @@ markdown_extensions:
212217
generic: true
213218
extra:
214219
generator: false # Disable watermark
215-
datajoint_version: "2.0" # DataJoint Python version this documentation covers
220+
datajoint_version: "2.1" # DataJoint Python version this documentation covers
216221
social:
217222
- icon: main/company-logo
218223
link: https://www.datajoint.com

scripts/drop_tutorial_schemas.py

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Drop all tutorial schemas from the database.
4+
5+
Usage:
6+
python drop_tutorial_schemas.py --backend mysql
7+
python drop_tutorial_schemas.py --backend postgresql
8+
9+
This script drops all schemas matching the 'tutorial_%' pattern,
10+
preparing for a fresh notebook execution run.
11+
"""
12+
13+
import argparse
14+
import os
15+
import sys
16+
17+
18+
def get_connection(backend: str):
19+
"""
20+
Create a database connection for the specified backend.
21+
22+
Parameters
23+
----------
24+
backend : str
25+
Either 'mysql' or 'postgresql'
26+
27+
Returns
28+
-------
29+
connection
30+
Database connection object
31+
"""
32+
if backend == "postgresql":
33+
import psycopg2
34+
35+
return psycopg2.connect(
36+
host=os.environ.get("DJ_HOST", "127.0.0.1"),
37+
port=int(os.environ.get("DJ_PORT", "5432")),
38+
user=os.environ.get("DJ_USER", "postgres"),
39+
password=os.environ.get("DJ_PASS", "tutorial"),
40+
dbname="postgres",
41+
)
42+
else: # mysql
43+
import pymysql
44+
45+
return pymysql.connect(
46+
host=os.environ.get("DJ_HOST", "127.0.0.1"),
47+
port=int(os.environ.get("DJ_PORT", "3306")),
48+
user=os.environ.get("DJ_USER", "root"),
49+
password=os.environ.get("DJ_PASS", "tutorial"),
50+
)
51+
52+
53+
def drop_tutorial_schemas(backend: str, dry_run: bool = False) -> list[str]:
54+
"""
55+
Drop all tutorial schemas from the database.
56+
57+
Parameters
58+
----------
59+
backend : str
60+
Either 'mysql' or 'postgresql'
61+
dry_run : bool
62+
If True, only list schemas without dropping
63+
64+
Returns
65+
-------
66+
list[str]
67+
List of dropped schema names
68+
"""
69+
conn = get_connection(backend)
70+
cursor = conn.cursor()
71+
72+
# Find tutorial schemas
73+
if backend == "postgresql":
74+
cursor.execute(
75+
"""
76+
SELECT schema_name FROM information_schema.schemata
77+
WHERE schema_name LIKE 'tutorial_%'
78+
ORDER BY schema_name
79+
"""
80+
)
81+
else: # mysql
82+
cursor.execute(
83+
"""
84+
SELECT schema_name FROM information_schema.schemata
85+
WHERE schema_name LIKE 'tutorial_%'
86+
ORDER BY schema_name
87+
"""
88+
)
89+
90+
schemas = [row[0] for row in cursor.fetchall()]
91+
92+
if not schemas:
93+
print("No tutorial schemas found.")
94+
return []
95+
96+
print(f"Found {len(schemas)} tutorial schema(s):")
97+
for schema in schemas:
98+
print(f" - {schema}")
99+
100+
if dry_run:
101+
print("\nDry run - no schemas dropped.")
102+
return schemas
103+
104+
print("\nDropping schemas...")
105+
106+
for schema in schemas:
107+
if backend == "postgresql":
108+
# PostgreSQL uses double quotes for identifiers
109+
cursor.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')
110+
else: # mysql
111+
# MySQL uses backticks for identifiers
112+
cursor.execute(f"DROP DATABASE IF EXISTS `{schema}`")
113+
print(f" Dropped: {schema}")
114+
115+
conn.commit()
116+
cursor.close()
117+
conn.close()
118+
119+
print(f"\nDropped {len(schemas)} schema(s).")
120+
return schemas
121+
122+
123+
def main():
124+
parser = argparse.ArgumentParser(
125+
description="Drop all tutorial schemas from the database"
126+
)
127+
parser.add_argument(
128+
"--backend",
129+
choices=["mysql", "postgresql"],
130+
default="mysql",
131+
help="Database backend (default: mysql)",
132+
)
133+
parser.add_argument(
134+
"--dry-run",
135+
action="store_true",
136+
help="List schemas without dropping them",
137+
)
138+
139+
args = parser.parse_args()
140+
141+
print(f"{'=' * 60}")
142+
print(f"Drop Tutorial Schemas ({args.backend.upper()})")
143+
print(f"{'=' * 60}\n")
144+
145+
try:
146+
drop_tutorial_schemas(args.backend, args.dry_run)
147+
except Exception as e:
148+
print(f"Error: {e}", file=sys.stderr)
149+
sys.exit(1)
150+
151+
152+
if __name__ == "__main__":
153+
main()

scripts/execute-notebooks.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ for dir in src/tutorials/domain/*/; do
3434
jupyter nbconvert --to notebook --execute --inplace "$dir"*.ipynb
3535
done
3636

37+
echo "Executing example tutorials..."
38+
jupyter nbconvert --to notebook --execute --inplace src/tutorials/examples/*.ipynb
39+
3740
echo "Executing how-to notebooks..."
3841
jupyter nbconvert --to notebook --execute --inplace src/how-to/*.ipynb
3942

0 commit comments

Comments
 (0)