tech-lab: analyst notebooks

api/alembic/versions/h1i2j3k4l5m6_add_lab_notebook_tables.py

@@ -0,0 +1,105 @@
+"""add lab notebook tables (tech lab)
+
+Revision ID: h1i2j3k4l5m6
+Revises: g7h8i9j0k1l2
+Create Date: 2026-05-07 00:00:00.000000
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import JSONB
+
+
+# revision identifiers, used by Alembic.
+revision = "h1i2j3k4l5m6"
+down_revision = "g7h8i9j0k1l2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "lab_folders",
+        sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("parent_id", sa.Integer(), nullable=True),
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("visibility", sa.String(length=16), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
+        sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(
+            ["parent_id"],
+            ["lab_folders.id"],
+            ondelete="CASCADE",
+            use_alter=True,
+        ),
+    )
+    op.create_index("ix_lab_folders_user_id", "lab_folders", ["user_id"])
+    op.create_index("ix_lab_folders_parent_id", "lab_folders", ["parent_id"])
+    op.create_index("ix_lab_folders_visibility", "lab_folders", ["visibility"])
+
+    op.create_table(
+        "lab_notebooks",
+        sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("folder_id", sa.Integer(), nullable=True),
+        sa.Column("visibility", sa.String(length=16), nullable=False),
+        sa.Column("name", sa.String(length=255), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("source", sa.Text(), nullable=False, server_default=""),
+        sa.Column("cell_outputs", JSONB, nullable=False, server_default="{}"),
+        sa.Column("last_executed_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
+        sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(
+            ["folder_id"], ["lab_folders.id"], ondelete="SET NULL"
+        ),
+    )
+    op.create_index("ix_lab_notebooks_user_id", "lab_notebooks", ["user_id"])
+    op.create_index("ix_lab_notebooks_folder_id", "lab_notebooks", ["folder_id"])
+    op.create_index("ix_lab_notebooks_visibility", "lab_notebooks", ["visibility"])
+
+    op.create_table(
+        "lab_executions",
+        sa.Column("id", sa.Integer, autoincrement=True, primary_key=True),
+        sa.Column("notebook_id", sa.Integer(), nullable=False),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("cell_id", sa.String(length=64), nullable=False),
+        sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("error", sa.Text(), nullable=True),
+        sa.Column("outputs", JSONB, nullable=False, server_default="[]"),
+        sa.Column("execution_count", sa.Integer(), nullable=True),
+        sa.Column("celery_task_id", sa.String(length=128), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["notebook_id"], ["lab_notebooks.id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
+    )
+    op.create_index(
+        "ix_lab_executions_notebook_id_started_at",
+        "lab_executions",
+        ["notebook_id", sa.text("started_at DESC")],
+    )
+    op.create_index("ix_lab_executions_status", "lab_executions", ["status"])
+
+
+def downgrade():
+    op.drop_index("ix_lab_executions_status", table_name="lab_executions")
+    op.drop_index(
+        "ix_lab_executions_notebook_id_started_at", table_name="lab_executions"
+    )
+    op.drop_table("lab_executions")
+    op.drop_index("ix_lab_notebooks_visibility", table_name="lab_notebooks")
+    op.drop_index("ix_lab_notebooks_folder_id", table_name="lab_notebooks")
+    op.drop_index("ix_lab_notebooks_user_id", table_name="lab_notebooks")
+    op.drop_table("lab_notebooks")
+    op.drop_index("ix_lab_folders_visibility", table_name="lab_folders")
+    op.drop_index("ix_lab_folders_parent_id", table_name="lab_folders")
+    op.drop_index("ix_lab_folders_user_id", table_name="lab_folders")
+    op.drop_table("lab_folders")

api/app/auth/auth.py

@@ -121,6 +121,11 @@ class TokenData(BaseModel):
         "reactor:update": "Update Tech Lab reactor scripts.",
         "reactor:delete": "Delete Tech Lab reactor scripts.",
         "reactor:run": "Run a Tech Lab reactor script test execution.",
+        "lab:create": "Create Tech Lab notebooks and folders.",
+        "lab:read": "Read Tech Lab notebooks, folders, and executions.",
+        "lab:update": "Update Tech Lab notebooks and folders.",
+        "lab:delete": "Delete Tech Lab notebooks and folders.",
+        "lab:run": "Execute cells in a Tech Lab notebook.",
         "notifications:read": "Read notifications.",
         "notifications:update": "Update notifications.",
         "user_settings:read": "Read user settings.",

api/app/cli/__main__.py

@@ -1,11 +1,17 @@
+import json
+from datetime import datetime, timezone
+from pathlib import Path
 from typing import Optional
 
 import typer
+
 from app.database import SessionLocal
+from app.models import lab as lab_models
 from app.repositories import organisations as organisations_repository
 from app.repositories import users as user_repository
 from app.schemas import organisations as organisation_schemas
 from app.schemas import user as user_schemas
+from app.services.tech_lab.lab import nbformat_io
 from app.worker import tasks
 
 app = typer.Typer()
@@ -78,5 +84,90 @@ def load_taxonomies(user_id: Optional[int] = None):
     tasks.load_taxonomies.delay()
 
 
+@app.command()
+def seed_lab_library(
+    owner_email: str = typer.Option(
+        ...,
+        "--owner-email",
+        help="Email of the user that will own the seeded notebooks",
+    ),
+    directory: Path = typer.Option(
+        Path("/code/lab_library"),
+        "--directory",
+        help="Directory containing .ipynb files to seed",
+    ),
+):
+    """Upsert notebooks from .ipynb files into the Library section.
+
+    Each file becomes a notebook with visibility=library, named after the file
+    stem. Re-running the command updates the source of existing library
+    notebooks (matched by name) without touching personal forks.
+    """
+    if not directory.exists() or not directory.is_dir():
+        typer.echo(f"Directory not found: {directory}", err=True)
+        raise typer.Exit(code=1)
+
+    db = SessionLocal()
+    owner = user_repository.get_user_by_email(db, email=owner_email)
+    if owner is None:
+        typer.echo(f"User not found: {owner_email}", err=True)
+        raise typer.Exit(code=1)
+
+    files = sorted(directory.glob("*.ipynb"))
+    if not files:
+        typer.echo(f"No .ipynb files in {directory}")
+        return
+
+    created = updated = 0
+    for path in files:
+        try:
+            blob = json.loads(path.read_text(encoding="utf-8"))
+        except (UnicodeDecodeError, json.JSONDecodeError) as e:
+            typer.echo(f"Skipping {path.name}: invalid JSON ({e})", err=True)
+            continue
+        if not isinstance(blob, dict) or "cells" not in blob:
+            typer.echo(f"Skipping {path.name}: not a valid .ipynb", err=True)
+            continue
+        source, fallback_name = nbformat_io.from_nbformat(blob)
+        # Prefer the file stem so re-running with renamed metadata still
+        # matches the same library entry.
+        name = path.stem
+        description = ((blob.get("metadata") or {}).get("misp_workbench") or {}).get(
+            "description"
+        ) or fallback_name
+
+        existing = (
+            db.query(lab_models.LabNotebook)
+            .filter(
+                lab_models.LabNotebook.visibility == "library",
+                lab_models.LabNotebook.name == name,
+            )
+            .first()
+        )
+        now = datetime.now(timezone.utc)
+        if existing is None:
+            row = lab_models.LabNotebook(
+                user_id=owner.id,
+                folder_id=None,
+                visibility="library",
+                name=name,
+                description=description,
+                source=source,
+                cell_outputs={},
+                created_at=now,
+            )
+            db.add(row)
+            created += 1
+        else:
+            existing.source = source
+            existing.description = description
+            existing.cell_outputs = {}
+            existing.last_executed_at = None
+            existing.updated_at = now
+            updated += 1
+    db.commit()
+    typer.echo(f"Library seeded: {created} created, {updated} updated.")
+
+
 if __name__ == "__main__":
     app()

api/app/main.py

@@ -11,6 +11,7 @@
     feeds,
     galaxies,
     hunts,
+    lab,
     mcp,
     modules,
     object_templates,
@@ -153,6 +154,9 @@
 # Tech Lab — Reactor Scripts
 app.include_router(reactor.router, tags=["Tech Lab / Reactor"])
 
+# Tech Lab — Notebooks
+app.include_router(lab.router, tags=["Tech Lab / Notebooks"])
+
 # MCP config endpoint (must be registered before the /mcp mount)
 app.include_router(mcp.router, tags=["MCP"])
 

api/app/models/__init__.py

@@ -3,6 +3,7 @@
 from app.models.hunt import Hunt  # noqa
 from app.models.module import ModuleSettings  # noqa
 from app.models.organisation import Organisation  # noqa
+from app.models.lab import LabExecution, LabFolder, LabNotebook  # noqa
 from app.models.reactor import ReactorRun, ReactorScript  # noqa
 from app.models.role import Role  # noqa
 from app.models.server import Server  # noqa

api/app/models/lab.py

@@ -0,0 +1,67 @@
+from app.database import Base
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text
+from sqlalchemy.dialects.postgresql import JSONB
+
+
+class LabFolder(Base):
+    __tablename__ = "lab_folders"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(
+        Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    parent_id = Column(
+        Integer,
+        ForeignKey("lab_folders.id", ondelete="CASCADE"),
+        nullable=True,
+    )
+    name = Column(String(255), nullable=False)
+    visibility = Column(String(16), nullable=False)
+    created_at = Column(DateTime(timezone=True), nullable=False)
+    updated_at = Column(DateTime(timezone=True), nullable=True)
+
+
+class LabNotebook(Base):
+    __tablename__ = "lab_notebooks"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(
+        Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    folder_id = Column(
+        Integer,
+        ForeignKey("lab_folders.id", ondelete="SET NULL"),
+        nullable=True,
+    )
+    visibility = Column(String(16), nullable=False)
+    name = Column(String(255), nullable=False)
+    description = Column(Text, nullable=True)
+    source = Column(Text, nullable=False, default="", server_default="")
+    cell_outputs = Column(JSONB, nullable=False, default=dict, server_default="{}")
+    last_executed_at = Column(DateTime(timezone=True), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False)
+    updated_at = Column(DateTime(timezone=True), nullable=True)
+
+
+class LabExecution(Base):
+    __tablename__ = "lab_executions"
+
+    id = Column(Integer, primary_key=True, index=True)
+    notebook_id = Column(
+        Integer,
+        ForeignKey("lab_notebooks.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    user_id = Column(
+        Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False
+    )
+    cell_id = Column(String(64), nullable=False)
+    status = Column(String(32), nullable=False, default="queued")
+    started_at = Column(DateTime(timezone=True), nullable=True)
+    finished_at = Column(DateTime(timezone=True), nullable=True)
+    error = Column(Text, nullable=True)
+    outputs = Column(JSONB, nullable=False, default=list, server_default="[]")
+    execution_count = Column(Integer, nullable=True)
+    celery_task_id = Column(String(128), nullable=True)
+    created_at = Column(DateTime(timezone=True), nullable=False)