Add db components from ispyb for processing outputs

src/murfey/cli/create_db.py

@@ -1,25 +1,41 @@
 import argparse
 
 from murfey.util.db import clear, setup
+from murfey.util.processing_db import (
+    clear as processing_db_clear,
+    setup as processing_db_setup,
+)
 
 
 def run():
     parser = argparse.ArgumentParser(
         description="Generate the necessary tables for the Murfey database"
     )
-
     parser.add_argument(
         "--no-clear",
         dest="clear",
         default=True,
         action="store_false",
         help="Do not clear current database tables before creating specified tables",
     )
+    parser.add_argument(
+        "--include-processing",
+        dest="processing",
+        default=True,
+        action="store_true",
+        help="Include processing results tables (MotionCorr, CTF, etc)",
+    )
 
     args = parser.parse_args()
 
     from murfey.server.murfey_db import url
 
-    if args.clear:
+    if args.clear and args.processing:
+        processing_db_clear(url())
+    elif args.clear:
         clear(url())
-    setup(url())
+
+    if args.processing:
+        processing_db_setup(url())
+    else:
+        setup(url())

src/murfey/server/api/workflow.py

@@ -478,6 +478,7 @@ async def request_spa_preprocessing(
             db.add(feedback_params)
         movie = Movie(
             murfey_id=murfey_ids[0],
+            data_collection_id=detached_ids[1],
             path=proc_file.path,
             image_number=proc_file.image_number,
             tag=proc_file.tag,
@@ -730,6 +731,17 @@ async def request_tomography_preprocessing(
                 0
             ].eer_fractionation_file
 
+        movie = Movie(
+            murfey_id=murfey_ids[0],
+            data_collection_id=dcid,
+            path=proc_file.path,
+            image_number=proc_file.image_number,
+            tag=proc_file.tag,
+        )
+        db.add(movie)
+        db.commit()
+        db.close()
+
         zocalo_message: dict = {
             "recipes": [recipe_name],
             "parameters": {

src/murfey/server/feedback.py

@@ -1428,6 +1428,7 @@ def _flush_tomography_preprocessing(message: dict, _db):
             p.parent.mkdir(parents=True)
         movie = db.Movie(
             murfey_id=murfey_ids[0],
+            data_collection_id=detached_ids[1],
             path=f.file_path,
             image_number=f.image_number,
             tag=f.tag,

src/murfey/util/db.py

@@ -4,14 +4,26 @@
 """
 
 from datetime import datetime
-from typing import List, Optional
+from typing import TYPE_CHECKING, List, Optional
 
 import sqlalchemy
 from sqlmodel import Field, Relationship, SQLModel, create_engine
 
+if TYPE_CHECKING:
+    from murfey.util.processing_db import (
+        CTF,
+        MotionCorrection,
+        ParticleClassificationGroup,
+        ParticlePicker,
+        RelativeIceThickness,
+        TiltImageAlignment,
+        Tomogram,
+    )
+
 """
 GENERAL
 """
+mapper_registry = sqlalchemy.orm.registry()
 
 
 class MurfeyUser(SQLModel, table=True):  # type: ignore
@@ -414,7 +426,7 @@
     atlas_pixel_size: Optional[float] = None
     atlas: str = ""
     sample: Optional[int] = None
-    session: Optional[Session] = Relationship(back_populates="data_collection_groups")
+    session: Optional["Session"] = Relationship(back_populates="data_collection_groups")
     data_collections: List["DataCollection"] = Relationship(
         back_populates="data_collection_group",
         sa_relationship_kwargs={"cascade": "delete"},
@@ -433,6 +445,14 @@
             sa_relationship_kwargs={"cascade": "delete"},
         )
     )
+    grid_squares: Optional[List["GridSquare"]] = Relationship(
+        back_populates="data_collection_group",
+        sa_relationship_kwargs={"cascade": "delete"},
+    )
+    search_maps: Optional[List["SearchMap"]] = Relationship(
+        back_populates="data_collection_group",
+        sa_relationship_kwargs={"cascade": "delete"},
+    )
 
 
 class NotificationParameter(SQLModel, table=True):  # type: ignore
@@ -472,6 +492,15 @@
     processing_jobs: List["ProcessingJob"] = Relationship(
         back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"}
     )
+    movies: List["Movie"] = Relationship(
+        back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"}
+    )
+    motion_correction: Optional[List["MotionCorrection"]] = Relationship(
+        back_populates="data_collection"
+    )
+    tomogram: Optional[List["Tomogram"]] = Relationship(
+        back_populates="data_collection"
+    )
 
 
 class ProcessingJob(SQLModel, table=True):  # type: ignore
@@ -573,6 +602,22 @@
     murfey_ids: List["MurfeyLedger"] = Relationship(
         back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"}
     )
+    motion_correction: Optional[List["MotionCorrection"]] = Relationship(
+        back_populates="data_collection"
+    )
+    tomogram: Optional[List["Tomogram"]] = Relationship(
+        back_populates="auto_proc_program"
+    )
+    ctf: Optional[List["CTF"]] = Relationship(back_populates="auto_proc_program")
+    particle_picker: Optional[List["ParticlePicker"]] = Relationship(
+        back_populates="auto_proc_program"
+    )
+    relative_ice_thickness: Optional[List["RelativeIceThickness"]] = Relationship(
+        back_populates="auto_proc_program"
+    )
+    particle_classification_group: Optional[List["ParticleClassificationGroup"]] = (
+        Relationship(back_populates="auto_proc_program")
+    )
 
 
 class MurfeyLedger(SQLModel, table=True):  # type: ignore
@@ -631,6 +676,17 @@
     foil_holes: List["FoilHole"] = Relationship(
         back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"}
     )
+    atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id")
+    scaled_pixel_size: Optional[float] = None
+    pixel_location_x: Optional[int] = None
+    pixel_location_y: Optional[int] = None
+    height: Optional[int] = None
+    width: Optional[int] = None
+    angle: Optional[float] = None
+    quality_indicator: Optional[float] = None
+    data_collection_group: Optional["DataCollectionGroup"] = Relationship(
+        back_populates="grid_squares"
+    )
 
 
 class FoilHole(SQLModel, table=True):  # type: ignore
@@ -656,6 +712,11 @@
     preprocess_stashes: List[PreprocessStash] = Relationship(
         back_populates="foil_hole", sa_relationship_kwargs={"cascade": "delete"}
     )
+    scaled_pixel_size: Optional[float] = None
+    pixel_location_x: Optional[int] = None
+    pixel_location_y: Optional[int] = None
+    diameter: Optional[int] = None
+    quality_indicator: Optional[float] = None
 
 
 class SearchMap(SQLModel, table=True):  # type: ignore
@@ -688,17 +749,37 @@
     tilt_series: List["TiltSeries"] = Relationship(
         back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"}
     )
+    atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id")
+    scaled_pixel_size: Optional[float] = None
+    pixel_location_x: Optional[int] = None
+    pixel_location_y: Optional[int] = None
+    scaled_height: Optional[int] = None
+    scaled_width: Optional[int] = None
+    angle: Optional[float] = None
+    quality_indicator: Optional[float] = None
+    data_collection_group: Optional["DataCollectionGroup"] = Relationship(
+        back_populates="search_maps"
+    )
+    tomogram: Optional[List["Tomogram"]] = Relationship(back_populates="search_map")
 
 
 class Movie(SQLModel, table=True):  # type: ignore
     murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id")
+    data_collection_id: Optional[int] = Field(foreign_key="datacollection.id")
     foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None)
     path: str
     image_number: int
     tag: str
     preprocessed: bool = False
     murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies")
+    data_collection: Optional["DataCollection"] = Relationship(back_populates="movies")
     foil_hole: Optional[FoilHole] = Relationship(back_populates="movies")
+    motion_correction: Optional[List["MotionCorrection"]] = Relationship(
+        back_populates="movie"
+    )
+    tilt_image_alignment: Optional[List["TiltImageAlignment"]] = Relationship(
+        back_populates="movie"
+    )
 
 
 class CtfParameters(SQLModel, table=True):  # type: ignore