diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/pipelines/README.md b/mpcontribs-lux/mpcontribs/lux/projects/alab/pipelines/README.md new file mode 100644 index 000000000..fda81bfe2 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/pipelines/README.md @@ -0,0 +1,13 @@ +# A-Lab Pipelines + +Pipeline code for the A-Lab project is maintained in a separate repository: + +**Repository:** [alab-pipeline](https://github.com/SilenNaihin/alab-pipeline.git) + +## Getting Started + +To clone the pipeline repository: + +```bash +git clone https://github.com/SilenNaihin/alab-pipeline.git +``` diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py new file mode 100644 index 000000000..e30cfc64a --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/__init__.py @@ -0,0 +1,36 @@ +""" +A-Lab Pydantic Schemas + +This package contains Pydantic schemas for all A-Lab parquet tables. +These schemas are the source of truth for data validation. + +Each schema corresponds to one parquet file. +Integrates team's validation patterns (constraints, Literal types) from results_schema.py. +""" + +from .base import ExcludeFromUpload +from .experiments import Experiment +from .experiment_elements import ExperimentElement +from .powder_doses import PowderDose, PowderItem +from .temperature_logs import TemperatureLog, TemperatureLogEntry +from .workflow_tasks import WorkflowTask +from .xrd_data_points import XRDDataPoint +from .xrd_refinements import XRDRefinement +from .xrd_phases import XRDPhase + +__all__ = [ + # Base + "ExcludeFromUpload", + # Parquet table schemas (one per .parquet file) + "Experiment", + "ExperimentElement", + "PowderDose", + "PowderItem", + "TemperatureLogEntry", + "TemperatureLog", + "WorkflowTask", + "XRDDataPoint", + "XRDRefinement", + "XRDPhase", +] + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py new file mode 100644 index 000000000..5bae46d12 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/base.py @@ -0,0 +1,69 @@ +""" +Base utilities for A-Lab Pydantic schemas. + +Provides common types, validators, and field utilities. +""" + +from typing import Any, Dict +from pydantic import Field + + +def ExcludeFromUpload( + default: Any = None, + description: str = "", + **kwargs +) -> Any: + """ + Field that should NOT be uploaded to MPContribs. + + Use this for sensitive data that must remain private until publication. + Examples: weight_collected, mass measurements that are embargoed. + + Usage: + weight_collected: float | None = ExcludeFromUpload( + description="Weight of powder collected (embargoed)" + ) + """ + return Field( + default=default, + description=description, + json_schema_extra={"exclude_from_upload": True}, + **kwargs + ) + + +def get_uploadable_fields(model_class) -> list[str]: + """ + Get list of fields that should be uploaded to MPContribs. + + Args: + model_class: Pydantic model class + + Returns: + List of field names that are NOT marked with exclude_from_upload + """ + uploadable = [] + for field_name, field_info in model_class.model_fields.items(): + extra = field_info.json_schema_extra or {} + if not extra.get("exclude_from_upload", False): + uploadable.append(field_name) + return uploadable + + +def get_excluded_fields(model_class) -> list[str]: + """ + Get list of fields that should NOT be uploaded to MPContribs. + + Args: + model_class: Pydantic model class + + Returns: + List of field names that ARE marked with exclude_from_upload + """ + excluded = [] + for field_name, field_info in model_class.model_fields.items(): + extra = field_info.json_schema_extra or {} + if extra.get("exclude_from_upload", False): + excluded.append(field_name) + return excluded + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py new file mode 100644 index 000000000..ad90a56dd --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiment_elements.py @@ -0,0 +1,30 @@ +""" +Experiment Elements Schema + +Elements present in each experiment (1:N relationship). +Maps to: experiment_elements.parquet +""" + +from pydantic import BaseModel, Field + + +class ExperimentElement(BaseModel, extra="forbid"): + """ + Element present in an experiment. + + Each experiment can have multiple elements (1:N relationship). + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + element_symbol: str = Field( + description="Element symbol (e.g., Na, Mg, O)" + ) + + target_atomic_percent: float | None = Field( + default=None, + description="Target atomic percentage of this element" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py new file mode 100644 index 000000000..5ca1ffdd3 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/experiments.py @@ -0,0 +1,227 @@ +""" +Experiment Schema (Consolidated) + +This is the main experiment table with ALL 1:1 data merged. +Contains ~45 columns from: experiments + heating + recovery + xrd + finalization + dosing. + +Maps to: experiments.parquet +""" + +from datetime import datetime +from typing import Literal +from pydantic import BaseModel, Field + +# Import ExcludeFromUpload utility +try: + from .base import ExcludeFromUpload +except ImportError: + # When imported dynamically, use absolute import + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).parent)) + from base import ExcludeFromUpload + + +class Experiment(BaseModel, extra="forbid"): + """ + Main experiment schema with all 1:1 data consolidated. + + This is the primary table - one row per experiment. + All related 1:1 data (heating, recovery, xrd, finalization, dosing) is merged here. + """ + + # === Core experiment fields === + experiment_id: str = Field( + description="Unique experiment identifier (MongoDB _id)" + ) + + name: str = Field( + description="Experiment name (e.g., NSC_249, MINES_12)" + ) + + experiment_type: str = Field( + description="Root experiment type (NSC, Na, PG, MINES, TRI)" + ) + + experiment_subgroup: str | None = Field( + default=None, + description="Experiment subgroup (e.g., NSC_249, Na_123)" + ) + + target_formula: str = Field( + description="Target chemical formula" + ) + + last_updated: datetime = Field( + description="Last modification timestamp" + ) + + status: Literal["completed", "error", "active", "unknown"] = Field( + description="Workflow status" + ) + + notes: str | None = Field( + default=None, + description="Optional notes about the experiment" + ) + + # === Heating fields (prefix: heating_) === + heating_method: Literal["standard", "atmosphere", "manual", "none"] | None = Field( + default=None, + description="Heating method used" + ) + + heating_temperature: float | None = Field( + default=None, + description="Target heating temperature in °C" + ) + + heating_time: float | None = Field( + default=None, + description="Heating duration in minutes" + ) + + heating_cooling_rate: float | None = Field( + default=None, + description="Cooling rate in °C/min" + ) + + heating_atmosphere: str | None = Field( + default=None, + description="Atmosphere used during heating (e.g., N2, Ar, Air)" + ) + + heating_flow_rate_ml_min: float | None = Field( + default=None, + description="Gas flow rate during heating in mL/min" + ) + + heating_low_temp_calcination: bool | None = Field( + default=None, + description="Whether low temperature calcination was used" + ) + + # === Recovery fields (prefix: recovery_) === + recovery_total_dosed_mass_mg: float | None = Field( + default=None, + description="Total mass of all powders dosed in mg" + ) + + # EXCLUDED FROM UPLOAD per team request + recovery_weight_collected_mg: float | None = ExcludeFromUpload( + description="Weight of powder collected after heating in mg (EMBARGOED)" + ) + + recovery_yield_percent: float | None = Field( + default=None, + description="Recovery yield (collected / dosed * 100)" + ) + + recovery_initial_crucible_weight_mg: float | None = Field( + default=None, + description="Initial crucible weight before experiment in mg" + ) + + recovery_failure_classification: str | None = Field( + default=None, + description="Classification of any failure during recovery" + ) + + # === XRD measurement fields (prefix: xrd_) === + xrd_sampleid_in_aeris: str | None = Field( + default=None, + description="Sample ID in Aeris XRD system" + ) + + xrd_holder_index: int | None = Field( + default=None, + description="XRD sample holder position index" + ) + + # EXCLUDED FROM UPLOAD per team request + xrd_total_mass_dispensed_mg: float | None = ExcludeFromUpload( + description="Mass dispensed for XRD measurement in mg (EMBARGOED)" + ) + + xrd_met_target_mass: bool | None = Field( + default=None, + description="Whether target mass was achieved for XRD" + ) + + # === Finalization fields (prefix: finalization_) === + finalization_decoded_sample_id: str | None = Field( + default=None, + description="Decoded sample ID from barcode" + ) + + finalization_successful_labeling: bool | None = Field( + default=None, + description="Whether sample was successfully labeled" + ) + + finalization_storage_location: str | None = Field( + default=None, + description="Final storage location of sample" + ) + + # === Dosing fields (prefix: dosing_) === + dosing_crucible_position: int | None = Field( + default=None, + description="Crucible position in rack", + ge=1, + le=4 + ) + + dosing_crucible_sub_rack: Literal["SubRackA", "SubRackB", "SubRackC", "SubRackD"] | None = Field( + default=None, + description="Sub-rack identifier" + ) + + dosing_mixing_pot_position: int | None = Field( + default=None, + description="Mixing pot position", + ge=1, + le=16 + ) + + dosing_ethanol_dispense_volume: int | None = Field( + default=None, + description="Volume of ethanol dispensed in µL", + ge=0 + ) + + dosing_target_transfer_volume: int | None = Field( + default=None, + description="Target transfer volume in µL", + ge=0 + ) + + dosing_actual_transfer_mass: float | None = Field( + default=None, + description="Actual mass transferred in g", + ge=0 + ) + + dosing_dac_duration: int | None = Field( + default=None, + description="DAC duration in seconds", + ge=0 + ) + + dosing_dac_speed: int | None = Field( + default=None, + description="DAC rotation speed in rpm", + ge=0 + ) + + dosing_actual_heat_duration: int | None = Field( + default=None, + description="Actual heating duration during dosing in seconds", + ge=0 + ) + + dosing_end_reason: str | None = Field( + default=None, + description="Reason for ending dosing session" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py new file mode 100644 index 000000000..f2f376f32 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/powder_doses.py @@ -0,0 +1,76 @@ +""" +Powder Doses Schema + +Individual powder doses for each experiment (1:N relationship). +Maps to: powder_doses.parquet + +NOTE: This is distinct from PowderDosingSampleResult which is the raw +MongoDB structure. This is the flattened parquet schema. +""" + +from pydantic import BaseModel, Field + + +class PowderItem(BaseModel, extra="forbid"): + """Schema for a powder item within a dose.""" + + PowderName: str | None = Field( + default=None, + description="Name of the powder" + ) + TargetMass: float | None = Field( + default=None, + description="Target mass in mg", + ge=0 + ) + Doses: list[dict] | None = Field( + default=None, + description="List of individual doses" + ) + + +class PowderDose(BaseModel, extra="forbid"): + """ + Individual powder dose event. + + Each experiment can have multiple powder doses (1:N relationship). + This is the flattened representation from the nested Powders[].Doses[] structure. + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + powder_name: str = Field( + description="Name of the powder material" + ) + + target_mass: float = Field( + description="Target mass in mg", + ge=0 + ) + + actual_mass: float = Field( + description="Actual mass dispensed in mg", + ge=0 + ) + + accuracy_percent: float = Field( + description="Dosing accuracy (actual/target * 100)" + ) + + dose_sequence: int = Field( + description="Sequence number within the experiment", + ge=0 + ) + + head_position: int | None = Field( + default=None, + description="Dispenser head position" + ) + + dose_timestamp: str | None = Field( + default=None, + description="Timestamp of the dose event" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py new file mode 100644 index 000000000..980c7af9d --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/temperature_logs.py @@ -0,0 +1,51 @@ +""" +Temperature Logs Schema + +Temperature readings during heating (1:N relationship). +Maps to: temperature_logs.parquet + +NOTE: This is the flattened parquet schema. The raw MongoDB schema uses +nested arrays (time_minutes[], temperature_celsius[]). +""" + +from pydantic import BaseModel, Field + + +class TemperatureLogEntry(BaseModel, extra="forbid"): + """ + Single temperature reading. + + Each experiment can have thousands of temperature log entries (1:N relationship). + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + sequence_number: int = Field( + description="Sequence number in the time series", + ge=0 + ) + + time_minutes: float = Field( + description="Time elapsed since heating start in minutes" + ) + + temperature_celsius: float = Field( + description="Temperature reading in °C" + ) + + +# For backward compatibility with nested structure +class TemperatureLog(BaseModel, extra="forbid"): + """Temperature log with array structure (MongoDB format).""" + + time_minutes: list[float] | None = Field( + default=None, + description="Time elapsed in minutes since the start of the heating process" + ) + temperature_celsius: list[float] | None = Field( + default=None, + description="Temperature of the sample in Celsius" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py new file mode 100644 index 000000000..7b6c6a1d2 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/workflow_tasks.py @@ -0,0 +1,55 @@ +""" +Workflow Tasks Schema + +Task execution history for each experiment (1:N relationship). +Maps to: workflow_tasks.parquet +""" + +from datetime import datetime +from typing import Literal +from pydantic import BaseModel, Field + + +class WorkflowTask(BaseModel, extra="forbid"): + """ + Single task in the experiment workflow. + + Each experiment has multiple tasks (1:N relationship). + Tasks include: PowderDosing, Heating, RecoverPowder, Diffraction, Ending, etc. + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + task_id: str = Field( + description="Unique task identifier" + ) + + task_type: str = Field( + description="Type of task (PowderDosing, Heating, RecoverPowder, Diffraction, Ending)" + ) + + status: Literal["completed", "error", "pending", "running"] = Field( + description="Task execution status" + ) + + created_at: datetime = Field( + description="When the task was created" + ) + + started_at: datetime | None = Field( + default=None, + description="When the task started execution" + ) + + completed_at: datetime | None = Field( + default=None, + description="When the task completed" + ) + + message: str | None = Field( + default=None, + description="Task result message or error" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py new file mode 100644 index 000000000..4db376e72 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_data_points.py @@ -0,0 +1,38 @@ +""" +XRD Data Points Schema + +Raw XRD diffraction pattern data (1:N relationship). +Maps to: xrd_data_points.parquet + +NOTE: This is the flattened parquet schema. The raw MongoDB schema uses +nested arrays (twotheta[], counts[]). +""" + +from pydantic import BaseModel, Field + + +class XRDDataPoint(BaseModel, extra="forbid"): + """ + Single XRD data point (2θ, counts). + + Each experiment can have ~8000 data points (1:N relationship). + This is the flattened representation from twotheta[] and counts[] arrays. + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + point_index: int = Field( + description="Index in the diffraction pattern", + ge=0 + ) + + twotheta: float = Field( + description="2θ angle in degrees" + ) + + counts: float = Field( + description="Intensity counts at this angle" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py new file mode 100644 index 000000000..dccd7ef92 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_phases.py @@ -0,0 +1,64 @@ +""" +XRD Phases Schema + +Identified crystal phases from DARA analysis (1:N relationship with experiments). +Maps to: xrd_phases.parquet +""" + +from pydantic import BaseModel, Field + + +class XRDPhase(BaseModel, extra="forbid"): + """ + Identified crystal phase from XRD refinement. + + Each experiment can have multiple phases (1:N relationship). + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + experiment_name: str = Field( + description="Experiment name (for convenience)" + ) + + phase_name: str = Field( + description="Name of the identified phase" + ) + + spacegroup: str = Field( + description="Space group of the phase (e.g., Fm-3m)" + ) + + weight_fraction: float = Field( + description="Weight fraction of this phase (0-1)", + ge=0, + le=1 + ) + + weight_fraction_error: float | None = Field( + default=None, + description="Error in weight fraction" + ) + + lattice_a_nm: float | None = Field( + default=None, + description="Lattice parameter a in nm" + ) + + lattice_b_nm: float | None = Field( + default=None, + description="Lattice parameter b in nm" + ) + + lattice_c_nm: float | None = Field( + default=None, + description="Lattice parameter c in nm" + ) + + r_phase: float | None = Field( + default=None, + description="R-factor for this phase" + ) + diff --git a/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py new file mode 100644 index 000000000..9087b1136 --- /dev/null +++ b/mpcontribs-lux/mpcontribs/lux/projects/alab/schemas/xrd_refinements.py @@ -0,0 +1,89 @@ +""" +XRD Refinements Schema + +DARA Rietveld refinement analysis results (1:1 relationship with experiments). +Maps to: xrd_refinements.parquet +""" + +from pydantic import BaseModel, Field + + +class XRDRefinement(BaseModel, extra="forbid"): + """ + XRD Rietveld refinement result from DARA analysis. + + One result per experiment (1:1 relationship). + """ + + experiment_id: str = Field( + description="Reference to parent experiment" + ) + + experiment_name: str = Field( + description="Experiment name (for convenience)" + ) + + success: bool = Field( + description="Whether the refinement was successful" + ) + + error: str | None = Field( + default=None, + description="Error message if refinement failed" + ) + + error_type: str | None = Field( + default=None, + description="Classification of error type" + ) + + rwp: float | None = Field( + default=None, + description="Weighted profile R-factor (goodness of fit)" + ) + + rp: float | None = Field( + default=None, + description="Profile R-factor" + ) + + rexp: float | None = Field( + default=None, + description="Expected R-factor" + ) + + num_phases: int = Field( + default=0, + description="Number of phases identified" + ) + + chemical_system: str | None = Field( + default=None, + description="Chemical system (e.g., Na-Mg-P-O)" + ) + + target_formula: str | None = Field( + default=None, + description="Target formula" + ) + + analysis_timestamp: str | None = Field( + default=None, + description="When the analysis was performed" + ) + + mode: str | None = Field( + default=None, + description="Analysis mode" + ) + + wmin: int | None = Field( + default=None, + description="Minimum 2θ angle for refinement" + ) + + wmax: int | None = Field( + default=None, + description="Maximum 2θ angle for refinement" + ) +