feat: Add support/config for InstanceSegmentationMask

.infra/prod/values.yaml

@@ -2,13 +2,13 @@ stack:
   services:
     apiv2:
       image:
-        tag: sha-2e0e7d0
+        tag: sha-3a64fa9
       initContainers:
         # Install cerbos policies where the cerbos sidecar can grab them.
         - name: install-cerbos-policies
           image:
             repository: 533267185808.dkr.ecr.us-west-2.amazonaws.com/core-platform/cryoet-data-portal-backend/apiv2/apiv2
-            tag: sha-2e0e7d0
+            tag: sha-3a64fa9
           command: ["cp", "-r", "./cerbos/", "/var/policies/"]
           volumeMounts:
             - mountPath: /var/policies
@@ -24,7 +24,7 @@ stack:
         - name: run-migrations
           image:
             repository: 533267185808.dkr.ecr.us-west-2.amazonaws.com/core-platform/cryoet-data-portal-backend/apiv2/apiv2
-            tag: sha-2e0e7d0
+            tag: sha-3a64fa9
           command: ["alembic", "upgrade", "head"]
           resources:
             limits:
@@ -37,7 +37,7 @@ stack:
         - name: gen-keypair
           image:
             repository: 533267185808.dkr.ecr.us-west-2.amazonaws.com/core-platform/cryoet-data-portal-backend/apiv2/apiv2
-            tag: sha-2e0e7d0
+            tag: sha-3a64fa9
           command: ["bash", "./etc/gen_keys.sh", "/var/keys/"]
           volumeMounts:
             - mountPath: /var/keys

.infra/staging/values.yaml

@@ -2,13 +2,13 @@ stack:
   services:
     apiv2:
       image:
-        tag: sha-2e0e7d0
+        tag: sha-3a64fa9
       initContainers:
         # Install cerbos policies where the cerbos sidecar can grab them.
         - name: install-cerbos-policies
           image:
             repository: 533267185808.dkr.ecr.us-west-2.amazonaws.com/core-platform/cryoet-data-portal-backend/apiv2/apiv2
-            tag: sha-2e0e7d0
+            tag: sha-3a64fa9
           command: ["cp", "-r", "./cerbos/", "/var/policies/"]
           volumeMounts:
             - mountPath: /var/policies
@@ -24,7 +24,7 @@ stack:
         - name: run-migrations
           image:
             repository: 533267185808.dkr.ecr.us-west-2.amazonaws.com/core-platform/cryoet-data-portal-backend/apiv2/apiv2
-            tag: sha-2e0e7d0
+            tag: sha-3a64fa9
           command: ["alembic", "upgrade", "head"]
           resources:
             limits:
@@ -37,7 +37,7 @@ stack:
         - name: gen-keypair
           image:
             repository: 533267185808.dkr.ecr.us-west-2.amazonaws.com/core-platform/cryoet-data-portal-backend/apiv2/apiv2
-            tag: sha-2e0e7d0
+            tag: sha-3a64fa9
           command: ["bash", "./etc/gen_keys.sh", "/var/keys/"]
           volumeMounts:
             - mountPath: /var/keys

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  "apiv2": "1.13.2"
+  "apiv2": "1.14.0"
 }

apiv2/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [1.14.0](https://github.com/chanzuckerberg/cryoet-data-portal-backend/compare/apiv2-v1.13.2...apiv2-v1.14.0) (2025-11-26)
+
+
+### Features
+
+* add ingestion config (10453) for EMPIAR 12794 - isolated synaptic vesicles from mouse brain ([#538](https://github.com/chanzuckerberg/cryoet-data-portal-backend/issues/538)) ([584fc12](https://github.com/chanzuckerberg/cryoet-data-portal-backend/commit/584fc123e1ecbac4557cefbc299231b6f646548b))
+
+
+### Bug Fixes
+
+* neuroglancer config ingestion & db import (fixes staging tomogram viewer)  ([#545](https://github.com/chanzuckerberg/cryoet-data-portal-backend/issues/545)) ([dee5844](https://github.com/chanzuckerberg/cryoet-data-portal-backend/commit/dee584432dc596e94323ea2e79f4e8b582145317))
+
 ## [1.13.2](https://github.com/chanzuckerberg/cryoet-data-portal-backend/compare/apiv2-v1.13.1...apiv2-v1.13.2) (2025-10-30)
 
 

apiv2/db_import/common/config.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 from functools import lru_cache
 from pathlib import PurePath
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Final
 
 import sqlalchemy as sa
 from botocore.exceptions import ClientError
@@ -20,6 +20,9 @@
 
 logger = logging.getLogger("config")
 
+STAGING_URL: Final[str] = "https://files.cryoet.staging.si.czi.technology"
+PROD_URL: Final[str] = "https://files.cryoetdataportal.cziscience.com"
+
 
 class DBImportConfig:
     s3_client: S3Client
@@ -41,7 +44,7 @@ def __init__(
         self.s3fs = s3fs
         self.bucket_name = bucket_name
         self.s3_prefix = f"s3://{bucket_name}"
-        self.https_prefix = https_prefix if https_prefix else "https://files.cryoetdataportal.cziscience.com"
+        self.https_prefix = https_prefix if https_prefix else PROD_URL
         self.session = session
         self.deposition_map: dict[int, models.Deposition] = {}
 
@@ -133,11 +136,21 @@ def load_key_json(self, key: str, is_file_required: bool = True) -> dict[str, An
         Loads file matching the key value as json. If file does not exist, will raise error if is_file_required is True
         else it will return None.
         """
+        text = self.load_key_text(key, is_file_required)
+        if text is None:
+            return None
+        return json.loads(text)
+
+    def load_key_text(self, key: str, is_file_required: bool = True) -> str | None:
+        """
+        Loads file matching the key value as text. If file does not exist, will raise error if is_file_required is True
+        else it will return None.
+        """
         try:
             if key.startswith(self.bucket_name):
                 key = key[len(self.bucket_name) + 1 :]
             text = self.s3_client.get_object(Bucket=self.bucket_name, Key=key)
-            return json.loads(text["Body"].read())
+            return text["Body"].read().decode("utf-8")
         except ClientError as ex:
             if ex.response["Error"]["Code"] == "NoSuchKey" and not is_file_required:
                 logger.warning("NoSuchKey on bucket_name=%s key=%s", self.bucket_name, key)

apiv2/db_import/importers/tomogram.py

@@ -3,6 +3,7 @@
 from typing import Any
 
 from database import models
+from db_import.common.config import PROD_URL, STAGING_URL
 from db_import.common.finders import MetadataFileFinder
 from db_import.common.normalize_fields import normalize_fiducial_alignment
 from db_import.importers.base import IntegratedDBImporter, ItemDBImporter
@@ -39,6 +40,16 @@ class TomogramItem(ItemDBImporter):
         "is_visualization_default": ["is_visualization_default"],
     }
 
+    def _update_ng_urls(self, ng_config_text: str) -> str:
+        """
+        A simple function to find and replace all production URLs in the neuroglancer config with the staging URL when in the staging environment.
+        This is done to ensure that the URLs in the neuroglancer config are correct for the staging environment.
+        """
+        if self.config.https_prefix != STAGING_URL:
+            return ng_config_text
+
+        return ng_config_text.replace(PROD_URL, self.config.https_prefix)
+
     def normalize_to_unknown_str(self, value: str) -> str:
         return value.replace(" ", "_") if value else "Unknown"
 
@@ -47,9 +58,13 @@ def generate_neuroglancer_data(self, path) -> str | None:
             # Handle the case where there is no neuroglancer config file specified which is expected when
             # visualization_default is set to False.
             return None
-        config = self.config.load_key_json(path, is_file_required=False)
+        ng_config_text = self.config.load_key_text(path, is_file_required=False)
+        if ng_config_text:
+            ng_config_text = self._update_ng_urls(ng_config_text)
+            ng_config_json = json.loads(ng_config_text)
+
         # TODO: Log warning
-        return json.dumps(config, separators=(",", ":")) if config else "{}"
+        return json.dumps(ng_config_json, separators=(",", ":")) if ng_config_json else "{}"
 
     def load_computed_fields(self):
         https_prefix = self.config.https_prefix

apiv2/db_import/tests/conftest.py

@@ -133,5 +133,5 @@ def expected_dataset(http_prefix: str) -> dict[str, Any]:
         "key_photo_url": f"{http_prefix}/{DATASET_ID}/KeyPhoto/snapshot.png",
         "key_photo_thumbnail_url": f"{http_prefix}/{DATASET_ID}/KeyPhoto/thumbnail.png",
         "deposition_id": 300,
-        "file_size": 1374354.0,
+        "file_size": 1374808.0,
     }

apiv2/db_import/tests/test_db_annotation_import.py

@@ -53,6 +53,24 @@ def expected_annotations(http_prefix: str) -> list[dict[str, Any]]:
 def expected_annotation_files(http_prefix: str) -> list[dict[str, Any]]:
     path = f"{DATASET_ID}/RUN1/Reconstructions/VoxelSpacing12.300/Annotations/"
     return [
+        {
+            "tomogram_voxel_spacing_id": TOMOGRAM_VOXEL_ID1,
+            "s3_path": f"s3://test-public-bucket/{path}100-foo-1.0_instancesegmask.mrc",
+            "https_path": f"{http_prefix}/{path}100-foo-1.0_instancesegmask.mrc",
+            "source": "community",
+            "format": "mrc",
+            "is_visualization_default": False,
+            "file_size": 0,
+        },
+        {
+            "tomogram_voxel_spacing_id": TOMOGRAM_VOXEL_ID1,
+            "s3_path": f"s3://test-public-bucket/{path}100-foo-1.0_instancesegmask.zarr",
+            "https_path": f"{http_prefix}/{path}100-foo-1.0_instancesegmask.zarr",
+            "source": "community",
+            "format": "zarr",
+            "is_visualization_default": False,
+            "file_size": 0,
+        },
         {
             "id": ANNOTATION_FILE_ID,
             "tomogram_voxel_spacing_id": TOMOGRAM_VOXEL_ID1,