keeping resources in bionemo-core when needed

Author: polinabinder1

sub-packages/bionemo-core/src/bionemo/core/data/resources/scdl.yaml

@@ -0,0 +1,15 @@
+- tag: sample
+  ngc: nvidia/clara/scdl_sample_test:1.0
+  ngc_registry: resource
+  pbss: "s3://bionemo-ci/test-data/scdl_sample_test.tar.gz"
+  sha256: 7a4237537bf535dfa00301ce8cc7073e0a23d5bc8aa902ad65db9f51b57a6df9 # pragma: allowlist secret
+  owner: Polina Binder <pbinder@nvidia.com>
+  description: Sample test data for SCDL.
+
+- tag: sample_scdl_feature_ids
+  ngc: nvidia/clara/scdl_sample_test_feature_ids:1.0
+  ngc_registry: resource
+  pbss: s3://bionemo-ci/test-data/scdl_sample_test_feat_ids.tar.gz
+  sha256: 9020ba336dbfe33bddadba26ca0cde49958cbd73c5ad44f0960a5a4837c9db26 # pragma: allowlist secret
+  owner: Savitha Srinivasan <savithas@nvidia.com>
+  description: Sample test data for SCDL with feature IDs appended.

sub-packages/bionemo-scdl/src/bionemo/scdl/data/load.py

@@ -78,9 +78,6 @@ class Resource(pydantic.BaseModel):
     Should be in format [org/[team/]]name[:version]. If None, the resource is not available on NGC.
     """
 
-    ngc_registry: Literal["model", "resource"] | None = None
-    """The NGC resource type (model or resource) for the data. Must be provided if ngc is not None."""
-
     pbss: Annotated[pydantic.AnyUrl, pydantic.UrlConstraints(allowed_schemes=["s3"])]
     """The PBSS (NVIDIA-internal) URL of the resource."""
 
@@ -99,12 +96,6 @@ class Resource(pydantic.BaseModel):
     decompress: Literal[False, None] = None
     """Whether the resource should be decompressed after download. If None, will defer to the file extension."""
 
-    @pydantic.model_validator(mode="after")
-    def _validate_ngc_registry(self):
-        if self.ngc and not self.ngc_registry:
-            raise ValueError(f"ngc_registry must be provided if ngc is not None: {self.tag}")
-        return self
-
 
 @functools.cache
 def get_all_resources(resource_path: Path | None = None) -> dict[str, Resource]:
@@ -216,17 +207,14 @@ class NGCDownloader:
     """
 
     filename: str
-    ngc_registry: Literal["model", "resource"]
 
     def __call__(self, url: str, output_file: str | Path, _: pooch.Pooch) -> None:
         """Download a file from NGC."""
         client = default_ngc_client()
         nest_asyncio.apply()
 
-        download_fns = {
-            "model": client.registry.model.download_version,
-            "resource": client.registry.resource.download_version,
-        }
+        # SCDL only uses NGC resources, never models
+        download_fn = client.registry.resource.download_version
 
         output_file = Path(output_file)
         output_file.parent.mkdir(parents=True, exist_ok=True)
@@ -235,7 +223,7 @@ def __call__(self, url: str, output_file: str | Path, _: pooch.Pooch) -> None:
         ngc_dirname = Path(url).name.replace(":", "_v")
 
         with tempfile.TemporaryDirectory(dir=output_file.parent) as temp_dir:
-            download_fns[self.ngc_registry](url, temp_dir, file_patterns=[self.filename])
+            download_fn(url, temp_dir, file_patterns=[self.filename])
             shutil.move(Path(temp_dir) / ngc_dirname / self.filename, output_file)
 
 
@@ -289,8 +277,7 @@ def load(
         url = resource.pbss
 
     elif source == "ngc":
-        assert resource.ngc_registry is not None
-        download_fn = NGCDownloader(filename=filename, ngc_registry=resource.ngc_registry)
+        download_fn = NGCDownloader(filename=filename)
         url = resource.ngc
 
     else:

sub-packages/bionemo-scdl/src/bionemo/scdl/data/resources/scdl.yaml

@@ -1,30 +1,26 @@
 - tag: sample
   ngc: nvidia/clara/scdl_sample_test:1.0
-  ngc_registry: resource
   pbss: "s3://bionemo-ci/test-data/scdl_sample_test.tar.gz"
   sha256: 7a4237537bf535dfa00301ce8cc7073e0a23d5bc8aa902ad65db9f51b57a6df9 # pragma: allowlist secret
   owner: Polina Binder <pbinder@nvidia.com>
   description: Sample test data for SCDL.
 
 - tag: sample_scdl_feature_ids
   ngc: nvidia/clara/scdl_sample_test_feature_ids:1.0
-  ngc_registry: resource
   pbss: s3://bionemo-ci/test-data/scdl_sample_test_feat_ids.tar.gz
   sha256: 9020ba336dbfe33bddadba26ca0cde49958cbd73c5ad44f0960a5a4837c9db26 # pragma: allowlist secret
   owner: Savitha Srinivasan <savithas@nvidia.com>
   description: Sample test data for SCDL with feature IDs appended.
 
 - tag: sample_scdl_neighbor
   ngc: nvidia/clara/scdl_neighbor_testdata:1.0
-  ngc_registry: resource
   pbss: "s3://bionemo-ci/test-data/scdl_neighbor_test_20250616.tar.gz"
   sha256: f64a723e5a1d3223d7ad636c2b7601fe5927be47fb1a418a60687ef80eab83d0 # pragma: allowlist secret
   owner: Camir Ricketts <camirr@nvidia.com>
   description: Sample test data for SCDL with neighbors.
 
 - tag: testdata-20240506
   ngc: nvidia/clara/singlecell-testdata:1.0
-  ngc_registry: resource
   pbss: "s3://bionemo-ci/test-data/singlecell/singlecell-testdata-20240506.tar.gz"
   sha256: db24ba3858005680e343d0e4714c7c91fde6d738e2bf4018d489c0b1541544df # pragma: allowlist secret
   owner: John St John <jstjohn@nvidia.com>