Merge branch 'main' into feature/improveTypedEntities-CMEM-6243

Author: seebi

CHANGELOG.md

@@ -11,12 +11,33 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 
 - RDF Quad entity type (CMEM-6243).
 
-## [4.10.1] 2025-05-08 - shipped with DI v25.1.1
+
+## [4.11.0] 2025-06-19
+
+### Added
+
+- `File` entities: add `entry_path` attribute and `read_stream` method
+
+### Changed
+
+- dependency cmem-cmempy >=25.2.0
+- dependency python-ulid ^3.0.0
+
+
+## [4.10.2] 2025-05-15 - shipped with DI v25.1.1
+
+### Fixed
+
+- Resolved an issue in `FileEntitySchema` for empty MIME types (CMEM-6623).
+
+
+## [4.10.1] 2025-05-08
 
 ### Fixed
 
 - Adapted FileEntitySchema so it can be used with datasets (CMEM-6615).
 
+
 ## [4.10.0] 2025-03-31 - shipped with DI v25.1.0
 
 ### Added
@@ -28,6 +49,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 
 - Check if passwords can be decrypted, i.e., if the key is valid (CMEM-5932)
 
+
 ## [4.9.0] 2025-02-20
 
 ### Added

TaskfileCustom.yaml

@@ -0,0 +1,29 @@
+---
+version: '3'
+
+tasks:
+
+  container:install:
+    desc: "Install package in container (then restart)"
+    summary: >
+      Note that `cmemc admin workspace python uninstall cmem-plugin-base` will
+      uninstall the dev-version and reveal the shipped package version again.
+    cmds:
+      - task clean build
+      - docker cp dist/{{.PACKAGE_FILE}} {{.DI_CONTAINER}}:/tmp/
+      - docker exec -t {{.DI_CONTAINER}} bash -c "{{.PIP_COMMAND}} /tmp/{{.PACKAGE_FILE}}"
+      - docker exec -t {{.DI_CONTAINER}} bash -c "rm -f /tmp/{{.PACKAGE_FILE}}"
+      - docker restart {{.DI_CONTAINER}}
+    preconditions:
+      - sh: '[ "{{.DI_RUNNING}}" == "true" ]'
+        msg: "Container {{.DI_CONTAINER}} not running ({{.DI_RUNNING}})"
+    vars:
+      DI_CONTAINER: dockerlocalhost-dataintegration-1
+      DI_RUNNING:
+        sh: docker inspect -f '{{ "{{" }}.State.Running{{ "}}" }}' {{.DI_CONTAINER}}
+      PACKAGE: cmem_plugin_base
+      PACKAGE_VERSION:
+        sh: poetry version -s
+      PACKAGE_FILE: "{{.PACKAGE}}-{{.PACKAGE_VERSION}}.tar.gz"
+      PIP_COMMAND: pip install --target /data/python-packages -q --upgrade
+

cmem_plugin_base/dataintegration/typed_entities/file.py

@@ -1,5 +1,13 @@
 """File entities"""
 
+import zipfile
+from abc import abstractmethod
+from io import BytesIO
+from pathlib import Path
+from typing import IO
+
+from cmem.cmempy.workspace.projects.resources.resource import get_resource_response
+
 from cmem_plugin_base.dataintegration.entity import Entity, EntityPath
 from cmem_plugin_base.dataintegration.typed_entities import instance_uri, path_uri, type_uri
 from cmem_plugin_base.dataintegration.typed_entities.typed_entities import (
@@ -8,26 +16,83 @@
 
 
 class File:
-    """A file entity that can be held in a FileEntitySchema."""
+    """A file entity that can be held in a FileEntitySchema.
+
+    :param path: The file path.
+    :param file_type: The type of the file (one of: "Local", "Project").
+    :param mime: The MIME type of the file, if known.
+    :param entry_path: If the file path points to a archive, the entry within the archive.
+    """
 
-    def __init__(self, path: str, file_type: str, mime: str | None) -> None:
+    def __init__(self, path: str, file_type: str, mime: str | None, entry_path: str | None) -> None:
         self.path = path
         self.file_type = file_type
         self.mime = mime
+        self.entry_path = entry_path
+
+    @abstractmethod
+    def read_stream(self, project_id: str) -> IO[bytes]:
+        """Open the referenced file as a stream.
+
+        Returns a file-like object (stream) in binary mode.
+        Caller is responsible for closing the stream.
+        """
 
 
 class LocalFile(File):
     """A file that's located on the local file system."""
 
-    def __init__(self, path: str, mime: str | None = None) -> None:
-        super().__init__(path, "Local", mime)
+    def __init__(self, path: str, mime: str | None = None, entry_path: str | None = None) -> None:
+        super().__init__(path, "Local", mime, entry_path)
+
+    def read_stream(self, project_id: str) -> IO[bytes]:
+        """Open the referenced file as a stream.
+
+        Returns a file-like object (stream) in binary mode.
+        Caller is responsible for closing the stream.
+        """
+        if self.entry_path:
+            archive = zipfile.ZipFile(self.path, "r")
+            try:
+                return archive.open(self.entry_path, "r")
+            except KeyError as err:
+                archive.close()
+                raise FileNotFoundError(
+                    f"Entry '{self.entry_path}' not found in archive '{self.path}'."
+                ) from err
+        else:
+            if not Path(self.path).is_file():
+                raise FileNotFoundError(f"File '{self.path}' does not exist.")
+            return Path(self.path).open("rb")
 
 
 class ProjectFile(File):
     """A project file"""
 
-    def __init__(self, path: str, mime: str | None = None) -> None:
-        super().__init__(path, "Project", mime)
+    def __init__(self, path: str, mime: str | None = None, entry_path: str | None = None) -> None:
+        super().__init__(path, "Project", mime, entry_path)
+
+    def read_stream(self, project_id: str) -> IO[bytes]:
+        """Open the referenced file as a stream.
+
+        Returns a file-like object (stream) in binary mode.
+        Caller is responsible for closing the stream.
+        """
+        response = get_resource_response(project_id, self.path)
+        if response.status_code != 200:  # noqa: PLR2004
+            raise FileNotFoundError(f"Project file '{self.path}' not found.")
+        response_bytes = BytesIO(response.raw.read())
+        if self.entry_path:
+            archive = zipfile.ZipFile(response_bytes, "r")
+            try:
+                return archive.open(self.entry_path, "r")
+            except KeyError as err:
+                archive.close()
+                raise FileNotFoundError(
+                    f"Entry '{self.entry_path}' not found in project file '{self.path}'."
+                ) from err
+        else:
+            return response_bytes
 
 
 class FileEntitySchema(TypedEntitySchema[File]):
@@ -49,18 +114,25 @@ def to_entity(self, value: File) -> Entity:
         """Create a generic entity from a file"""
         return Entity(
             uri=instance_uri(value.path),
-            values=[[value.path], [value.file_type], [value.mime or ""]],
+            values=[
+                [value.path],
+                [value.file_type],
+                [value.mime] if value.mime else [],
+                [value.entry_path] if value.entry_path else [],
+            ],
         )
 
     def from_entity(self, entity: Entity) -> File:
         """Create a file entity from a generic entity."""
         path = entity.values[0][0]
         file_type = entity.values[1][0]
-        mime = entity.values[2][0] if entity.values[2][0] else None
+        mime = entity.values[2][0] if entity.values[2] and entity.values[2][0] else None
+        entry_path = entity.values[3][0] if entity.values[3] and entity.values[3][0] else None
+
         match file_type:
             case "Local":
-                return LocalFile(path, mime)
+                return LocalFile(path, mime, entry_path)
             case "Project":
-                return ProjectFile(path, mime)
+                return ProjectFile(path, mime, entry_path)
             case _:
                 raise ValueError(f"File '{path}' has unexpected type '{file_type}'.")