Support for Azure storage

- Among all the Azure storages, Khiops supports only (via its specific driver) "Files" and "Blobs" (Binary Large Objects)
 - The only supported authentication method is currently the `AZURE_STORAGE_CONNECTION_STRING` embedding the account name and account key

Author: Thierry RAMORASOAVINA

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Added
 - (`sklearn`) `keep_selected_variables_only` parameter to the predictors (`KhiopsClassifier` and `KhiopsRegressor`)
+- (General) Support for Azure storage
 
 ### Changed
 - (`core`) Rename `variable_part_dimensions` to `inner_variable_dimensions` in Coclustering results.

khiops/core/internals/filesystems.py

@@ -38,6 +38,18 @@
 except ImportError as import_error:
     gcs_import_error = import_error
 
+# Import azure packages if available
+# Delay an ImportError raising to an instantiation of the resource
+try:
+    from azure.core import credentials, utils
+
+    # pylint: disable=redefined-outer-name,unused-import
+    from azure.storage import blob, fileshare
+
+    azure_import_error = None
+except ImportError as import_error:
+    azure_import_error = import_error
+
 # pylint: enable=invalid-name
 
 ######################
@@ -76,6 +88,7 @@ def create_resource(uri_or_path):
         - ``file`` or empty: Local filesystem resource
         - ``s3``: Amazon S3 resource
         - ``gs``: Google Cloud Storage resource
+        - ``https``: Azure Storage resource (files or blobs)
 
     Returns
     -------
@@ -94,6 +107,14 @@ def create_resource(uri_or_path):
                 return AmazonS3Resource(uri_or_path)
             elif uri_info.scheme == "gs":
                 return GoogleCloudStorageResource(uri_or_path)
+            elif uri_info.scheme == "https":
+                # Create the corresponding instance of Azure storage resource
+                # based on the well-known name pattern
+                if AzureStorageResourceMixin.is_netloc_of_file_share(uri_info.netloc):
+                    return AzureStorageFileResource(uri_or_path)
+                else:
+                    # Assume we have a netloc of a blob
+                    return AzureStorageBlobResource(uri_or_path)
             elif scheme == "file":
                 # Reject URI if authority is not empty
                 if uri_info.netloc:
@@ -503,7 +524,7 @@ def create_parent(self):
 class GoogleCloudStorageResource(FilesystemResource):
     """Google Cloud Storage Resource
 
-    By default it reads the configuration from standard location.
+    By default, it reads the configuration from standard location.
     """
 
     def __init__(self, uri):
@@ -733,3 +754,313 @@ def create_parent(self):
 
 
 # pylint: enable=no-member
+
+
+class AzureStorageResourceMixin:
+    """Azure compatible Storage Resource Mixin
+
+    Among all the Azure storages, Khiops supports only (via its specific driver) :
+        - Files
+        - Blobs (Binary Large Objects)
+
+    For shared Files, the URI pattern of a resource is the following :
+    https://<storage-account-name>.file.core.windows.net/<share name>/...
+                                                <folder s>/<file name>
+    One may confuse the "share name" with a "folder name"
+    because nothing helps distinguish one from the other.
+
+    For blobs, the URI pattern of a resource is the following :
+    https://<storage-account-name>.blob.core.windows.net/<container name>/...
+                                                <folder s>/<file name>
+    One may confuse the "container name" with a "folder name"
+    because nothing helps distinguish one from the other.
+
+    By default, this resource reads the configuration from standard location
+    (environment variables for the moment)
+    """
+
+    def __init__(self, uri):
+        """
+        Azure Storage Resource initializer common to Files and Blobs
+        """
+
+        # Stop initialization if Azure modules are not available
+        if azure_import_error is not None:
+            warnings.warn(
+                "Could not import azure modules. "
+                "Make sure you have installed the azure.core and "
+                "azure.storage.fileshare packages to "
+                "access Azure Storage files."
+            )
+            raise azure_import_error
+
+        super().__init__(uri)
+
+        # Create the authentication object using the connection string
+        connection_string = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
+        mappings = utils.parse_connection_string(connection_string)
+        creds = credentials.AzureNamedKeyCredential(
+            mappings.get("accountname"), mappings.get("accountkey")
+        )
+
+        # Allow the extraction from the URL, the "share name" (for file shares)
+        # or the "container name" (for blobs)
+        # assuming it is the first part of the path.
+        # Warning : parts[0] contains '/'
+        parts = __class__._splitall(self.uri_info.path)
+
+        if __class__.is_netloc_of_file_share(self.uri_info.netloc):
+            share_url = f"{self.uri_info.scheme}://{self.uri_info.netloc}/{parts[1]}"
+            # Instantiate a ShareClient and attach it to the current instance.
+            # Later, instances of ShareFileClient or ShareDirectoryClient
+            # will be created.
+            self.azure_share_client = fileshare.ShareClient.from_share_url(
+                share_url=share_url, credential=creds
+            )
+        else:
+            # For blobs : most of the time, there is no need to distinguish
+            # between `container_url`
+            # and the remaining parts of the path.
+            # The `BlobClient` knows how to handle with the absolute path.
+            # But for blobs listing, there is an exception
+            # as we need a `ContainerClient`
+            container_url = (
+                f"{self.uri_info.scheme}://" f"{self.uri_info.netloc}/{parts[1]}"
+            )
+
+            # Instantiate a `ContainerClient` and attach it to the current instance.
+            self.azure_blob_container_client = blob.ContainerClient.from_container_url(
+                container_url=container_url, credential=creds
+            )
+
+            # Instantiate a `BlobClient` and attach it to the current instance.
+            self.azure_blob_client = blob.BlobClient.from_blob_url(
+                blob_url=self.uri_info.geturl(), credential=creds
+            )
+
+        # This attribute will be used in a "Shared File" only,
+        # for blobs the client will manage with absolute paths
+        self.relative_remaining_path = "/".join(parts[2:])
+
+    @staticmethod
+    def _splitall(path):
+        """
+        Build a list of path parts form a path as a str
+        """
+        allparts = []
+        while 1:
+            parts = os.path.split(path)
+            if parts[0] == path:  # sentinel for absolute paths
+                allparts.insert(0, parts[0])
+                break
+            elif parts[1] == path:  # sentinel for relative paths
+                allparts.insert(0, parts[1])
+                break
+            else:
+                path = parts[0]
+                allparts.insert(0, parts[1])
+        return allparts
+
+    @staticmethod
+    def is_netloc_of_file_share(netloc):
+        return netloc.endswith(".file.core.windows.net")
+
+
+class AzureStorageFileResource(AzureStorageResourceMixin, FilesystemResource):
+    """Azure compatible Storage File Resource
+
+    The remote "file share" on Azure MUST exist before any call
+
+    """
+
+    def write(self, data):
+        """
+        Notes:
+            FIXME :
+             In order to be consistent with all the other filesystems,
+             if the directory hierarchy does not exist,
+             it must be created
+        """
+        file_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        file_share_client.upload_file(data)
+
+    def exists(self):
+        """
+        Check if the target resource exists (file or directory)
+        """
+        file_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        directory_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        # Both clients are required because
+        # - the check against `file_share_client` is ``False``
+        #   if the target is a directory.
+        # - the check against `directory_share_client` is ``False``
+        #   if the target is a file.
+        return file_share_client.exists() or directory_share_client.exists()
+
+    def remove(self):
+        """
+        Remove the target resource either it is a file or a directory.
+        If an error occurs an exception is raised.
+        """
+        file_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        delete_file_error = None
+        delete_directory_error = None
+        try:
+            file_share_client.delete_file()
+            # If the deletion of the file succeeds then return immediately
+            return
+        except Exception as delete_error:  # pylint: disable=broad-exception-caught
+            delete_file_error = delete_error
+
+        directory_share_client = self.azure_share_client.get_directory_client(
+            self.relative_remaining_path
+        )
+        try:
+            directory_share_client.delete_directory()
+            # If the deletion of the directory succeeds then return immediately
+            return
+        except Exception as delete_error:  # pylint: disable=broad-exception-caught
+            delete_directory_error = delete_error
+
+        raise delete_file_error or delete_directory_error
+
+    def copy_from_local(self, local_path):
+        """
+        Notes:
+        Compared to other storages implementations,
+        if the parent directories are missing they are NOT created.
+        The caller must ensure the whole hierarchy exist otherwise an error will occur.
+        """
+        file_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        with open(local_path) as input_file:
+            file_share_client.upload_file(input_file.read())
+
+    def copy_to_local(self, local_path):
+        file_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        with open(local_path, "wb", encoding="utf8") as output_file:
+            data = file_share_client.download_file()
+            data.readinto(output_file)
+
+    def list_dir(self):
+        """
+        List the files (not the directories) of the current directory
+        Notes:
+            This is not a recursive listing operation
+        """
+        directory_share_client = self.azure_share_client.get_directory_client(
+            self.relative_remaining_path
+        )
+        return [
+            item["name"]
+            for item in directory_share_client.list_directories_and_files()
+            if not item["is_directory"]
+        ]
+
+    def make_dir(self):
+        """
+        Notes:
+        To avoid any exception while attempting to create an existing directory,
+        that would occur otherwise, its existence is first checked
+        """
+        directory_share_client = self.azure_share_client.get_directory_client(
+            self.relative_remaining_path
+        )
+        if not directory_share_client.exists():
+            self.azure_share_client.create_directory(self.relative_remaining_path)
+
+    def read(self, size=None):
+        file_share_client = self.azure_share_client.get_file_client(
+            self.relative_remaining_path
+        )
+        return file_share_client.download_file(length=size).readall()
+
+    def create_child(self, file_name):
+        return create_resource(child_uri_info(self.uri_info, file_name).geturl())
+
+    def create_parent(self):
+        return create_resource(parent_uri_info(self.uri_info).geturl())
+
+
+class AzureStorageBlobResource(AzureStorageResourceMixin, FilesystemResource):
+    """Azure compatible Storage Blob Resource
+
+    The remote "container" on Azure MUST exist before any call.
+
+    """
+
+    def read(self, size=None):
+        return self.azure_blob_client.download_blob(length=size).readall()
+
+    def write(self, data):
+        """
+        Notes:
+             in order to be consistent with the other drivers,
+             this method will overwrite the destination blob if it exists.
+             This is not the default behavior.
+        """
+        self.azure_blob_client.upload_blob(data, overwrite=True)
+
+    def exists(self):
+        return self.azure_blob_client.exists()
+
+    def remove(self):
+        self.azure_blob_client.delete_blob()
+
+    def copy_from_local(self, local_path):
+        """
+        Notes:
+             in order to be consistent with the other drivers,
+             this method will overwrite the destination blob if it exists.
+             This is not the default behavior.
+        """
+        with open(local_path) as input_file:
+            self.azure_blob_client.upload_blob(input_file.read(), overwrite=True)
+
+    def copy_to_local(self, local_path):
+        with open(local_path, "wb", encoding="utf8") as output_file:
+            data = self.azure_blob_client.download_blob()
+            data.readinto(output_file)
+
+    def list_dir(self):
+        """
+        Notes:
+             by default, the sdk will list all the blobs belonging to the container.
+             An extract filter is then required to simulate a directory listing.
+             FIXME : suppress the virtual parent directory
+        """
+        return [
+            item
+            for item in self.azure_blob_container_client.list_blob_names(
+                # Keep only the blobs belonging to the "virtual directory"
+                name_starts_with=self.relative_remaining_path,
+            )
+        ]
+
+    def make_dir(self):
+        warnings.warn(
+            "'make_dir' is a non-operation on Azure Storage for Blobs. "
+            "See the documentation at "
+            "https://learn.microsoft.com/en-us/rest/api/storageservices/"
+            "operations-on-containers and "
+            "https://learn.microsoft.com/en-us/rest/api/storageservices/"
+            "naming-and-referencing-containers--blobs--and-metadata#blob-names "
+            "(a virtual hierarchy can be created in naming blobs)"
+        )
+
+    def create_child(self, file_name):
+        return create_resource(child_uri_info(self.uri_info, file_name).geturl())
+
+    def create_parent(self, file_name):
+        return create_resource(parent_uri_info(self.uri_info).geturl())

pyproject.toml

@@ -113,13 +113,20 @@ dependencies = [
 Homepage = "https://khiops.org"
 
 [project.optional-dependencies]
+# Warning : Conda does not fully support PEP508 requirements specification
+# namely the "~=" operator. Use instead version ranges.
 s3 = [
     # do not use the latest version, to avoid undesired breaking changes
     "boto3>=1.17.39,<=1.35.69",
 ]
 gcs = [
     "google-cloud-storage>=1.37.0",
 ]
+azure = [
+    "azure-core>=1.39.0,<2.0.0",
+    "azure-storage-file-share>=12.10.1,<13.0.0",
+    "azure-storage-blob>=12.28.0,<13.0.0",
+]
 
 [tool.setuptools.packages.find]
 include = ["khiops", "khiops.*"]