Merge pull request #350 from Duke-GCB/azure-ddd

Add ddd command line tool for Cloud Storage

Author: johnbradley

ddsc/azure/__main__.py

@@ -1,13 +1,28 @@
 """Duke data service command line project management utility."""
 import sys
-from ddsc.ddsclient import DDSClient, AZURE_BACKING_STORAGE
 from ddsc.exceptions import DDSUserException
+from ddsc.ddsclient import DDSClient, AZURE_BACKING_STORAGE
+from ddsc.config import create_config
+
+
+class AzureDDSClient(DDSClient):
+    def __init__(self):
+        super().__init__(backing_storage=AZURE_BACKING_STORAGE)
+
+    def _create_config(self, args):
+        azure_container_name = None
+        if "azure_container_name" in args:
+            azure_container_name = args.azure_container_name
+        return create_config(
+            allow_insecure_config_file=args.allow_insecure_config_file,
+            azure_container_name=azure_container_name
+        )
 
 
 def main(args=None):
     if args is None:
         args = sys.argv[1:]
-    client = DDSClient(backing_storage=AZURE_BACKING_STORAGE)
+    client = AzureDDSClient()
     try:
         client.run_command(args)
     except DDSUserException as ex:

ddsc/azure/api.py

@@ -1,12 +1,10 @@
 import re
 import json
 import os.path
-from datetime import datetime, timedelta
-from azure.mgmt.storage import StorageManagementClient
-from azure.storage.filedatalake import DataLakeServiceClient, generate_file_system_sas
+from azure.storage.filedatalake import DataLakeServiceClient
 from azure.core.exceptions import ResourceNotFoundError
-from azure.identity import TokenCachePersistenceOptions, SharedTokenCacheCredential, ChainedTokenCredential, \
-    DeviceCodeCredential
+from azure.identity import DeviceCodeCredential, TokenCachePersistenceOptions, ChainedTokenCredential,\
+    SharedTokenCacheCredential
 from msgraph.core import GraphClient
 from ddsc.azure.azcopy import create_azcopy, group_by_dirname
 from ddsc.azure.delivery import DataDelivery
@@ -91,11 +89,8 @@ def ensure_user_exists(self, netid):
 
 
 class Bucket(object):
-    def __init__(self, credential, subscription_id, resource_group, storage_account, container_name):
-        self.resource_group = resource_group
-        self.storage_mgmt_client = StorageManagementClient(credential=credential, subscription_id=subscription_id)
-        self.service = DataLakeServiceClient(f"https://{storage_account}.dfs.core.windows.net/", credential=credential,
-                                             scopes=DLS_SCOPES)
+    def __init__(self, credential, storage_account, container_name):
+        self.service = DataLakeServiceClient(f"https://{storage_account}.dfs.core.windows.net/", credential=credential)
         self.file_system = self.service.get_file_system_client(file_system=container_name)
         self.azcopy = create_azcopy()
 
@@ -116,28 +111,6 @@ def move_directory(self, source, destination):
     def get_file_properties(self, file_path):
         return self.file_system.get_file_client(file_path).get_file_properties()
 
-    def get_storage_account_key1(self):
-        return self.storage_mgmt_client.storage_accounts.list_keys(
-            resource_group_name=self.resource_group,
-            account_name=self.service.account_name).keys[0].value
-
-    def get_sas_token(self, hours=6):
-        account_key = self.get_storage_account_key1()
-        return generate_file_system_sas(
-            account_name=self.service.account_name,
-            credential=account_key,
-            file_system_name=self.file_system.file_system_name,
-            permission="rwdl",
-            protocol='https',
-            expiry=datetime.utcnow() + timedelta(hours=hours)
-        )
-
-    def get_sas_url(self, path, hours=6):
-        account_name = self.service.account_name
-        bucket_name = self.file_system.file_system_name
-        token = self.get_sas_token(hours=hours)
-        return f"https://{account_name}.blob.core.windows.net/{bucket_name}/{path}?{token}"
-
     def get_url(self, path):
         account_name = self.service.account_name
         bucket_name = self.file_system.file_system_name
@@ -254,20 +227,21 @@ def __str__(self):
 
 
 class AzureApi(object):
-    def __init__(self, config, credential, subscription_id, resource_group, storage_account, container_name):
+    def __init__(self, config, credential):
         self.config = config
         self.users = Users(credential)
         self.current_user_netid = self.users.get_current_user_netid()
-        self.bucket = Bucket(credential, subscription_id, resource_group, storage_account, container_name)
+        container_name = config.azure_container_name
+        if not container_name:
+            container_name = self.current_user_netid
+        self.bucket = Bucket(credential, config.azure_storage_account, container_name)
 
     def list_projects(self):
-        path_dicts = self.bucket.get_paths(path=self.current_user_netid, recursive=False)
+        path_dicts = self.bucket.get_paths(path="", recursive=False)
         return [AzureProject(self, path_dict) for path_dict in path_dicts if path_dict["is_directory"] is True]
 
     def get_project_by_name(self, name):
         path = name
-        if "/" not in path:
-            path = f"{self.current_user_netid}/{name}"
         try:
             return AzureProject(self, self.bucket.get_directory_properties(path))
         except ResourceNotFoundError:
@@ -301,9 +275,6 @@ def get_file_paths(self, path):
     def get_file_properties(self, file_path):
         return self.bucket.get_file_properties(file_path)
 
-    def get_sas_url(self):
-        return self.bucket.get_sas_url(path=self.current_user_netid)
-
     def add_user_to_project(self, project, netid, auth_role):
         user_id, user_name = self.users.get_id_and_name(netid)
         role = self.get_auth_role_by_id(auth_role)
@@ -317,15 +288,11 @@ def remove_user_from_project(self, project, netid):
 
     def upload_paths(self, project_name, paths, dry_run):
         project_path = project_name
-        if "/" not in project_path:
-            project_path = f"{self.current_user_netid}/{project_name}/"
         self.bucket.upload_paths(project_path, paths, dry_run)
         print("\nUpload complete.\nSee azcopy log file for details about transferred files.\n\n")
 
     def download_paths(self, project_name, include_paths, exclude_paths, destination, dry_run):
         project_path = project_name
-        if "/" not in project_path:
-            project_path = f"{self.current_user_netid}/{project_name}/"
         self.bucket.download_paths(project_path, include_paths, exclude_paths, destination, dry_run=dry_run)
         print("\nDownload complete.\nSee azcopy log file for details about transferred files.\n\n")
 
@@ -352,15 +319,10 @@ def deliver(self, project, netid, resend, user_message, share_usernames):
 
 
 def create_azure_api(config):
-    # Setup to use token cache or prompt user to login with a URL (DeviceCodeCredential)
     cache_persistence_options = TokenCachePersistenceOptions(allow_unencrypted_storage=True)
     credential = ChainedTokenCredential(
         SharedTokenCacheCredential(),
         DeviceCodeCredential(cache_persistence_options=cache_persistence_options))
     return AzureApi(
         config=config,
-        credential=credential,
-        subscription_id=config.azure_subscription_id,
-        resource_group=config.azure_resource_group,
-        storage_account=config.azure_storage_account,
-        container_name=config.azure_container_name)
+        credential=credential)

ddsc/azure/commands.py

@@ -8,6 +8,7 @@
 CHECK_COMMAND_NOT_SUPPORTED_MSG = "Error: The check command is not supported or needed for the Azure backend.\n"
 SHARE_NOT_SUPPORTED_MSG = "Error: The share command is not supported for the Azure backend.\n"
 COPY_NOT_SUPPORTED_FOR_AZURE_MSG = "Error: The --copy option is not supported for the Azure backend.\n"
+COMMAND_NOT_SUPPORTED_MSG = "Error: This command is not supported for the Azure backend.\n"
 
 
 class BaseAzureCommand(object):
@@ -74,16 +75,12 @@ def run(self, args):
 
 class AzureAddUserCommand(BaseAzureCommand):
     def run(self, args):
-        project = self.get_project(args)
-        netid = self.get_netid(args)
-        self.azure_api.add_user_to_project(project=project, netid=netid, auth_role=args.auth_role)
+        raise DDSUserException(COMMAND_NOT_SUPPORTED_MSG)
 
 
 class AzureRemoveUserCommand(BaseAzureCommand):
     def run(self, args):
-        project = self.get_project(args)
-        netid = self.get_netid(args)
-        self.azure_api.remove_user_from_project(project=project, netid=netid)
+        raise DDSUserException(COMMAND_NOT_SUPPORTED_MSG)
 
 
 class AzureDownloadCommand(BaseAzureCommand):
@@ -135,8 +132,7 @@ def run(self, args):
 
 class AzureListAuthRolesCommand(BaseAzureCommand):
     def run(self, args):
-        for auth_role in self.azure_api.get_auth_roles():
-            print(auth_role.id, "-", auth_role.description)
+        raise DDSUserException(COMMAND_NOT_SUPPORTED_MSG)
 
 
 class AzureMoveCommand(BaseAzureCommand):

ddsc/azure/delivery.py

@@ -1,10 +1,14 @@
 import requests
 import requests.exceptions
 import json
-from ddsc.core.ddsapi import DataServiceAuth
 from ddsc.exceptions import DDSUserException
 from ddsc.core.d4s2 import UNAUTHORIZED_MESSAGE
 
+MISSING_DELIVERY_TOKEN_MSG = """
+ERROR: Missing credential to deliver projects.
+Please add 'delivery_token' to ~/.ddsclient config file.
+"""
+
 
 class DataDelivery(object):
     def __init__(self, config, api):
@@ -52,12 +56,13 @@ def _ensure_user_exists(self, netid):
 
 class DeliveryApi(object):
     def __init__(self, config):
-        auth = DataServiceAuth(config)
+        if not config.delivery_token:
+            raise DDSUserException(MISSING_DELIVERY_TOKEN_MSG)
         # Switch to v3 for azure endpoints (Once DDS API is deprecated switch default to v3)
         self.deliveries_url = config.azure_delivery_url + "/az-deliveries/"
         self.json_headers = {
             'Content-Type': 'application/json',
-            'X-DukeDS-Authorization': auth.get_auth()
+            'Authorization': 'Token ' + config.delivery_token
         }
 
     def create_delivery(self, payload):
@@ -105,5 +110,5 @@ def _check_response(response):
         if response.status_code == 401:
             raise DDSUserException(UNAUTHORIZED_MESSAGE)
         if not 200 <= response.status_code < 300:
-            msg_fmt = "Request to {} failed with {}:\n{}."
+            msg_fmt = "Request to {} failed with {}:\n{}"
             raise DDSUserException(msg_fmt.format(response.url, response.status_code, response.text))

ddsc/azure/tests/test_azure_api.py

@@ -67,7 +67,7 @@ def test_get_id_and_name(self, mock_graph_client):
 
 class TestBucket(TestCase):
     def setUp(self):
-        self.bucket = Bucket("Credentials", "subscription1", "resourceGroup2", "storageAccount3", "container4")
+        self.bucket = Bucket("Credentials", "storageAccount3", "container4")
         self.bucket.storage_mgmt_client = Mock()
         self.bucket.service = Mock(account_name="storageAccount3")
         self.bucket.file_system = Mock(file_system_name="container4")
@@ -103,16 +103,6 @@ def test_get_file_properties(self):
         fc = self.bucket.file_system.get_file_client.return_value
         self.assertEqual(result, fc.get_file_properties.return_value)
 
-    def test_get_storage_account_key1(self):
-        result = self.bucket.get_storage_account_key1()
-        self.assertEqual(result, "SomeKey")
-
-    @patch('ddsc.azure.api.generate_file_system_sas')
-    def test_get_sas_url(self, mock_generate_file_system_sas):
-        mock_generate_file_system_sas.return_value = "sas=Key"
-        url = self.bucket.get_sas_url("user1/mouse/file1.txt")
-        self.assertEqual(url, "https://storageAccount3.blob.core.windows.net/container4/user1/mouse/file1.txt?sas=Key")
-
     def test_get_url(self):
         url = self.bucket.get_url("user1/mouse/file1.txt")
         self.assertEqual(url, "https://storageAccount3.blob.core.windows.net/container4/user1/mouse/file1.txt")
@@ -239,8 +229,7 @@ class TestAzureApi(TestCase):
     def setUp(self, mock_bucket, mock_users):
         self.config = Mock()
         self.credential = Mock()
-        self.api = AzureApi(config=self.config, credential=self.credential, subscription_id='sub2',
-                            resource_group='rg3', storage_account='sa4', container_name='cn5')
+        self.api = AzureApi(config=self.config, credential=self.credential)
         self.api.current_user_netid = 'user1'
         self.bucket = self.api.bucket
         self.users = self.api.users
@@ -262,7 +251,7 @@ def test_list_projects(self):
     def test_get_project_by_name(self):
         project = self.api.get_project_by_name(name="mouse")
         self.assertEqual(project.name, "mouse")
-        self.bucket.get_directory_properties.assert_called_with('user1/mouse')
+        self.bucket.get_directory_properties.assert_called_with('mouse')
         self.bucket.get_directory_properties.side_effect = ResourceNotFoundError()
         self.assertEqual(self.api.get_project_by_name(name="mouse"), None)
 
@@ -292,10 +281,6 @@ def test_get_file_properties(self):
         self.assertEqual(self.api.get_file_properties("user1/mouse/file1.txt"),
                          self.bucket.get_file_properties.return_value)
 
-    def test_get_sas_url(self):
-        self.assertEqual(self.api.get_sas_url(), self.bucket.get_sas_url.return_value)
-        self.bucket.get_sas_url.assert_called_with(path='user1')
-
     @patch('ddsc.azure.api.print')
     def test_add_user_to_project(self, mock_print):
         self.api.add_user_to_project(Mock(path="user1/mouse"), netid="user2", auth_role="file_uploader")
@@ -315,13 +300,13 @@ def test_remove_user_from_project(self, mock_print):
     @patch('ddsc.azure.api.print')
     def test_upload_paths(self, mock_print):
         self.api.upload_paths(project_name="mouse", paths="/tmp/data.txt", dry_run=False)
-        self.bucket.upload_paths.assert_called_with('user1/mouse/', '/tmp/data.txt', False)
+        self.bucket.upload_paths.assert_called_with('mouse', '/tmp/data.txt', False)
 
     @patch('ddsc.azure.api.print')
     def test_download_paths(self, mock_print):
         self.api.download_paths(project_name="mouse", include_paths=None, exclude_paths=None, destination="/tmp/mouse",
                                 dry_run=False)
-        self.bucket.download_paths.assert_called_with('user1/mouse/', None, None, '/tmp/mouse', dry_run=False)
+        self.bucket.download_paths.assert_called_with('mouse', None, None, '/tmp/mouse', dry_run=False)
 
     @patch('ddsc.azure.api.print')
     def test_move_path(self, mock_print):