diff --git a/src/azure-cli/azure/cli/command_modules/storage/__init__.py b/src/azure-cli/azure/cli/command_modules/storage/__init__.py index 052cbf6db0b..735e2f3c030 100644 --- a/src/azure-cli/azure/cli/command_modules/storage/__init__.py +++ b/src/azure-cli/azure/cli/command_modules/storage/__init__.py @@ -42,29 +42,6 @@ def load_arguments(self, command): load_arguments(self, command) -class AzureStackStorageCommandsLoader(AzCommandsLoader): - def __init__(self, cli_ctx=None): - from azure.cli.core.commands import CliCommandType - - storage_custom = CliCommandType(operations_tmpl='azure.cli.command_modules.storage.custom#{}') - super().__init__(cli_ctx=cli_ctx, - resource_type=ResourceType.DATA_STORAGE, - custom_command_type=storage_custom, - command_group_cls=AzureStackStorageCommandGroup, - argument_context_cls=StorageArgumentContext) - - def load_command_table(self, args): - super().load_command_table(args) - from azure.cli.command_modules.storage.commands_azure_stack import load_command_table - load_command_table(self, args) - return self.command_table - - def load_arguments(self, command): - super().load_arguments(command) - from azure.cli.command_modules.storage._params_azure_stack import load_arguments - load_arguments(self, command) - - class StorageArgumentContext(AzArgumentContext): def register_sas_arguments(self): from azure.cli.command_modules.storage._validators import ipv4_range_type, get_datetime_type @@ -382,44 +359,6 @@ def _register_data_plane_oauth_arguments(self, command_name): 'allowed data actions, even if there are ACLs in place for those files/directories.') -class AzureStackStorageCommandGroup(StorageCommandGroup): - - @classmethod - def get_handler_suppress_some_400(cls): - def handler(ex): - if hasattr(ex, 'status_code') and ex.status_code == 403: - # TODO: Revisit the logic here once the service team updates their response - if 'AuthorizationPermissionMismatch' in ex.args[0]: - message = """ -You do not have the required permissions needed to perform this operation. -Depending on your operation, you may need to be assigned one of the following roles: - "Storage Blob Data Contributor" - "Storage Blob Data Reader" - "Storage Queue Data Contributor" - "Storage Queue Data Reader" - "Storage Table Data Contributor" - "Storage Table Data Reader" - -If you want to use the old authentication method and allow querying for the right account key, please use the "--auth-mode" parameter and "key" value. - """ - ex.args = (message,) - elif 'AuthorizationFailure' in ex.args[0]: - message = """ -The request may be blocked by network rules of storage account. Please check network rule set using 'az storage account show -n accountname --query networkRuleSet'. -If you want to change the default action to apply when no rule matches, please use 'az storage account update'. - """ - ex.args = (message,) - elif 'AuthenticationFailed' in ex.args[0]: - message = """ -Authentication failure. This may be caused by either invalid account key, connection string or sas token value provided for your storage account. - """ - ex.args = (message,) - if hasattr(ex, 'status_code') and ex.status_code == 409 and 'NoPendingCopyOperation' in ex.args[0]: - pass - - return handler - - def _merge_new_exception_handler(kwargs, handler): first = kwargs.get('exception_handler') @@ -431,8 +370,4 @@ def new_handler(ex): kwargs['exception_handler'] = new_handler -def get_command_loader(cli_ctx): - if cli_ctx.cloud.profile.lower() != 'latest': - return AzureStackStorageCommandsLoader - - return StorageCommandsLoader +COMMAND_LOADER_CLS = StorageCommandsLoader diff --git a/src/azure-cli/azure/cli/command_modules/storage/_client_factory_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/_client_factory_azure_stack.py deleted file mode 100644 index f76d4f18d52..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/_client_factory_azure_stack.py +++ /dev/null @@ -1,250 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from azure.cli.core.commands.client_factory import get_mgmt_service_client, get_data_service_client -from azure.cli.core.profiles import ResourceType, get_sdk - -from azure.cli.command_modules.storage.sdkutil import get_table_data_type - -MISSING_CREDENTIALS_ERROR_MESSAGE = """ -Missing credentials to access storage service. The following variations are accepted: - (1) account name and key (--account-name and --account-key options or - set AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY environment variables) - (2) account name and SAS token (--sas-token option used with either the --account-name - option or AZURE_STORAGE_ACCOUNT environment variable) - (3) account name (--account-name option or AZURE_STORAGE_ACCOUNT environment variable; - this will make calls to query for a storage account key using login credentials) - (4) connection string (--connection-string option or - set AZURE_STORAGE_CONNECTION_STRING environment variable); some shells will require - quoting to preserve literal character interpretation. -""" - - -def get_storage_data_service_client(cli_ctx, service, name=None, key=None, connection_string=None, sas_token=None, - socket_timeout=None, token_credential=None): - return get_data_service_client(cli_ctx, service, name, key, connection_string, sas_token, - socket_timeout=socket_timeout, - token_credential=token_credential, - endpoint_suffix=cli_ctx.cloud.suffixes.storage_endpoint) - - -def generic_data_service_factory(cli_ctx, service, name=None, key=None, connection_string=None, sas_token=None, - socket_timeout=None, token_credential=None): - try: - return get_storage_data_service_client(cli_ctx, service, name, key, connection_string, sas_token, - socket_timeout, token_credential) - except ValueError as val_exception: - _ERROR_STORAGE_MISSING_INFO = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, - 'common._error#_ERROR_STORAGE_MISSING_INFO') - message = str(val_exception) - if message == _ERROR_STORAGE_MISSING_INFO: - message = MISSING_CREDENTIALS_ERROR_MESSAGE - from knack.util import CLIError - raise CLIError(message) - - -def storage_client_factory(cli_ctx, **_): - return get_mgmt_service_client(cli_ctx, ResourceType.MGMT_STORAGE) - - -def file_data_service_factory(cli_ctx, kwargs): - t_file_svc = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'file#FileService') - return generic_data_service_factory(cli_ctx, t_file_svc, kwargs.pop('account_name', None), - kwargs.pop('account_key', None), - connection_string=kwargs.pop('connection_string', None), - sas_token=kwargs.pop('sas_token', None)) - - -def page_blob_service_factory(cli_ctx, kwargs): - t_page_blob_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'blob.pageblobservice#PageBlobService') - return generic_data_service_factory(cli_ctx, t_page_blob_service, kwargs.pop('account_name', None), - kwargs.pop('account_key', None), - connection_string=kwargs.pop('connection_string', None), - sas_token=kwargs.pop('sas_token', None), - token_credential=kwargs.pop('token_credential', None)) - - -def blob_data_service_factory(cli_ctx, kwargs): - if 'encryption_scope' in kwargs and kwargs['encryption_scope']: - return cf_blob_client(cli_ctx, kwargs) - from azure.cli.command_modules.storage.sdkutil import get_blob_service_by_type - blob_type = kwargs.get('blob_type') - blob_service = get_blob_service_by_type(cli_ctx, blob_type) or get_blob_service_by_type(cli_ctx, 'block') - - return generic_data_service_factory(cli_ctx, blob_service, kwargs.pop('account_name', None), - kwargs.pop('account_key', None), - connection_string=kwargs.pop('connection_string', None), - sas_token=kwargs.pop('sas_token', None), - socket_timeout=kwargs.pop('socket_timeout', None), - token_credential=kwargs.pop('token_credential', None)) - - -def table_data_service_factory(cli_ctx, kwargs): - return generic_data_service_factory(cli_ctx, - get_table_data_type(cli_ctx, 'table', 'TableService'), - kwargs.pop('account_name', None), - kwargs.pop('account_key', None), - connection_string=kwargs.pop('connection_string', None), - sas_token=kwargs.pop('sas_token', None)) - - -def queue_data_service_factory(cli_ctx, kwargs): - t_queue_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'queue#QueueService') - return generic_data_service_factory( - cli_ctx, t_queue_service, - kwargs.pop('account_name', None), - kwargs.pop('account_key', None), - connection_string=kwargs.pop('connection_string', None), - sas_token=kwargs.pop('sas_token', None), - token_credential=kwargs.pop('token_credential', None)) - - -def cloud_storage_account_service_factory(cli_ctx, kwargs): - t_cloud_storage_account = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'common#CloudStorageAccount') - account_name = kwargs.pop('account_name', None) - account_key = kwargs.pop('account_key', None) - sas_token = kwargs.pop('sas_token', None) - kwargs.pop('connection_string', None) - return t_cloud_storage_account(account_name, account_key, sas_token) - - -def multi_service_properties_factory(cli_ctx, kwargs): - """Create multiple data services properties instance based on the services option""" - from .services_wrapper_azure_stack import ServiceProperties - - t_base_blob_service, t_file_service, t_queue_service, = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, - 'blob.baseblobservice#BaseBlobService', - 'file#FileService', 'queue#QueueService') - - t_table_service = get_table_data_type(cli_ctx, 'table', 'TableService') - - account_name = kwargs.pop('account_name', None) - account_key = kwargs.pop('account_key', None) - connection_string = kwargs.pop('connection_string', None) - sas_token = kwargs.pop('sas_token', None) - services = kwargs.pop('services', []) - - def get_creator(name, service_type): - return lambda: ServiceProperties(cli_ctx, name, service_type, account_name, account_key, connection_string, - sas_token) - - creators = {'b': get_creator('blob', t_base_blob_service), 'f': get_creator('file', t_file_service), - 'q': get_creator('queue', t_queue_service), 't': get_creator('table', t_table_service)} - - return [creators[s]() for s in services] - - -def cf_sa(cli_ctx, _): - return storage_client_factory(cli_ctx).storage_accounts - - -def cf_sa_for_keys(cli_ctx, _): - from knack.log import get_logger - logger = get_logger(__name__) - logger.debug('Disable HTTP logging to avoid having storage keys in debug logs') - client = storage_client_factory(cli_ctx) - return client.storage_accounts - - -def cf_mgmt_policy(cli_ctx, _): - return storage_client_factory(cli_ctx).management_policies - - -def cf_blob_container_mgmt(cli_ctx, _): - return storage_client_factory(cli_ctx).blob_containers - - -def cf_mgmt_blob_services(cli_ctx, _): - return storage_client_factory(cli_ctx).blob_services - - -def cf_mgmt_file_services(cli_ctx, _): - return storage_client_factory(cli_ctx).file_services - - -def cf_mgmt_file_shares(cli_ctx, _): - return storage_client_factory(cli_ctx).file_shares - - -def cf_blob_data_gen_update(cli_ctx, kwargs): - return blob_data_service_factory(cli_ctx, kwargs.copy()) - - -def cf_private_link(cli_ctx, _): - return storage_client_factory(cli_ctx).private_link_resources - - -def cf_private_endpoint(cli_ctx, _): - return storage_client_factory(cli_ctx).private_endpoint_connections - - -def cf_mgmt_encryption_scope(cli_ctx, _): - return storage_client_factory(cli_ctx).encryption_scopes - - -def get_account_url(cli_ctx, account_name, service): - from knack.util import CLIError - if account_name is None: - raise CLIError("Please provide storage account name or connection string.") - storage_endpoint = cli_ctx.cloud.suffixes.storage_endpoint - return "https://{}.{}.{}".format(account_name, service, storage_endpoint) - - -def cf_blob_service(cli_ctx, kwargs): - from knack.util import CLIError - t_blob_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE_BLOB, - '_blob_service_client#BlobServiceClient') - connection_string = kwargs.pop('connection_string', None) - account_name = kwargs.pop('account_name', None) - account_key = kwargs.pop('account_key', None) - token_credential = kwargs.pop('token_credential', None) - sas_token = kwargs.pop('sas_token', None) - if connection_string: - return t_blob_service.from_connection_string(conn_str=connection_string) - - account_url = get_account_url(cli_ctx, account_name=account_name, service='blob') - credential = account_key or sas_token or token_credential - - if account_url and credential: - return t_blob_service(account_url=account_url, credential=credential) - raise CLIError("Please provide valid connection string, or account name with account key, " - "sas token or login auth mode.") - - -def cf_blob_client(cli_ctx, kwargs): - return cf_blob_service(cli_ctx, kwargs).get_blob_client(container=kwargs['container_name'], - blob=kwargs['blob_name'], - snapshot=kwargs.pop('snapshot', None)) - - -def cf_adls_service(cli_ctx, kwargs): - t_adls_service = get_sdk(cli_ctx, ResourceType.DATA_STORAGE_FILEDATALAKE, - '_data_lake_service_client#DataLakeServiceClient') - connection_string = kwargs.pop('connection_string', None) - account_key = kwargs.pop('account_key', None) - token_credential = kwargs.pop('token_credential', None) - sas_token = kwargs.pop('sas_token', None) - if connection_string: - return t_adls_service.from_connection_string(connection_string=connection_string) - - account_url = get_account_url(cli_ctx, account_name=kwargs.pop('account_name', None), service='dfs') - credential = account_key or sas_token or token_credential - - if account_url and credential: - return t_adls_service(account_url=account_url, credential=credential) - return None - - -def cf_adls_file_system(cli_ctx, kwargs): - return cf_adls_service(cli_ctx, kwargs).get_file_system_client(file_system=kwargs.pop('file_system_name')) - - -def cf_adls_directory(cli_ctx, kwargs): - return cf_adls_file_system(cli_ctx, kwargs).get_directory_client(directory=kwargs.pop('directory_path')) - - -def cf_adls_file(cli_ctx, kwargs): - return cf_adls_service(cli_ctx, kwargs).get_file_client(file_system=kwargs.pop('file_system_name', None), - file_path=kwargs.pop('path', None)) diff --git a/src/azure-cli/azure/cli/command_modules/storage/_format_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/_format_azure_stack.py deleted file mode 100644 index b2ffc0c1c3f..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/_format_azure_stack.py +++ /dev/null @@ -1,35 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from knack.log import get_logger -from azure.cli.core.profiles import get_sdk, ResourceType - -logger = get_logger(__name__) - - -def transform_file_directory_result(cli_ctx): - """ - Transform a the result returned from file and directory listing API. - This transformer add and remove properties from File and Directory objects in the given list - in order to align the object's properties so as to offer a better view to the file and dir - list. - """ - def transformer(result): - if getattr(result, 'next_marker', None): - logger.warning('Next Marker:') - logger.warning(result.next_marker) - - t_file, t_dir = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'File', 'Directory', mod='file.models') - return_list = [] - for each in result: - if isinstance(each, t_file): - delattr(each, 'content') - setattr(each, 'type', 'file') - elif isinstance(each, t_dir): - setattr(each, 'type', 'dir') - return_list.append(each) - - return return_list - return transformer diff --git a/src/azure-cli/azure/cli/command_modules/storage/_params_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/_params_azure_stack.py deleted file mode 100644 index 7dd08e00eff..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/_params_azure_stack.py +++ /dev/null @@ -1,1262 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from azure.cli.core.profiles import ResourceType -from azure.cli.core.commands.validators import get_default_location_from_resource_group -from azure.cli.core.commands.parameters import (tags_type, file_type, get_location_type, get_enum_type, - get_three_state_flag) -from azure.cli.core.local_context import LocalContextAttribute, LocalContextAction, ALL - -from ._validators import (get_datetime_type, validate_metadata, get_permission_validator, get_permission_help_string, - resource_type_type, services_type, validate_select, validate_blob_type, - validate_included_datasets, validate_custom_domain, - validate_table_payload_format, add_progress_callback, process_resource_group, - storage_account_key_options, process_file_download_namespace, process_metric_update_namespace, - get_char_options_validator, validate_bypass, validate_encryption_source, validate_marker, - validate_storage_data_plane_list, validate_azcopy_upload_destination_url, - as_user_validator, parse_storage_account, - validator_delete_retention_days, validate_delete_retention_days, - validate_fs_public_access) -from ._validators_azure_stack import validate_entity, validate_container_public_access, validate_azcopy_remove_arguments - - -def load_arguments(self, _): # pylint: disable=too-many-locals, too-many-statements, too-many-lines - from argcomplete.completers import FilesCompleter - - from knack.arguments import ignore_type, CLIArgumentType - - from azure.cli.core.commands.parameters import get_resource_name_completion_list - - from .sdkutil import get_table_data_type - from .completers import get_storage_name_completion_list - - t_base_blob_service = self.get_sdk('blob.baseblobservice#BaseBlobService') - t_file_service = self.get_sdk('file#FileService') - t_queue_service = self.get_sdk('queue#QueueService') - t_table_service = get_table_data_type(self.cli_ctx, 'table', 'TableService') - - storage_account_type = CLIArgumentType(options_list='--storage-account', - help='The name or ID of the storage account.', - validator=parse_storage_account, id_part='name') - - acct_name_type = CLIArgumentType(options_list=['--account-name', '-n'], help='The storage account name.', - id_part='name', - completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts'), - local_context_attribute=LocalContextAttribute( - name='storage_account_name', actions=[LocalContextAction.GET])) - blob_name_type = CLIArgumentType(options_list=['--blob-name', '-b'], help='The blob name.', - completer=get_storage_name_completion_list(t_base_blob_service, 'list_blobs', - parent='container_name')) - - container_name_type = CLIArgumentType(options_list=['--container-name', '-c'], help='The container name.', - completer=get_storage_name_completion_list(t_base_blob_service, - 'list_containers')) - directory_type = CLIArgumentType(options_list=['--directory-name', '-d'], help='The directory name.', - completer=get_storage_name_completion_list(t_file_service, - 'list_directories_and_files', - parent='share_name')) - file_name_type = CLIArgumentType(options_list=['--file-name', '-f'], - completer=get_storage_name_completion_list(t_file_service, - 'list_directories_and_files', - parent='share_name')) - share_name_type = CLIArgumentType(options_list=['--share-name', '-s'], help='The file share name.', - completer=get_storage_name_completion_list(t_file_service, 'list_shares')) - table_name_type = CLIArgumentType(options_list=['--table-name', '-t'], - completer=get_storage_name_completion_list(t_table_service, 'list_tables')) - queue_name_type = CLIArgumentType(options_list=['--queue-name', '-q'], help='The queue name.', - completer=get_storage_name_completion_list(t_queue_service, 'list_queues')) - progress_type = CLIArgumentType(help='Include this flag to disable progress reporting for the command.', - action='store_true', validator=add_progress_callback) - socket_timeout_type = CLIArgumentType(help='The socket timeout(secs), used by the service to regulate data flow.', - type=int) - num_results_type = CLIArgumentType( - default=5000, help='Specifies the maximum number of results to return. Provide "*" to return all.', - validator=validate_storage_data_plane_list) - - large_file_share_type = CLIArgumentType( - action='store_true', min_api='2019-04-01', - help='Enable the capability to support large file shares with more than 5 TiB capacity for storage account.' - 'Once the property is enabled, the feature cannot be disabled. Currently only supported for LRS and ' - 'ZRS replication types, hence account conversions to geo-redundant accounts would not be possible. ' - 'For more information, please refer to https://go.microsoft.com/fwlink/?linkid=2086047.') - adds_type = CLIArgumentType(arg_type=get_three_state_flag(), min_api='2019-04-01', - help='Enable Azure Files Active Directory Domain Service Authentication for ' - 'storage account. When --enable-files-adds is set to true, Azure Active ' - 'Directory Properties arguments must be provided.') - aadds_type = CLIArgumentType(arg_type=get_three_state_flag(), min_api='2018-11-01', - help='Enable Azure Active Directory Domain Services authentication for Azure Files') - domain_name_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties", - help="Specify the primary domain that the AD DNS server is authoritative for. " - "Required when --enable-files-adds is set to True") - net_bios_domain_name_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties", - help="Specify the NetBIOS domain name. " - "Required when --enable-files-adds is set to True") - forest_name_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties", - help="Specify the Active Directory forest to get. " - "Required when --enable-files-adds is set to True") - domain_guid_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties", - help="Specify the domain GUID. Required when --enable-files-adds is set to True") - domain_sid_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties", - help="Specify the security identifier (SID). Required when --enable-files-adds " - "is set to True") - azure_storage_sid_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties", - help="Specify the security identifier (SID) for Azure Storage. " - "Required when --enable-files-adds is set to True") - exclude_pattern_type = CLIArgumentType(arg_group='Additional Flags', help='Exclude these files where the name ' - 'matches the pattern list. For example: *.jpg;*.pdf;exactName. This ' - 'option supports wildcard characters (*)') - include_pattern_type = CLIArgumentType(arg_group='Additional Flags', help='Include only these files where the name ' - 'matches the pattern list. For example: *.jpg;*.pdf;exactName. This ' - 'option supports wildcard characters (*)') - exclude_path_type = CLIArgumentType(arg_group='Additional Flags', help='Exclude these paths. This option does not ' - 'support wildcard characters (*). Checks relative path prefix. For example: ' - 'myFolder;myFolder/subDirName/file.pdf.') - include_path_type = CLIArgumentType(arg_group='Additional Flags', help='Include only these paths. This option does ' - 'not support wildcard characters (*). Checks relative path prefix. For example:' - 'myFolder;myFolder/subDirName/file.pdf') - recursive_type = CLIArgumentType(options_list=['--recursive', '-r'], action='store_true', - help='Look into sub-directories recursively.') - sas_help = 'The permissions the SAS grants. Allowed values: {}. Do not use if a stored access policy is ' \ - 'referenced with --id that specifies this value. Can be combined.' - t_routing_choice = self.get_models('RoutingChoice', resource_type=ResourceType.MGMT_STORAGE) - routing_choice_type = CLIArgumentType( - arg_group='Routing Preference', arg_type=get_enum_type(t_routing_choice), - help='Routing Choice defines the kind of network routing opted by the user.', - is_preview=True, min_api='2019-06-01') - publish_microsoft_endpoints_type = CLIArgumentType( - arg_group='Routing Preference', arg_type=get_three_state_flag(), is_preview=True, min_api='2019-06-01', - help='A boolean flag which indicates whether microsoft routing storage endpoints are to be published.') - publish_internet_endpoints_type = CLIArgumentType( - arg_group='Routing Preference', arg_type=get_three_state_flag(), is_preview=True, min_api='2019-06-01', - help='A boolean flag which indicates whether internet routing storage endpoints are to be published.') - - umask_type = CLIArgumentType( - help='When creating a file or directory and the parent folder does not have a default ACL, the umask restricts ' - 'the permissions of the file or directory to be created. The resulting permission is given by p & ^u, ' - 'where p is the permission and u is the umask. For more information, please refer to ' - 'https://learn.microsoft.com/azure/storage/blobs/data-lake-storage-access-control#umask.') - permissions_type = CLIArgumentType( - help='POSIX access permissions for the file owner, the file owning group, and others. Each class may be ' - 'granted read, write, or execute permission. The sticky bit is also supported. Both symbolic (rwxrw-rw-) ' - 'and 4-digit octal notation (e.g. 0766) are supported. For more information, please refer to https://' - 'learn.microsoft.com/azure/storage/blobs/data-lake-storage-access-control#levels-of-permission.') - - timeout_type = CLIArgumentType( - help='Request timeout in seconds. Applies to each call to the service.', type=int - ) - - with self.argument_context('storage') as c: - c.argument('container_name', container_name_type) - c.argument('directory_name', directory_type) - c.argument('share_name', share_name_type) - c.argument('table_name', table_name_type) - c.argument('retry_wait', options_list=('--retry-interval',)) - c.ignore('progress_callback') - c.argument('metadata', nargs='+', - help='Metadata in space-separated key=value pairs. This overwrites any existing metadata.', - validator=validate_metadata) - c.argument('timeout', help='Request timeout in seconds. Applies to each call to the service.', type=int) - - with self.argument_context('storage', arg_group='Precondition') as c: - c.argument('if_modified_since', - help='Commence only if modified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')', - type=get_datetime_type(False)) - c.argument('if_unmodified_since', - help='Commence only if unmodified since supplied UTC datetime (Y-m-d\'T\'H:M\'Z\')', - type=get_datetime_type(False)) - c.argument('if_match') - c.argument('if_none_match') - - for item in ['delete', 'show', 'update', 'show-connection-string', 'keys', 'network-rule', 'revoke-delegation-keys', 'failover']: # pylint: disable=line-too-long - with self.argument_context('storage account {}'.format(item)) as c: - c.argument('account_name', acct_name_type, options_list=['--name', '-n']) - c.argument('resource_group_name', required=False, validator=process_resource_group) - - with self.argument_context('storage account check-name') as c: - c.argument('name', options_list=['--name', '-n'], - help='The name of the storage account within the specified resource group') - - with self.argument_context('storage account delete') as c: - c.argument('account_name', acct_name_type, options_list=['--name', '-n'], local_context_attribute=None) - - with self.argument_context('storage account create', resource_type=ResourceType.MGMT_STORAGE) as c: - t_account_type, t_sku_name, t_kind = self.get_models('AccountType', 'SkuName', 'Kind', - resource_type=ResourceType.MGMT_STORAGE) - - c.register_common_storage_account_options() - c.argument('location', get_location_type(self.cli_ctx), validator=get_default_location_from_resource_group) - c.argument('account_type', help='The storage account type', arg_type=get_enum_type(t_account_type)) - c.argument('account_name', acct_name_type, options_list=['--name', '-n'], completer=None, - local_context_attribute=LocalContextAttribute( - name='storage_account_name', actions=[LocalContextAction.SET], scopes=[ALL])) - # Azure Stack always requires default kind is Storage - c.argument('kind', help='Indicate the type of storage account.', - arg_type=get_enum_type(t_kind), default='Storage') - c.argument('https_only', arg_type=get_three_state_flag(), min_api='2019-04-01', - help='Allow https traffic only to storage service if set to true. The default value is true.') - c.argument('https_only', arg_type=get_three_state_flag(), max_api='2018-11-01', - help='Allow https traffic only to storage service if set to true. The default value is false.') - c.argument('tags', tags_type) - c.argument('custom_domain', help='User domain assigned to the storage account. Name is the CNAME source.') - c.argument('sku', help='The storage account SKU.', arg_type=get_enum_type(t_sku_name, default='standard_ragrs')) - c.argument('enable_files_aadds', aadds_type) - c.argument('enable_files_adds', adds_type) - c.argument('enable_large_file_share', arg_type=large_file_share_type) - c.argument('domain_name', domain_name_type) - c.argument('net_bios_domain_name', net_bios_domain_name_type) - c.argument('forest_name', forest_name_type) - c.argument('domain_guid', domain_guid_type) - c.argument('domain_sid', domain_sid_type) - c.argument('azure_storage_sid', azure_storage_sid_type) - c.argument('enable_hierarchical_namespace', arg_type=get_three_state_flag(), - options_list=['--enable-hierarchical-namespace', '--hns'], - help=" Allow the blob service to exhibit filesystem semantics. This property can be enabled only " - "when storage account kind is StorageV2.", - min_api='2018-02-01') - c.argument('encryption_key_type_for_table', arg_type=get_enum_type(['Account', 'Service']), - help='Set the encryption key type for Table service. "Account": Table will be encrypted ' - 'with account-scoped encryption key. "Service": Table will always be encrypted with ' - 'service-scoped keys. Currently the default encryption key type is "Service".', - min_api='2019-06-01', options_list=['--encryption-key-type-for-table', '-t']) - c.argument('encryption_key_type_for_queue', arg_type=get_enum_type(['Account', 'Service']), - help='Set the encryption key type for Queue service. "Account": Queue will be encrypted ' - 'with account-scoped encryption key. "Service": Queue will always be encrypted with ' - 'service-scoped keys. Currently the default encryption key type is "Service".', - min_api='2019-06-01', options_list=['--encryption-key-type-for-queue', '-q']) - c.argument('routing_choice', routing_choice_type) - c.argument('publish_microsoft_endpoints', publish_microsoft_endpoints_type) - c.argument('publish_internet_endpoints', publish_internet_endpoints_type) - - with self.argument_context('storage account private-endpoint-connection', - resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('private_endpoint_connection_name', options_list=['--name', '-n'], - help='The name of the private endpoint connection associated with the Storage Account.') - for item in ['approve', 'reject', 'show', 'delete']: - with self.argument_context('storage account private-endpoint-connection {}'.format(item), - resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('private_endpoint_connection_name', options_list=['--name', '-n'], required=False, - help='The name of the private endpoint connection associated with the Storage Account.') - c.extra('connection_id', options_list=['--id'], - help='The ID of the private endpoint connection associated with the Storage Account. You can get ' - 'it using `az storage account show`.') - c.argument('account_name', help='The storage account name.', required=False) - c.argument('resource_group_name', help='The resource group name of specified storage account.', - required=False) - c.argument('description', help='Comments for {} operation.'.format(item)) - - with self.argument_context('storage account update', resource_type=ResourceType.MGMT_STORAGE) as c: - c.register_common_storage_account_options() - c.argument('custom_domain', - help='User domain assigned to the storage account. Name is the CNAME source. Use "" to clear ' - 'existing value.', - validator=validate_custom_domain) - c.argument('use_subdomain', help='Specify whether to use indirect CNAME validation.', - arg_type=get_enum_type(['true', 'false'])) - c.argument('tags', tags_type, default=None) - c.argument('enable_files_aadds', aadds_type) - c.argument('enable_files_adds', adds_type) - c.argument('enable_large_file_share', arg_type=large_file_share_type) - c.argument('domain_name', domain_name_type) - c.argument('net_bios_domain_name', net_bios_domain_name_type) - c.argument('forest_name', forest_name_type) - c.argument('domain_guid', domain_guid_type) - c.argument('domain_sid', domain_sid_type) - c.argument('azure_storage_sid', azure_storage_sid_type) - c.argument('routing_choice', routing_choice_type) - c.argument('publish_microsoft_endpoints', publish_microsoft_endpoints_type) - c.argument('publish_internet_endpoints', publish_internet_endpoints_type) - - with self.argument_context('storage account update', arg_group='Customer managed key', min_api='2017-06-01') as c: - t_key_source = self.get_models('KeySource', resource_type=ResourceType.MGMT_STORAGE) - c.argument('encryption_key_name', help='The name of the KeyVault key.', ) - c.argument('encryption_key_vault', help='The Uri of the KeyVault.') - c.argument('encryption_key_version', - help='The version of the KeyVault key to use, which will opt out of implicit key rotation. ' - 'Please use "" to opt in key auto-rotation again.') - c.argument('encryption_key_source', - arg_type=get_enum_type(t_key_source), - help='The default encryption key source', - validator=validate_encryption_source) - - for scope in ['storage account create', 'storage account update']: - with self.argument_context(scope, resource_type=ResourceType.MGMT_STORAGE, min_api='2017-06-01', - arg_group='Network Rule') as c: - t_bypass, t_default_action = self.get_models('Bypass', 'DefaultAction', - resource_type=ResourceType.MGMT_STORAGE) - - c.argument('bypass', nargs='+', validator=validate_bypass, arg_type=get_enum_type(t_bypass), - help='Bypass traffic for space-separated uses.') - c.argument('default_action', arg_type=get_enum_type(t_default_action), - help='Default action to apply when no rule matches.') - - with self.argument_context('storage account show-connection-string') as c: - c.argument('protocol', help='The default endpoint protocol.', arg_type=get_enum_type(['http', 'https'])) - c.argument('sas_token', help='The SAS token to be used in the connection-string.') - c.argument('key_name', options_list=['--key'], help='The key to use.', - arg_type=get_enum_type(list(storage_account_key_options.keys()))) - for item in ['blob', 'file', 'queue', 'table']: - c.argument('{}_endpoint'.format(item), help='Custom endpoint for {}s.'.format(item)) - - with self.argument_context('storage account encryption-scope') as c: - c.argument('account_name', help='The storage account name.') - c.argument('resource_group_name', validator=process_resource_group, required=False) - c.argument('encryption_scope_name', options_list=['--name', '-n'], - help='The name of the encryption scope within the specified storage account.') - - for scope in ['storage account encryption-scope create', 'storage account encryption-scope update']: - with self.argument_context(scope, resource_type=ResourceType.MGMT_STORAGE) as c: - from ._validators import validate_encryption_key - t_encryption_key_source = self.get_models('EncryptionScopeSource', resource_type=ResourceType.MGMT_STORAGE) - c.argument('key_source', options_list=['-s', '--key-source'], - arg_type=get_enum_type(t_encryption_key_source, default="Microsoft.Storage"), - help='The provider for the encryption scope.', validator=validate_encryption_key) - c.argument('key_uri', options_list=['-u', '--key-uri'], - help='The object identifier for a key vault key object. When applied, the encryption scope will ' - 'use the key referenced by the identifier to enable customer-managed key support on this ' - 'encryption scope.') - - with self.argument_context('storage account encryption-scope update') as c: - t_state = self.get_models("EncryptionScopeState", resource_type=ResourceType.MGMT_STORAGE) - c.argument('key_source', options_list=['-s', '--key-source'], - arg_type=get_enum_type(t_encryption_key_source), - help='The provider for the encryption scope.', validator=validate_encryption_key) - c.argument('state', arg_type=get_enum_type(t_state), - help='Change the state the encryption scope. When disabled, ' - 'all blob read/write operations using this encryption scope will fail.') - - with self.argument_context('storage account keys list', resource_type=ResourceType.MGMT_STORAGE) as c: - t_expand_key_type = self.get_models('ListKeyExpand', resource_type=ResourceType.MGMT_STORAGE) - c.argument("expand", options_list=['--expand-key-type'], help='Specify the expanded key types to be listed.', - arg_type=get_enum_type(t_expand_key_type), min_api='2019-04-01', is_preview=True) - - with self.argument_context('storage account keys renew', resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('key_name', options_list=['--key'], help='The key options to regenerate.', - arg_type=get_enum_type(list(storage_account_key_options.keys()))) - c.extra('key_type', help='The key type to regenerate. If --key-type is not specified, one of access keys will ' - 'be regenerated by default.', arg_type=get_enum_type(['kerb']), min_api='2019-04-01') - c.argument('account_name', acct_name_type, id_part=None) - - with self.argument_context('storage account management-policy create') as c: - c.argument('policy', type=file_type, completer=FilesCompleter(), - help='The Storage Account ManagementPolicies Rules, in JSON format. See more details in: ' - 'https://learn.microsoft.com/azure/storage/common/storage-lifecycle-managment-concepts.') - - for item in ['create', 'update', 'show', 'delete']: - with self.argument_context('storage account management-policy {}'.format(item)) as c: - c.argument('account_name', help='The name of the storage account within the specified resource group.') - - with self.argument_context('storage account keys list') as c: - c.argument('account_name', acct_name_type, id_part=None) - - with self.argument_context('storage account network-rule') as c: - from ._validators import validate_subnet - from ._validators import validate_ip_address - c.argument('account_name', acct_name_type, id_part=None) - c.argument('ip_address', nargs='*', help='IPv4 address or CIDR range. Can supply a list: --ip-address ip1 ' - '[ip2]...', validator=validate_ip_address) - c.argument('subnet', help='Name or ID of subnet. If name is supplied, `--vnet-name` must be supplied.') - c.argument('vnet_name', help='Name of a virtual network.', validator=validate_subnet) - c.argument('action', help='The action of virtual network rule.') - - with self.argument_context('storage account blob-service-properties show', - resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('account_name', acct_name_type, id_part=None) - c.argument('resource_group_name', required=False, validator=process_resource_group) - - with self.argument_context('storage account blob-service-properties update', - resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('account_name', acct_name_type, id_part=None) - c.argument('resource_group_name', required=False, validator=process_resource_group) - c.argument('enable_change_feed', arg_type=get_three_state_flag(), min_api='2019-04-01') - c.argument('enable_delete_retention', arg_type=get_three_state_flag(), arg_group='Delete Retention Policy', - min_api='2018-07-01') - c.argument('delete_retention_days', type=int, arg_group='Delete Retention Policy', - validator=validator_delete_retention_days, min_api='2018-07-01') - c.argument('enable_restore_policy', arg_type=get_three_state_flag(), arg_group='Restore Policy', - min_api='2019-06-01', help="Enable blob restore policy when it set to true.") - c.argument('restore_days', type=int, arg_group='Restore Policy', - min_api='2019-06-01', help="The number of days for the blob can be restored. It should be greater " - "than zero and less than Delete Retention Days.") - c.argument('enable_versioning', arg_type=get_three_state_flag(), help='Versioning is enabled if set to true.', - min_api='2019-06-01') - - with self.argument_context('storage account file-service-properties show', - resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('account_name', acct_name_type, id_part=None) - c.argument('resource_group_name', required=False, validator=process_resource_group) - - with self.argument_context('storage account file-service-properties update', - resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('account_name', acct_name_type, id_part=None) - c.argument('resource_group_name', required=False, validator=process_resource_group) - c.argument('enable_delete_retention', arg_type=get_three_state_flag(), arg_group='Delete Retention Policy', - min_api='2019-06-01', help='Enable file service properties for share soft delete.') - c.argument('delete_retention_days', type=int, arg_group='Delete Retention Policy', - validator=validate_delete_retention_days, min_api='2019-06-01', - help=' Indicate the number of days that the deleted item should be retained. The minimum specified ' - 'value can be 1 and the maximum value can be 365.') - - with self.argument_context('storage account generate-sas') as c: - t_account_permissions = self.get_sdk('common.models#AccountPermissions') - c.register_sas_arguments() - c.argument('services', type=services_type(self)) - c.argument('resource_types', type=resource_type_type(self)) - c.argument('expiry', type=get_datetime_type(True)) - c.argument('start', type=get_datetime_type(True)) - c.argument('account_name', acct_name_type, options_list=['--account-name']) - c.argument('permission', options_list=('--permissions',), - help='The permissions the SAS grants. Allowed values: {}. Can be combined.'.format( - get_permission_help_string(t_account_permissions)), - validator=get_permission_validator(t_account_permissions)) - c.ignore('sas_token') - - for item in ['show', 'off']: - with self.argument_context('storage logging {}'.format(item)) as c: - c.extra('services', validator=get_char_options_validator('bqt', 'services'), default='bqt') - - with self.argument_context('storage logging update') as c: - c.extra('services', validator=get_char_options_validator('bqt', 'services'), options_list='--services', - required=True) - c.argument('log', validator=get_char_options_validator('rwd', 'log')) - c.argument('retention', type=int) - c.argument('version', type=float) - - with self.argument_context('storage metrics show') as c: - c.extra('services', validator=get_char_options_validator('bfqt', 'services'), default='bfqt') - c.argument('interval', arg_type=get_enum_type(['hour', 'minute', 'both'])) - - with self.argument_context('storage metrics update') as c: - c.extra('services', validator=get_char_options_validator('bfqt', 'services'), options_list='--services', - required=True) - c.argument('hour', validator=process_metric_update_namespace, arg_type=get_enum_type(['true', 'false'])) - c.argument('minute', arg_type=get_enum_type(['true', 'false'])) - c.argument('api', arg_type=get_enum_type(['true', 'false'])) - c.argument('retention', type=int) - - with self.argument_context('storage blob') as c: - c.argument('blob_name', options_list=('--name', '-n'), arg_type=blob_name_type) - c.argument('destination_path', help='The destination path that will be prepended to the blob name.') - - with self.argument_context('storage blob list') as c: - c.argument('include', validator=validate_included_datasets) - c.argument('num_results', arg_type=num_results_type) - - with self.argument_context('storage blob generate-sas') as c: - from .completers import get_storage_acl_name_completion_list - - t_blob_permissions = self.get_sdk('blob.models#BlobPermissions') - c.register_sas_arguments() - c.argument('cache_control', help='Response header value for Cache-Control when resource is accessed' - 'using this shared access signature.') - c.argument('content_disposition', help='Response header value for Content-Disposition when resource is accessed' - 'using this shared access signature.') - c.argument('content_encoding', help='Response header value for Content-Encoding when resource is accessed' - 'using this shared access signature.') - c.argument('content_language', help='Response header value for Content-Language when resource is accessed' - 'using this shared access signature.') - c.argument('content_type', help='Response header value for Content-Type when resource is accessed' - 'using this shared access signature.') - c.argument('full_uri', action='store_true', - help='Indicates that this command return the full blob URI and the shared access signature token.') - c.argument('as_user', min_api='2018-11-09', action='store_true', - validator=as_user_validator, - help="Indicates that this command return the SAS signed with the user delegation key. " - "The expiry parameter and '--auth-mode login' are required if this argument is specified. ") - c.argument('id', options_list='--policy-name', - help='The name of a stored access policy within the container\'s ACL.', - completer=get_storage_acl_name_completion_list(t_base_blob_service, 'container_name', - 'get_container_acl')) - c.argument('permission', options_list='--permissions', - help=sas_help.format(get_permission_help_string(t_blob_permissions)), - validator=get_permission_validator(t_blob_permissions)) - - with self.argument_context('storage blob restore', resource_type=ResourceType.MGMT_STORAGE) as c: - from ._validators import BlobRangeAddAction - c.argument('blob_ranges', options_list=['--blob-range', '-r'], action=BlobRangeAddAction, nargs='+', - help='Blob ranges to restore. You need to two values to specify start_range and end_range for each ' - 'blob range, e.g. -r blob1 blob2. Note: Empty means account start as start range value, and ' - 'means account end for end range.') - c.argument('account_name', acct_name_type, id_part=None) - c.argument('resource_group_name', required=False, validator=process_resource_group) - c.argument('time_to_restore', type=get_datetime_type(True), options_list=['--time-to-restore', '-t'], - help='Restore blob to the specified time, which should be UTC datetime in (Y-m-d\'T\'H:M:S\'Z\').') - - with self.argument_context('storage blob update') as c: - t_blob_content_settings = self.get_sdk('blob.models#ContentSettings') - c.register_content_settings_argument(t_blob_content_settings, update=True) - - with self.argument_context('storage blob exists') as c: - c.argument('blob_name', required=True) - - with self.argument_context('storage blob url') as c: - c.argument('protocol', arg_type=get_enum_type(['http', 'https'], 'https'), help='Protocol to use.') - c.argument('snapshot', help='An string value that uniquely identifies the snapshot. The value of' - 'this query parameter indicates the snapshot version.') - - with self.argument_context('storage blob set-tier') as c: - from azure.cli.command_modules.storage._validators import blob_tier_validator - - c.argument('blob_type', options_list=('--type', '-t'), arg_type=get_enum_type(('block', 'page'))) - c.argument('tier', validator=blob_tier_validator) - c.argument('timeout', type=int) - - with self.argument_context('storage blob service-properties delete-policy update') as c: - c.argument('enable', arg_type=get_enum_type(['true', 'false']), help='Enables/disables soft-delete.') - c.argument('days_retained', type=int, - help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].') - - with self.argument_context('storage blob service-properties update', min_api='2018-03-28') as c: - c.argument('delete_retention', arg_type=get_three_state_flag(), arg_group='Soft Delete', - help='Enables soft-delete.') - c.argument('delete_retention_period', type=int, arg_group='Soft Delete', - help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].') - c.argument('static_website', arg_group='Static Website', arg_type=get_three_state_flag(), - help='Enables static-website.') - c.argument('index_document', help='Represents the name of the index document. This is commonly "index.html".', - arg_group='Static Website') - c.argument('error_document_404_path', options_list=['--404-document'], arg_group='Static Website', - help='Represents the path to the error document that should be shown when an error 404 is issued,' - ' in other words, when a browser requests a page that does not exist.') - - with self.argument_context('storage blob upload') as c: - from ._validators import page_blob_tier_validator, validate_encryption_scope_client_params - from .sdkutil import get_blob_types, get_blob_tier_names - - t_blob_content_settings = self.get_sdk('blob.models#ContentSettings') - c.register_content_settings_argument(t_blob_content_settings, update=False) - - c.argument('file_path', options_list=('--file', '-f'), type=file_type, completer=FilesCompleter()) - c.argument('max_connections', type=int) - c.argument('blob_type', options_list=('--type', '-t'), validator=validate_blob_type, - arg_type=get_enum_type(get_blob_types())) - c.argument('validate_content', action='store_true', min_api='2016-05-31') - c.extra('no_progress', progress_type) - c.extra('socket_timeout', socket_timeout_type) - # TODO: Remove once #807 is complete. Smart Create Generation requires this parameter. - # register_extra_cli_argument('storage blob upload', '_subscription_id', options_list=('--subscription',), - # help=argparse.SUPPRESS) - c.argument('tier', validator=page_blob_tier_validator, - arg_type=get_enum_type(get_blob_tier_names(self.cli_ctx, 'PremiumPageBlobTier')), - min_api='2017-04-17') - c.argument('encryption_scope', validator=validate_encryption_scope_client_params, - help='A predefined encryption scope used to encrypt the data on the service.') - - with self.argument_context('storage blob upload-batch') as c: - from .sdkutil import get_blob_types - - t_blob_content_settings = self.get_sdk('blob.models#ContentSettings') - c.register_content_settings_argument(t_blob_content_settings, update=False, arg_group='Content Control') - c.ignore('source_files', 'destination_container_name') - - c.argument('source', options_list=('--source', '-s')) - c.argument('destination', options_list=('--destination', '-d')) - c.argument('max_connections', type=int, - help='Maximum number of parallel connections to use when the blob size exceeds 64MB.') - c.argument('maxsize_condition', arg_group='Content Control') - c.argument('validate_content', action='store_true', min_api='2016-05-31', arg_group='Content Control') - c.argument('blob_type', options_list=('--type', '-t'), arg_type=get_enum_type(get_blob_types())) - c.extra('no_progress', progress_type) - c.extra('socket_timeout', socket_timeout_type) - - with self.argument_context('storage blob download') as c: - c.argument('file_path', options_list=('--file', '-f'), type=file_type, completer=FilesCompleter()) - c.argument('max_connections', type=int) - c.argument('start_range', type=int) - c.argument('end_range', type=int) - c.argument('validate_content', action='store_true', min_api='2016-05-31') - c.extra('no_progress', progress_type) - c.extra('socket_timeout', socket_timeout_type) - - with self.argument_context('storage blob download-batch') as c: - c.ignore('source_container_name') - c.argument('destination', options_list=('--destination', '-d')) - c.argument('source', options_list=('--source', '-s')) - c.extra('no_progress', progress_type) - c.extra('socket_timeout', socket_timeout_type) - c.argument('max_connections', type=int, - help='Maximum number of parallel connections to use when the blob size exceeds 64MB.') - - with self.argument_context('storage blob delete') as c: - from .sdkutil import get_delete_blob_snapshot_type_names - c.argument('delete_snapshots', arg_type=get_enum_type(get_delete_blob_snapshot_type_names())) - - with self.argument_context('storage blob delete-batch') as c: - c.ignore('source_container_name') - c.argument('source', options_list=('--source', '-s')) - c.argument('delete_snapshots', arg_type=get_enum_type(get_delete_blob_snapshot_type_names()), - help='Required if the blob has associated snapshots.') - c.argument('lease_id', help='The active lease id for the blob.') - - with self.argument_context('storage blob lease') as c: - c.argument('lease_duration', type=int) - c.argument('lease_break_period', type=int) - c.argument('blob_name', arg_type=blob_name_type) - - with self.argument_context('storage copy') as c: - c.argument('destination', options_list=['--destination', '-d'], help="The path/url of copy destination. " - "It can be a local path, an url to azure storage server. If you provide destination parameter " - "here, you do not need to provide arguments in copy destination arguments group and copy " - "destination arguments will be deprecated in future.") - c.argument('source', options_list=['--source', '-s'], help="The path/url of copy source. It can be a local" - " path, an url to azure storage server or AWS S3 buckets. If you provide source parameter here," - " you do not need to provide arguments in copy source arguments group and copy source arguments" - " will be deprecated in future.") - for item in ['destination', 'source']: - c.argument('{}_account_name'.format(item), arg_group='Copy {}'.format(item), - help='Storage account name of copy {}'.format(item)) - c.argument('{}_container'.format(item), arg_group='Copy {}'.format(item), - help='Container name of copy {} storage account'.format(item)) - c.argument('{}_blob'.format(item), arg_group='Copy {}'.format(item), - help='Blob name in blob container of copy {} storage account'.format(item)) - c.argument('{}_share'.format(item), arg_group='Copy {}'.format(item), - help='File share name of copy {} storage account'.format(item)) - c.argument('{}_file_path'.format(item), arg_group='Copy {}'.format(item), - help='File path in file share of copy {} storage account'.format(item)) - c.argument('{}_local_path'.format(item), arg_group='Copy {}'.format(item), - help='Local file path') - c.argument('put_md5', arg_group='Additional Flags', action='store_true', - help='Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the ' - 'destination blob/file.Only available when uploading.') - c.argument('blob_type', arg_group='Additional Flags', - arg_type=get_enum_type(["BlockBlob", "PageBlob", "AppendBlob"]), - help='The type of blob at the destination.') - c.argument('preserve_s2s_access_tier', arg_group='Additional Flags', arg_type=get_three_state_flag(), - help='Preserve access tier during service to service copy. ' - 'Please refer to https://learn.microsoft.com/azure/storage/blobs/storage-blob-storage-tiers ' - 'to ensure destination storage account support setting access tier. In the cases that setting ' - 'access tier is not supported, please use `--preserve-s2s-access-tier false` to bypass copying ' - 'access tier. (Default true)') - c.argument('exclude_pattern', exclude_pattern_type) - c.argument('include_pattern', include_pattern_type) - c.argument('exclude_path', exclude_path_type) - c.argument('include_path', include_path_type) - c.argument('recursive', recursive_type) - c.argument('content_type', arg_group='Additional Flags', help="Specify content type of the file. ") - c.argument('follow_symlinks', arg_group='Additional Flags', action='store_true', - help='Follow symbolic links when uploading from local file system.') - c.argument('cap_mbps', arg_group='Additional Flags', help="Cap the transfer rate, in megabits per second. " - "Moment-by-moment throughput might vary slightly from the cap. " - "If this option is set to zero, or it is omitted, the throughput isn't capped. ") - - with self.argument_context('storage blob copy') as c: - for item in ['destination', 'source']: - c.argument('{}_if_modified_since'.format(item), arg_group='Pre-condition') - c.argument('{}_if_unmodified_since'.format(item), arg_group='Pre-condition') - c.argument('{}_if_match'.format(item), arg_group='Pre-condition') - c.argument('{}_if_none_match'.format(item), arg_group='Pre-condition') - c.argument('container_name', container_name_type, options_list=('--destination-container', '-c')) - c.argument('blob_name', blob_name_type, options_list=('--destination-blob', '-b'), - help='Name of the destination blob. If the exists, it will be overwritten.') - c.argument('source_lease_id', arg_group='Copy Source') - - with self.argument_context('storage blob copy start') as c: - from azure.cli.command_modules.storage._validators_azure_stack import validate_source_uri - - c.register_source_uri_arguments(validator=validate_source_uri) - c.argument('requires_sync', arg_type=get_three_state_flag(), - help='Enforce that the service will not return a response until the copy is complete.' - 'Not support for standard page blob.') - - with self.argument_context('storage blob copy start-batch', arg_group='Copy Source') as c: - from azure.cli.command_modules.storage._validators import get_source_file_or_blob_service_client - - c.argument('source_client', ignore_type, validator=get_source_file_or_blob_service_client) - - c.extra('source_account_name') - c.extra('source_account_key') - c.extra('source_uri') - c.argument('source_sas') - c.argument('source_container') - c.argument('source_share') - - with self.argument_context('storage blob incremental-copy start') as c: - from azure.cli.command_modules.storage._validators import process_blob_source_uri - - c.register_source_uri_arguments(validator=process_blob_source_uri, blob_only=True) - c.argument('destination_if_modified_since', arg_group='Pre-condition') - c.argument('destination_if_unmodified_since', arg_group='Pre-condition') - c.argument('destination_if_match', arg_group='Pre-condition') - c.argument('destination_if_none_match', arg_group='Pre-condition') - c.argument('container_name', container_name_type, options_list=('--destination-container', '-c')) - c.argument('blob_name', blob_name_type, options_list=('--destination-blob', '-b'), - help='Name of the destination blob. If the exists, it will be overwritten.') - c.argument('source_lease_id', arg_group='Copy Source') - - with self.argument_context('storage blob sync') as c: - c.extra('destination_container', options_list=['--container', '-c'], required=True, - help='The sync destination container.') - c.extra('destination_path', options_list=['--destination', '-d'], - validator=validate_azcopy_upload_destination_url, - help='The sync destination path.') - c.argument('source', options_list=['--source', '-s'], - help='The source file path to sync from.') - c.ignore('destination') - c.argument('exclude_pattern', exclude_pattern_type) - c.argument('include_pattern', include_pattern_type) - c.argument('exclude_path', exclude_path_type) - c.positional('extra_options', nargs='*', is_experimental=True, default=[], - help="Other options which will be passed through to azcopy as it is. " - "Please put all the extra options after a `--`") - - with self.argument_context('storage container') as c: - from .sdkutil import get_container_access_type_names - c.argument('container_name', container_name_type, options_list=('--name', '-n')) - c.argument('public_access', validator=validate_container_public_access, - arg_type=get_enum_type(get_container_access_type_names()), - help='Specifies whether data in the container may be accessed publicly.') - - with self.argument_context('storage container create') as c: - c.argument('container_name', container_name_type, options_list=('--name', '-n'), completer=None) - c.argument('fail_on_exist', help='Throw an exception if the container already exists.') - c.argument('account_name', help='Storage account name. Related environment variable: AZURE_STORAGE_ACCOUNT.') - c.argument('default_encryption_scope', options_list=['--default-encryption-scope', '-d'], - arg_group='Encryption Policy', is_preview=True, - help='Default the container to use specified encryption scope for all writes.') - c.argument('prevent_encryption_scope_override', options_list=['--prevent-encryption-scope-override', '-p'], - arg_type=get_three_state_flag(), arg_group='Encryption Policy', is_preview=True, - help='Block override of encryption scope from the container default.') - - with self.argument_context('storage container delete') as c: - c.argument('fail_not_exist', help='Throw an exception if the container does not exist.') - c.argument('bypass_immutability_policy', action='store_true', help='Bypasses upcoming service behavior that ' - 'will block a container from being deleted if it has a immutability-policy. Specifying this will ' - 'ignore arguments aside from those used to identify the container ("--name", "--account-name").') - c.argument('lease_id', help="If specified, delete_container only succeeds if the container's lease is active " - "and matches this ID. Required if the container has an active lease.") - c.ignore('processed_resource_group') - c.ignore('processed_account_name') - c.ignore('mgmt_client') - - with self.argument_context('storage container exists') as c: - c.ignore('blob_name', 'snapshot') - - for item in ['create', 'extend']: - with self.argument_context('storage container immutability-policy {}'.format(item)) as c: - c.extra('allow_protected_append_writes', options_list=['--allow-protected-append-writes', '-w'], - arg_type=get_three_state_flag(), help='This property can only be changed for unlocked time-based ' - 'retention policies. When enabled, new blocks can be ' - 'written to an append blob while maintaining immutability ' - 'protection and compliance. Only new blocks can be added ' - 'and any existing blocks cannot be modified or deleted. ' - 'This property cannot be changed with ' - 'ExtendImmutabilityPolicy API.') - c.extra('period', type=int, help='The immutability period for the blobs in the container since the policy ' - 'creation, in days.') - c.ignore('parameters') - - with self.argument_context('storage container list') as c: - c.argument('num_results', arg_type=num_results_type) - - with self.argument_context('storage container set-permission') as c: - c.ignore('signed_identifiers') - - with self.argument_context('storage container lease') as c: - c.argument('container_name', container_name_type) - - with self.argument_context('storage container') as c: - c.argument('account_name', completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts')) - c.argument('resource_group_name', required=False, validator=process_resource_group) - - with self.argument_context('storage container immutability-policy') as c: - c.argument('immutability_period_since_creation_in_days', options_list='--period') - c.argument('container_name', container_name_type) - - with self.argument_context('storage container legal-hold') as c: - c.argument('container_name', container_name_type) - c.argument('tags', nargs='+', - help='Space-separated tags. Each tag should be 3 to 23 alphanumeric characters and is normalized ' - 'to lower case') - - with self.argument_context('storage container policy') as c: - from .completers import get_storage_acl_name_completion_list - t_container_permissions = self.get_sdk('blob.models#ContainerPermissions') - - c.argument('container_name', container_name_type) - c.argument('policy_name', options_list=('--name', '-n'), help='The stored access policy name.', - completer=get_storage_acl_name_completion_list(t_base_blob_service, 'container_name', - 'get_container_acl')) - help_str = 'Allowed values: {}. Can be combined'.format(get_permission_help_string(t_container_permissions)) - c.argument('permission', options_list='--permissions', help=help_str, - validator=get_permission_validator(t_container_permissions)) - - c.argument('start', type=get_datetime_type(True), - help='start UTC datetime (Y-m-d\'T\'H:M:S\'Z\'). Defaults to time of request.') - c.argument('expiry', type=get_datetime_type(True), help='expiration UTC datetime in (Y-m-d\'T\'H:M:S\'Z\')') - - for item in ['create', 'delete', 'list', 'show', 'update']: - with self.argument_context('storage container policy {}'.format(item)) as c: - c.extra('lease_id', options_list='--lease-id', help='The container lease ID.') - - with self.argument_context('storage container generate-sas') as c: - from .completers import get_storage_acl_name_completion_list - t_container_permissions = self.get_sdk('blob.models#ContainerPermissions') - c.register_sas_arguments() - c.argument('id', options_list='--policy-name', - help='The name of a stored access policy within the container\'s ACL.', - completer=get_storage_acl_name_completion_list(t_container_permissions, 'container_name', - 'get_container_acl')) - c.argument('permission', options_list='--permissions', - help=sas_help.format(get_permission_help_string(t_container_permissions)), - validator=get_permission_validator(t_container_permissions)) - c.argument('cache_control', help='Response header value for Cache-Control when resource is accessed' - 'using this shared access signature.') - c.argument('content_disposition', help='Response header value for Content-Disposition when resource is accessed' - 'using this shared access signature.') - c.argument('content_encoding', help='Response header value for Content-Encoding when resource is accessed' - 'using this shared access signature.') - c.argument('content_language', help='Response header value for Content-Language when resource is accessed' - 'using this shared access signature.') - c.argument('content_type', help='Response header value for Content-Type when resource is accessed' - 'using this shared access signature.') - c.argument('as_user', min_api='2018-11-09', action='store_true', - validator=as_user_validator, - help="Indicates that this command return the SAS signed with the user delegation key. " - "The expiry parameter and '--auth-mode login' are required if this argument is specified. ") - with self.argument_context('storage container lease') as c: - c.argument('lease_duration', type=int) - c.argument('lease_break_period', type=int) - - with self.argument_context('storage share') as c: - c.argument('share_name', share_name_type, options_list=('--name', '-n')) - - for item in ['create', 'delete', 'exists', 'list', 'show', 'update']: - with self.argument_context('storage share-rm {}'.format(item), resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('resource_group_name', required=False) - c.argument('account_name', storage_account_type) - c.argument('share_name', share_name_type, options_list=('--name', '-n'), id_part='child_name_2') - c.argument('share_quota', type=int, options_list='--quota') - c.argument('metadata', nargs='+', - help='Metadata in space-separated key=value pairs that is associated with the share. ' - 'This overwrites any existing metadata', - validator=validate_metadata) - c.argument('expand', default=None) - c.ignore('filter', 'maxpagesize', 'skip_token') - - with self.argument_context('storage share-rm list', resource_type=ResourceType.MGMT_STORAGE) as c: - c.argument('account_name', storage_account_type, id_part=None) - - with self.argument_context('storage share url') as c: - c.argument('unc', action='store_true', help='Output UNC network path.') - c.argument('protocol', arg_type=get_enum_type(['http', 'https'], 'https'), help='Protocol to use.') - - with self.argument_context('storage share list') as c: - c.argument('num_results', arg_type=num_results_type) - - with self.argument_context('storage share exists') as c: - c.ignore('directory_name', 'file_name') - - with self.argument_context('storage share policy') as c: - from .completers import get_storage_acl_name_completion_list - - t_file_svc = self.get_sdk('file#FileService') - t_share_permissions = self.get_sdk('file.models#SharePermissions') - - c.argument('container_name', share_name_type) - c.argument('policy_name', options_list=('--name', '-n'), help='The stored access policy name.', - completer=get_storage_acl_name_completion_list(t_file_svc, 'container_name', 'get_share_acl')) - - help_str = 'Allowed values: {}. Can be combined'.format(get_permission_help_string(t_share_permissions)) - c.argument('permission', options_list='--permissions', help=help_str, - validator=get_permission_validator(t_share_permissions)) - - c.argument('start', type=get_datetime_type(True), - help='start UTC datetime (Y-m-d\'T\'H:M:S\'Z\'). Defaults to time of request.') - c.argument('expiry', type=get_datetime_type(True), help='expiration UTC datetime in (Y-m-d\'T\'H:M:S\'Z\')') - - with self.argument_context('storage share delete') as c: - from .sdkutil import get_delete_file_snapshot_type_names - c.argument('delete_snapshots', arg_type=get_enum_type(get_delete_file_snapshot_type_names()), - help='Specify the deletion strategy when the share has snapshots.') - - with self.argument_context('storage share generate-sas') as c: - from .completers import get_storage_acl_name_completion_list - - t_share_permissions = self.get_sdk('file.models#SharePermissions') - c.register_sas_arguments() - c.argument('id', options_list='--policy-name', - help='The name of a stored access policy within the share\'s ACL.', - completer=get_storage_acl_name_completion_list(t_share_permissions, 'share_name', 'get_share_acl')) - c.argument('permission', options_list='--permissions', - help=sas_help.format(get_permission_help_string(t_share_permissions)), - validator=get_permission_validator(t_share_permissions)) - - with self.argument_context('storage directory') as c: - c.argument('directory_name', directory_type, options_list=('--name', '-n')) - - with self.argument_context('storage directory exists') as c: - c.ignore('file_name') - c.argument('directory_name', required=True) - - with self.argument_context('storage file') as c: - c.argument('file_name', file_name_type, options_list=('--name', '-n')) - c.argument('directory_name', directory_type, required=False) - - with self.argument_context('storage file copy') as c: - c.argument('share_name', share_name_type, options_list=('--destination-share', '-s'), - help='Name of the destination share. The share must exist.') - - with self.argument_context('storage file copy cancel') as c: - c.register_path_argument(options_list=('--destination-path', '-p')) - - with self.argument_context('storage file delete') as c: - c.register_path_argument() - - with self.argument_context('storage file download') as c: - c.register_path_argument() - c.argument('file_path', options_list=('--dest',), type=file_type, required=False, - help='Path of the file to write to. The source filename will be used if not specified.', - validator=process_file_download_namespace, completer=FilesCompleter()) - c.argument('path', validator=None) # validator called manually from process_file_download_namespace - c.extra('no_progress', progress_type) - c.argument('max_connections', type=int) - c.argument('start_range', type=int) - c.argument('end_range', type=int) - - with self.argument_context('storage file exists') as c: - c.register_path_argument() - - with self.argument_context('storage file generate-sas') as c: - from .completers import get_storage_acl_name_completion_list - - c.register_path_argument() - c.register_sas_arguments() - - t_file_svc = self.get_sdk('file.fileservice#FileService') - t_file_permissions = self.get_sdk('file.models#FilePermissions') - c.argument('id', options_list='--policy-name', - help='The name of a stored access policy within the container\'s ACL.', - completer=get_storage_acl_name_completion_list(t_file_svc, 'container_name', 'get_container_acl')) - c.argument('permission', options_list='--permissions', - help=sas_help.format(get_permission_help_string(t_file_permissions)), - validator=get_permission_validator(t_file_permissions)) - - with self.argument_context('storage file list') as c: - from .completers import dir_path_completer - c.argument('directory_name', options_list=('--path', '-p'), help='The directory path within the file share.', - completer=dir_path_completer) - c.argument('num_results', arg_type=num_results_type) - - with self.argument_context('storage file metadata show') as c: - c.register_path_argument() - - with self.argument_context('storage file metadata update') as c: - c.register_path_argument() - - with self.argument_context('storage file resize') as c: - c.register_path_argument() - c.argument('content_length', options_list='--size') - - with self.argument_context('storage file show') as c: - c.register_path_argument() - - with self.argument_context('storage file update') as c: - t_file_content_settings = self.get_sdk('file.models#ContentSettings') - - c.register_path_argument() - c.register_content_settings_argument(t_file_content_settings, update=True) - - with self.argument_context('storage file upload') as c: - t_file_content_settings = self.get_sdk('file.models#ContentSettings') - - c.register_path_argument(default_file_param='local_file_path') - c.register_content_settings_argument(t_file_content_settings, update=False, guess_from_file='local_file_path') - c.argument('local_file_path', options_list='--source', type=file_type, completer=FilesCompleter()) - c.extra('no_progress', progress_type) - c.argument('max_connections', type=int) - - with self.argument_context('storage file url') as c: - c.register_path_argument() - c.argument('protocol', arg_type=get_enum_type(['http', 'https'], 'https'), help='Protocol to use.') - - with self.argument_context('storage file upload-batch') as c: - from ._validators import process_file_upload_batch_parameters - c.argument('source', options_list=('--source', '-s'), validator=process_file_upload_batch_parameters) - c.argument('destination', options_list=('--destination', '-d')) - c.argument('max_connections', arg_group='Download Control', type=int) - c.argument('validate_content', action='store_true', min_api='2016-05-31') - c.register_content_settings_argument(t_file_content_settings, update=False, arg_group='Content Settings') - c.extra('no_progress', progress_type) - - with self.argument_context('storage file download-batch') as c: - from ._validators import process_file_download_batch_parameters - c.argument('source', options_list=('--source', '-s'), validator=process_file_download_batch_parameters) - c.argument('destination', options_list=('--destination', '-d')) - c.argument('max_connections', arg_group='Download Control', type=int) - c.argument('validate_content', action='store_true', min_api='2016-05-31') - c.extra('no_progress', progress_type) - - with self.argument_context('storage file delete-batch') as c: - from ._validators_azure_stack import process_file_batch_source_parameters - c.argument('source', options_list=('--source', '-s'), validator=process_file_batch_source_parameters) - - with self.argument_context('storage file copy start') as c: - from azure.cli.command_modules.storage._validators_azure_stack import validate_source_uri - - c.register_path_argument(options_list=('--destination-path', '-p')) - c.register_source_uri_arguments(validator=validate_source_uri) - c.extra('file_snapshot', default=None, arg_group='Copy Source', - help='The file snapshot for the source storage account.') - - with self.argument_context('storage file copy start-batch', arg_group='Copy Source') as c: - from ._validators import get_source_file_or_blob_service_client - c.argument('source_client', ignore_type, validator=get_source_file_or_blob_service_client) - c.extra('source_account_name') - c.extra('source_account_key') - c.extra('source_uri') - c.argument('source_sas') - c.argument('source_container') - c.argument('source_share') - - with self.argument_context('storage cors list') as c: - c.extra('services', validator=get_char_options_validator('bfqt', 'services'), default='bqft', - options_list='--services', required=False) - - with self.argument_context('storage cors add') as c: - c.extra('services', validator=get_char_options_validator('bfqt', 'services'), required=True, - options_list='--services') - c.argument('max_age') - c.argument('origins', nargs='+') - c.argument('methods', nargs='+', - arg_type=get_enum_type(['DELETE', 'GET', 'HEAD', 'MERGE', 'POST', 'OPTIONS', 'PUT'])) - c.argument('allowed_headers', nargs='+') - c.argument('exposed_headers', nargs='+') - - with self.argument_context('storage cors clear') as c: - c.extra('services', validator=get_char_options_validator('bfqt', 'services'), required=True, - options_list='--services') - - with self.argument_context('storage queue generate-sas') as c: - from .completers import get_storage_acl_name_completion_list - - t_queue_permissions = self.get_sdk('queue.models#QueuePermissions') - - c.register_sas_arguments() - - c.argument('id', options_list='--policy-name', - help='The name of a stored access policy within the share\'s ACL.', - completer=get_storage_acl_name_completion_list(t_queue_permissions, 'queue_name', 'get_queue_acl')) - c.argument('permission', options_list='--permissions', - help=sas_help.format(get_permission_help_string(t_queue_permissions)), - validator=get_permission_validator(t_queue_permissions)) - - with self.argument_context('storage queue') as c: - c.argument('queue_name', queue_name_type, options_list=('--name', '-n')) - - with self.argument_context('storage queue create') as c: - c.argument('queue_name', queue_name_type, options_list=('--name', '-n'), completer=None) - - with self.argument_context('storage queue policy') as c: - from .completers import get_storage_acl_name_completion_list - - t_queue_permissions = self.get_sdk('queue.models#QueuePermissions') - - c.argument('container_name', queue_name_type) - c.argument('policy_name', options_list=('--name', '-n'), help='The stored access policy name.', - completer=get_storage_acl_name_completion_list(t_queue_service, 'container_name', 'get_queue_acl')) - - help_str = 'Allowed values: {}. Can be combined'.format(get_permission_help_string(t_queue_permissions)) - c.argument('permission', options_list='--permissions', help=help_str, - validator=get_permission_validator(t_queue_permissions)) - - c.argument('start', type=get_datetime_type(True), - help='start UTC datetime (Y-m-d\'T\'H:M:S\'Z\'). Defaults to time of request.') - c.argument('expiry', type=get_datetime_type(True), help='expiration UTC datetime in (Y-m-d\'T\'H:M:S\'Z\')') - - with self.argument_context('storage message') as c: - c.argument('queue_name', queue_name_type) - c.argument('message_id', options_list='--id') - c.argument('content', type=str, help='Message content, up to 64KB in size.') - - with self.argument_context('storage remove') as c: - from .completers import file_path_completer - - c.extra('container_name', container_name_type, validator=validate_azcopy_remove_arguments) - c.extra('blob_name', options_list=('--name', '-n'), arg_type=blob_name_type) - c.extra('share_name', share_name_type, help='The file share name.') - c.extra('path', options_list=('--path', '-p'), - help='The path to the file within the file share.', - completer=file_path_completer) - c.argument('exclude_pattern', exclude_pattern_type) - c.argument('include_pattern', include_pattern_type) - c.argument('exclude_path', exclude_path_type) - c.argument('include_path', include_path_type) - c.argument('recursive', recursive_type) - c.ignore('destination') - c.ignore('service') - c.ignore('target') - - with self.argument_context('storage table') as c: - c.argument('table_name', table_name_type, options_list=('--name', '-n')) - - with self.argument_context('storage table create') as c: - c.argument('table_name', table_name_type, options_list=('--name', '-n'), completer=None) - c.argument('fail_on_exist', help='Throw an exception if the table already exists.') - - with self.argument_context('storage table policy') as c: - from ._validators import table_permission_validator - from .completers import get_storage_acl_name_completion_list - - c.argument('container_name', table_name_type) - c.argument('policy_name', options_list=('--name', '-n'), help='The stored access policy name.', - completer=get_storage_acl_name_completion_list(t_table_service, 'table_name', 'get_table_acl')) - - help_str = 'Allowed values: (r)ead/query (a)dd (u)pdate (d)elete. Can be combined.' - c.argument('permission', options_list='--permissions', help=help_str, validator=table_permission_validator) - - c.argument('start', type=get_datetime_type(True), - help='start UTC datetime (Y-m-d\'T\'H:M:S\'Z\'). Defaults to time of request.') - c.argument('expiry', type=get_datetime_type(True), help='expiration UTC datetime in (Y-m-d\'T\'H:M:S\'Z\')') - - with self.argument_context('storage table generate-sas') as c: - from .completers import get_storage_acl_name_completion_list - - c.register_sas_arguments() - c.argument('id', options_list='--policy-name', - help='The name of a stored access policy within the table\'s ACL.', - completer=get_storage_acl_name_completion_list(t_table_service, 'table_name', 'get_table_acl')) - c.argument('permission', options_list='--permissions', - help=sas_help.format('(r)ead/query (a)dd (u)pdate (d)elete'), - validator=table_permission_validator) - - with self.argument_context('storage entity') as c: - c.ignore('property_resolver') - c.argument('entity', options_list=('--entity', '-e'), validator=validate_entity, nargs='+') - c.argument('select', nargs='+', validator=validate_select, - help='Space-separated list of properties to return for each entity.') - - with self.argument_context('storage entity insert') as c: - c.argument('if_exists', arg_type=get_enum_type(['fail', 'merge', 'replace'])) - - with self.argument_context('storage entity query') as c: - c.argument('accept', default='minimal', validator=validate_table_payload_format, - arg_type=get_enum_type(['none', 'minimal', 'full']), - help='Specifies how much metadata to include in the response payload.') - c.argument('marker', validator=validate_marker, nargs='+') - - for item in ['create', 'show', 'delete', 'exists', 'metadata update', 'metadata show']: - with self.argument_context('storage fs {}'.format(item)) as c: - c.extra('file_system_name', options_list=['--name', '-n'], - help="File system name (i.e. container name).", required=True) - c.extra('timeout', timeout_type) - - with self.argument_context('storage fs create') as c: - from .sdkutil import get_fs_access_type_names - c.argument('public_access', arg_type=get_enum_type(get_fs_access_type_names()), - validator=validate_fs_public_access, - help="Specify whether data in the file system may be accessed publicly and the level of access.") - - with self.argument_context('storage fs list') as c: - c.argument('include_metadata', arg_type=get_three_state_flag(), - help='Specify that file system metadata be returned in the response. The default value is "False".') - c.argument('name_starts_with', options_list=['--prefix'], - help='Filter the results to return only file systems whose names begin with the specified prefix.') - - for item in ['create', 'show', 'delete', 'exists', 'move', 'metadata update', 'metadata show']: - with self.argument_context('storage fs directory {}'.format(item)) as c: - c.extra('file_system_name', options_list=['-f', '--file-system'], - help="File system name (i.e. container name).", required=True) - c.extra('directory_path', options_list=['--name', '-n'], - help="The name of directory.", required=True) - c.extra('timeout', timeout_type) - - with self.argument_context('storage fs directory create') as c: - c.extra('permissions', permissions_type) - c.extra('umask', umask_type) - - with self.argument_context('storage fs directory list') as c: - c.extra('file_system_name', options_list=['-f', '--file-system'], - help="File system name (i.e. container name).", required=True) - c.argument('recursive', arg_type=get_three_state_flag(), default=True, - help='Look into sub-directories recursively when set to true.') - c.argument('path', help="Filter the results to return only paths under the specified path.") - c.argument('num_results', type=int, help='Specify the maximum number of results to return.') - - with self.argument_context('storage fs directory move') as c: - c.argument('new_name', options_list=['--new-directory', '-d'], - help='The new directory name the users want to move to. The value must have the following format: ' - '"{filesystem}/{directory}/{subdirectory}".') - - with self.argument_context('storage fs file list') as c: - c.extra('file_system_name', options_list=['-f', '--file-system'], - help="File system name (i.e. container name).", required=True) - c.argument('recursive', arg_type=get_three_state_flag(), default=True, - help='Look into sub-directories recursively when set to true.') - c.argument('exclude_dir', action='store_true', - help='List only files in the given file system.') - c.argument('path', help='Filter the results to return only paths under the specified path.') - c.argument('num_results', type=int, default=5000, - help='Specify the maximum number of results to return. If the request does not specify num_results ' - 'or specifies a value greater than 5,000, the server will return up to 5,000 items.') - c.argument('marker', - help='An opaque continuation token. This value can be retrieved from the next_marker field of a ' - 'previous generator object. If specified, this generator will begin returning results from this ' - 'point.') - - for item in ['create', 'show', 'delete', 'exists', 'upload', 'append', 'download', 'show', 'metadata update', - 'metadata show']: - with self.argument_context('storage fs file {}'.format(item)) as c: - c.extra('file_system_name', options_list=['-f', '--file-system'], - help='File system name (i.e. container name).', required=True) - c.extra('path', options_list=['-p', '--path'], help="The file path in a file system.", - required=True) - c.extra('timeout', timeout_type) - c.argument('content', help='Content to be appended to file.') - - with self.argument_context('storage fs file create') as c: - t_file_content_settings = self.get_sdk('_models#ContentSettings', - resource_type=ResourceType.DATA_STORAGE_FILEDATALAKE) - c.register_content_settings_argument(t_file_content_settings, update=False) - c.extra('permissions', permissions_type) - c.extra('umask', umask_type) - c.extra('timeout', timeout_type) - - with self.argument_context('storage fs file download') as c: - c.argument('destination_path', options_list=['--destination', '-d'], type=file_type, - help='The local file where the file or folder will be downloaded to. The source filename will be ' - 'used if not specified.') - c.argument('overwrite', arg_type=get_three_state_flag(), - help="Overwrite an existing file when specified. Default value is false.") - - with self.argument_context('storage fs file move') as c: - t_file_content_settings = self.get_sdk('_models#ContentSettings', - resource_type=ResourceType.DATA_STORAGE_FILEDATALAKE) - c.register_content_settings_argument(t_file_content_settings, update=False) - c.extra('file_system_name', options_list=['-f', '--file-system'], - help='File system name (i.e. container name).', required=True) - c.extra('path', options_list=['-p', '--path'], required=True, - help="The original file path users want to move in a file system.") - c.argument('new_name', options_list=['--new-path'], - help='The new path the users want to move to. The value must have the following format: ' - '"{filesystem}/{directory}/{subdirectory}/{file}".') - - with self.argument_context('storage fs file upload') as c: - t_file_content_settings = self.get_sdk('_models#ContentSettings', - resource_type=ResourceType.DATA_STORAGE_FILEDATALAKE) - c.register_content_settings_argument(t_file_content_settings, update=False) - c.argument('local_path', options_list=['--source', '-s'], - help='Path of the local file to upload as the file content.') - c.argument('overwrite', arg_type=get_three_state_flag(), help="Overwrite an existing file when specified.") - c.argument('if_match', arg_group='Precondition', - help="An ETag value, or the wildcard character (*). Specify this header to perform the operation " - "only if the resource's ETag matches the value specified.") - c.argument('if_none_match', arg_group='Precondition', - help="An ETag value, or the wildcard character (*). Specify this header to perform the operation " - "only if the resource's ETag does not match the value specified.") - c.argument('if_modified_since', arg_group='Precondition', - help="A Commence only if modified since supplied UTC datetime (Y-m-d'T'H:M'Z').") - c.argument('if_unmodified_since', arg_group='Precondition', - help="A Commence only if unmodified since supplied UTC datetime (Y-m-d'T'H:M'Z').") - c.argument('permissions', permissions_type) - c.argument('umask', umask_type) - - for item in ['set', 'show']: - with self.argument_context('storage fs access {}'.format(item)) as c: - from ._validators import validate_access_control - c.extra('file_system_name', options_list=['-f', '--file-system'], - help='File system name (i.e. container name).', required=True) - c.extra('directory_path', options_list=['-p', '--path'], - help='The path to a file or directory in the specified file system.', required=True) - c.argument('permissions', validator=validate_access_control) - c.ignore('upn') diff --git a/src/azure-cli/azure/cli/command_modules/storage/_transformers_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/_transformers_azure_stack.py deleted file mode 100644 index eb5ce1d13d8..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/_transformers_azure_stack.py +++ /dev/null @@ -1,232 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- -"""Transform operations for storage commands""" - -import base64 -from knack.log import get_logger -from knack.util import todict -from .track2_util import _encode_bytes -from .url_quote_util import encode_url_path - -storage_account_key_options = {'primary': 'key1', 'secondary': 'key2'} -logger = get_logger(__name__) - - -def transform_acl_list_output(result): - """ Transform to convert SDK output into a form that is more readily - usable by the CLI and tools such as jpterm. """ - from collections import OrderedDict - new_result = [] - for key in sorted(result.keys()): - new_entry = OrderedDict() - new_entry['Name'] = key - new_entry['Start'] = result[key]['start'] - new_entry['Expiry'] = result[key]['expiry'] - new_entry['Permissions'] = result[key]['permission'] - new_result.append(new_entry) - return new_result - - -def transform_container_permission_output(result): - return {'publicAccess': result.public_access or 'off'} - - -def transform_cors_list_output(result): - from collections import OrderedDict - new_result = [] - for service in sorted(result.keys()): - for i, rule in enumerate(result[service]): - new_entry = OrderedDict() - new_entry['Service'] = service - new_entry['Rule'] = i + 1 - - new_entry['AllowedMethods'] = ', '.join(x for x in rule.allowed_methods) - new_entry['AllowedOrigins'] = ', '.join(x for x in rule.allowed_origins) - new_entry['ExposedHeaders'] = ', '.join(x for x in rule.exposed_headers) - new_entry['AllowedHeaders'] = ', '.join(x for x in rule.allowed_headers) - new_entry['MaxAgeInSeconds'] = rule.max_age_in_seconds - new_result.append(new_entry) - return new_result - - -def transform_entity_query_output(result): - from collections import OrderedDict - new_results = [] - ignored_keys = ['etag', 'Timestamp', 'RowKey', 'PartitionKey'] - for row in result['items']: - new_entry = OrderedDict() - new_entry['PartitionKey'] = row['PartitionKey'] - new_entry['RowKey'] = row['RowKey'] - other_keys = sorted([x for x in row.keys() if x not in ignored_keys]) - for key in other_keys: - new_entry[key] = row[key] - new_results.append(new_entry) - return new_results - - -def transform_entities_result(result): - for entity in result.items: - transform_entity_result(entity) - return result - - -def transform_entity_result(entity): - for key in entity.keys(): - entity_property = entity[key] - if hasattr(entity_property, 'value') and isinstance(entity_property.value, bytes): - entity_property.value = base64.b64encode(entity_property.value).decode() - return entity - - -def transform_logging_list_output(result): - from collections import OrderedDict - new_result = [] - for key in sorted(result.keys()): - new_entry = OrderedDict() - new_entry['Service'] = key - new_entry['Read'] = str(result[key]['read']) - new_entry['Write'] = str(result[key]['write']) - new_entry['Delete'] = str(result[key]['delete']) - new_entry['RetentionPolicy'] = str(result[key]['retentionPolicy']['days']) - new_result.append(new_entry) - return new_result - - -def transform_metrics_list_output(result): - from collections import OrderedDict - new_result = [] - for service in sorted(result.keys()): - service_name = service - for interval in sorted(result[service].keys()): - item = result[service][interval] - new_entry = OrderedDict() - new_entry['Service'] = service_name - service_name = '' - new_entry['Interval'] = str(interval) - new_entry['Enabled'] = str(item['enabled']) - new_entry['IncludeApis'] = str(item['includeApis']) - new_entry['RetentionPolicy'] = str(item['retentionPolicy']['days']) - new_result.append(new_entry) - return new_result - - -def create_boolean_result_output_transformer(property_name): - def _transformer(result): - return {property_name: result} - - return _transformer - - -def transform_storage_list_output(result): - if getattr(result, 'next_marker', None): - logger.warning('Next Marker:') - logger.warning(result.next_marker) - return list(result) - - -def transform_url(result): - """ Ensures the resulting URL string does not contain extra / characters """ - import re - result = re.sub('//', '/', result) - result = re.sub('/', '//', result, count=1) - return encode_url_path(result) - - -def transform_fs_access_output(result): - """ Transform to convert SDK output into a form that is more readily - usable by the CLI and tools such as jpterm. """ - - new_result = {} - useful_keys = ['acl', 'group', 'owner', 'permissions'] - for key in useful_keys: - new_result[key] = result[key] - return new_result - - -# TODO: Remove it when SDK is right for file system scenarios -def transform_fs_public_access_output(result): - """ Transform to convert SDK output into a form that is more readily - usable by the CLI and tools such as jpterm. """ - - if result.public_access == 'blob': - result.public_access = 'file' - if result.public_access == 'container': - result.public_access = 'filesystem' - return result - - -# TODO: Remove it when SDK is right for file system scenarios -def transform_fs_list_public_access_output(result): - """ Transform to convert SDK output into a form that is more readily - usable by the CLI and tools such as jpterm. """ - - new_result = list(result) - for i, item in enumerate(new_result): - new_result[i] = transform_fs_public_access_output(item) - return new_result - - -def transform_metadata(result): - return result.metadata - - -# ------------------Track2 Support----------------------- -def _transform_page_ranges(page_ranges): - # in track 2 sdk, page ranges result is tuple(list(dict(str, str), list(dict(str, str)) - if page_ranges and len(page_ranges) == 2: - result = page_ranges[0] if page_ranges[0] else [{}] - result[0]['isCleared'] = bool(page_ranges[1]) - return result - return None - - -def transform_blob_json_output(result): - result = todict(result) - new_result = { - "content": "", - "deleted": result.pop('deleted', None), - "metadata": result.pop('metadata', None), - "name": result.pop('name', None), - "properties": { - "appendBlobCommittedBlockCount": result.pop('appendBlobCommittedBlockCount', None), - "blobTier": result.pop('blobTier', None), - "blobTierChangeTime": result.pop('blobTierChangeTime', None), - "blobTierInferred": result.pop('blobTierInferred', None), - "blobType": result.pop('blobType', None), - "contentLength": result.pop('size', None), - "contentRange": result.pop('contentRange', None), - "contentSettings": { - "cacheControl": result['contentSettings']['cacheControl'], - "contentDisposition": result['contentSettings']['contentDisposition'], - "contentEncoding": result['contentSettings']['contentEncoding'], - "contentLanguage": result['contentSettings']['contentLanguage'], - "contentMd5": _encode_bytes(result['contentSettings']['contentMd5']), - "contentType": result['contentSettings']['contentType'] - }, - "copy": result.pop('copy', None), - "creationTime": result.pop('creationTime', None), - "deletedTime": result.pop('deletedTime', None), - "etag": result.pop('etag', None), - "lastModified": result.pop('lastModified', None), - "lease": result.pop('lease', None), - "pageBlobSequenceNumber": result.pop('pageBlobSequenceNumber', None), - "pageRanges": _transform_page_ranges(result.pop('pageRanges', None)), - "rehydrationStatus": result.pop('archiveStatus', None), - "remainingRetentionDays": result.pop('remainingRetentionDays', None), - "serverEncrypted": result.pop('serverEncrypted', None) - }, - "snapshot": result.pop('snapshot', None) - } - del result['contentSettings'] - new_result.update(result) - return new_result - - -def transform_immutability_policy(result): - # service returns policy with period value of "0" after it has been deleted - # this only shows the policy if the property value is greater than 0 - if result.immutability_period_since_creation_in_days: - return result - return None diff --git a/src/azure-cli/azure/cli/command_modules/storage/_validators_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/_validators_azure_stack.py deleted file mode 100644 index 39075a1ceec..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/_validators_azure_stack.py +++ /dev/null @@ -1,1296 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# pylint: disable=protected-access - -import argparse - -from azure.cli.core.commands.validators import validate_key_value_pairs -from azure.cli.core.profiles import ResourceType, get_sdk - -from azure.cli.command_modules.storage._client_factory import (get_storage_data_service_client, - blob_data_service_factory, - file_data_service_factory, - storage_client_factory) -from azure.cli.command_modules.storage.util import glob_files_locally, guess_content_type -from azure.cli.command_modules.storage.sdkutil import get_table_data_type -from azure.cli.command_modules.storage.url_quote_util import encode_for_url -from azure.cli.command_modules.storage.oauth_token_util import TokenUpdater - -from knack.log import get_logger -from knack.util import CLIError - -storage_account_key_options = {'primary': 'key1', 'secondary': 'key2'} -logger = get_logger(__name__) - - -# Utilities - - -# pylint: disable=inconsistent-return-statements,too-many-lines -def _query_account_key(cli_ctx, account_name): - """Query the storage account key. This is used when the customer doesn't offer account key but name.""" - rg, scf = _query_account_rg(cli_ctx, account_name) - t_storage_account_keys = get_sdk( - cli_ctx, ResourceType.MGMT_STORAGE, 'models.storage_account_keys#StorageAccountKeys') - - logger.debug('Disable HTTP logging to avoid having storage keys in debug logs') - if t_storage_account_keys: - return scf.storage_accounts.list_keys(rg, account_name, logging_enable=False).key1 - # of type: models.storage_account_list_keys_result#StorageAccountListKeysResult - return scf.storage_accounts.list_keys(rg, account_name, logging_enable=False).keys[0].value # pylint: disable=no-member - - -def _query_account_rg(cli_ctx, account_name): - """Query the storage account's resource group, which the mgmt sdk requires.""" - scf = storage_client_factory(cli_ctx) - acc = next((x for x in scf.storage_accounts.list() if x.name == account_name), None) - if acc: - from azure.mgmt.core.tools import parse_resource_id - return parse_resource_id(acc.id)['resource_group'], scf - raise ValueError("Storage account '{}' not found.".format(account_name)) - - -def _create_token_credential(cli_ctx): - from knack.cli import EVENT_CLI_POST_EXECUTE - - TokenCredential = get_sdk(cli_ctx, ResourceType.DATA_STORAGE, 'common#TokenCredential') - - token_credential = TokenCredential() - updater = TokenUpdater(token_credential, cli_ctx) - - def _cancel_timer_event_handler(_, **__): - updater.cancel() - cli_ctx.register_event(EVENT_CLI_POST_EXECUTE, _cancel_timer_event_handler) - return token_credential - - -# region PARAMETER VALIDATORS -def parse_storage_account(cmd, namespace): - """Parse storage account which can be either account name or account id""" - from azure.mgmt.core.tools import parse_resource_id, is_valid_resource_id - - if namespace.account_name and is_valid_resource_id(namespace.account_name): - namespace.resource_group_name = parse_resource_id(namespace.account_name)['resource_group'] - namespace.account_name = parse_resource_id(namespace.account_name)['name'] - elif namespace.account_name and not is_valid_resource_id(namespace.account_name) and \ - not namespace.resource_group_name: - namespace.resource_group_name = _query_account_rg(cmd.cli_ctx, namespace.account_name)[0] - - -def process_resource_group(cmd, namespace): - """Processes the resource group parameter from the account name""" - if namespace.account_name and not namespace.resource_group_name: - namespace.resource_group_name = _query_account_rg(cmd.cli_ctx, namespace.account_name)[0] - - -def validate_table_payload_format(cmd, namespace): - t_table_payload = get_table_data_type(cmd.cli_ctx, 'table', 'TablePayloadFormat') - if namespace.accept: - formats = { - 'none': t_table_payload.JSON_NO_METADATA, - 'minimal': t_table_payload.JSON_MINIMAL_METADATA, - 'full': t_table_payload.JSON_FULL_METADATA - } - namespace.accept = formats[namespace.accept.lower()] - - -def validate_bypass(namespace): - if namespace.bypass: - namespace.bypass = ', '.join(namespace.bypass) if isinstance(namespace.bypass, list) else namespace.bypass - - -def get_config_value(cmd, section, key, default): - return cmd.cli_ctx.config.get(section, key, default) - - -def is_storagev2(import_prefix): - return import_prefix.startswith('azure.multiapi.storagev2.') - - -def validate_client_parameters(cmd, namespace): - """ Retrieves storage connection parameters from environment variables and parses out connection string into - account name and key """ - n = namespace - - if hasattr(n, 'auth_mode'): - auth_mode = n.auth_mode or get_config_value(cmd, 'storage', 'auth_mode', None) - del n.auth_mode - if not n.account_name: - n.account_name = get_config_value(cmd, 'storage', 'account', None) - if auth_mode == 'login': - prefix = cmd.command_kwargs['resource_type'].value[0] - # is_storagv2() is used to distinguish if the command is in track2 SDK - # If yes, we will use get_login_credentials() as token credential - if is_storagev2(prefix): - from azure.cli.core._profile import Profile - profile = Profile(cli_ctx=cmd.cli_ctx) - n.token_credential, _, _ = profile.get_login_credentials(subscription_id=n._subscription) - # Otherwise, we will assume it is in track1 and keep previous token updater - else: - n.token_credential = _create_token_credential(cmd.cli_ctx) - - if hasattr(n, 'token_credential') and n.token_credential: - # give warning if there are account key args being ignored - account_key_args = [n.account_key and "--account-key", n.sas_token and "--sas-token", - n.connection_string and "--connection-string"] - account_key_args = [arg for arg in account_key_args if arg] - - if account_key_args: - logger.warning('In "login" auth mode, the following arguments are ignored: %s', - ' ,'.join(account_key_args)) - return - - if not n.connection_string: - n.connection_string = get_config_value(cmd, 'storage', 'connection_string', None) - - # if connection string supplied or in environment variables, extract account key and name - if n.connection_string: - conn_dict = validate_key_value_pairs(n.connection_string) - n.account_name = conn_dict.get('AccountName') - n.account_key = conn_dict.get('AccountKey') - n.sas_token = conn_dict.get('SharedAccessSignature') - - # otherwise, simply try to retrieve the remaining variables from environment variables - if not n.account_name: - n.account_name = get_config_value(cmd, 'storage', 'account', None) - if not n.account_key: - n.account_key = get_config_value(cmd, 'storage', 'key', None) - if not n.sas_token: - n.sas_token = get_config_value(cmd, 'storage', 'sas_token', None) - - # strip the '?' from sas token. the portal and command line are returns sas token in different - # forms - if n.sas_token: - n.sas_token = n.sas_token.lstrip('?') - - # if account name is specified but no key, attempt to query - if n.account_name and not n.account_key and not n.sas_token: - logger.warning('There are no credentials provided in your command and environment, we will query for the ' - 'account key inside your storage account. \nPlease provide --connection-string, ' - '--account-key or --sas-token as credentials, or use `--auth-mode login` if you ' - 'have required RBAC roles in your command. For more information about RBAC roles ' - 'in storage, visit ' - 'https://learn.microsoft.com/azure/storage/common/storage-auth-aad-rbac-cli. \n' - 'Setting the corresponding environment variables can avoid inputting credentials in ' - 'your command. Please use --help to get more information.') - n.account_key = _query_account_key(cmd.cli_ctx, n.account_name) - - -def validate_encryption_key(cmd, namespace): - encryption_key_source = cmd.get_models('EncryptionScopeSource', resource_type=ResourceType.MGMT_STORAGE) - if namespace.key_source == encryption_key_source.microsoft_key_vault and \ - not namespace.key_uri: - raise CLIError("usage error: Please specify --key-uri when using {} as key source." - .format(encryption_key_source.microsoft_key_vault)) - if namespace.key_source != encryption_key_source.microsoft_key_vault and namespace.key_uri: - raise CLIError("usage error: Specify `--key-source={}` and --key-uri to configure key vault properties." - .format(encryption_key_source.microsoft_key_vault)) - - -def process_blob_source_uri(cmd, namespace): - """ - Validate the parameters referenced to a blob source and create the source URI from them. - """ - from .util import create_short_lived_blob_sas - usage_string = \ - 'Invalid usage: {}. Supply only one of the following argument sets to specify source:' \ - '\n\t --source-uri' \ - '\n\tOR --source-container --source-blob --source-snapshot [--source-account-name & sas] ' \ - '\n\tOR --source-container --source-blob --source-snapshot [--source-account-name & key] ' - - ns = vars(namespace) - - # source as blob - container = ns.pop('source_container', None) - blob = ns.pop('source_blob', None) - snapshot = ns.pop('source_snapshot', None) - - # source credential clues - source_account_name = ns.pop('source_account_name', None) - source_account_key = ns.pop('source_account_key', None) - sas = ns.pop('source_sas', None) - - # source in the form of an uri - uri = ns.get('copy_source', None) - if uri: - if any([container, blob, sas, snapshot, source_account_name, source_account_key]): - raise ValueError(usage_string.format('Unused parameters are given in addition to the ' - 'source URI')) - # simplest scenario--no further processing necessary - return - - validate_client_parameters(cmd, namespace) # must run first to resolve storage account - - # determine if the copy will happen in the same storage account - if not source_account_name and source_account_key: - raise ValueError(usage_string.format('Source account key is given but account name is not')) - if not source_account_name and not source_account_key: - # neither source account name or key is given, assume that user intends to copy blob in - # the same account - source_account_name = ns.get('account_name', None) - source_account_key = ns.get('account_key', None) - elif source_account_name and not source_account_key: - if source_account_name == ns.get('account_name', None): - # the source account name is same as the destination account name - source_account_key = ns.get('account_key', None) - else: - # the source account is different from destination account but the key is missing - # try to query one. - try: - source_account_key = _query_account_key(cmd.cli_ctx, source_account_name) - except ValueError: - raise ValueError('Source storage account {} not found.'.format(source_account_name)) - # else: both source account name and key are given by user - - if not source_account_name: - raise ValueError(usage_string.format('Storage account name not found')) - - if not sas: - sas = create_short_lived_blob_sas(cmd, source_account_name, source_account_key, container, blob) - - query_params = [] - if sas: - query_params.append(sas) - if snapshot: - query_params.append('snapshot={}'.format(snapshot)) - - uri = 'https://{}.blob.{}/{}/{}{}{}'.format(source_account_name, - cmd.cli_ctx.cloud.suffixes.storage_endpoint, - container, - blob, - '?' if query_params else '', - '&'.join(query_params)) - - namespace.copy_source = uri - - -def validate_source_uri(cmd, namespace): # pylint: disable=too-many-statements - from .util import create_short_lived_blob_sas, create_short_lived_file_sas - usage_string = \ - 'Invalid usage: {}. Supply only one of the following argument sets to specify source:' \ - '\n\t --source-uri [--source-sas]' \ - '\n\tOR --source-container --source-blob [--source-account-name & sas] [--source-snapshot]' \ - '\n\tOR --source-container --source-blob [--source-account-name & key] [--source-snapshot]' \ - '\n\tOR --source-share --source-path' \ - '\n\tOR --source-share --source-path [--source-account-name & sas]' \ - '\n\tOR --source-share --source-path [--source-account-name & key]' - - ns = vars(namespace) - - # source as blob - container = ns.pop('source_container', None) - blob = ns.pop('source_blob', None) - snapshot = ns.pop('source_snapshot', None) - - # source as file - share = ns.pop('source_share', None) - path = ns.pop('source_path', None) - file_snapshot = ns.pop('file_snapshot', None) - - # source credential clues - source_account_name = ns.pop('source_account_name', None) - source_account_key = ns.pop('source_account_key', None) - source_sas = ns.pop('source_sas', None) - - # source in the form of an uri - uri = ns.get('copy_source', None) - if uri: - if any([container, blob, snapshot, share, path, file_snapshot, source_account_name, - source_account_key]): - raise ValueError(usage_string.format('Unused parameters are given in addition to the ' - 'source URI')) - if source_sas: - source_sas = source_sas.lstrip('?') - uri = '{}{}{}'.format(uri, '?', source_sas) - namespace.copy_source = uri - return - - # ensure either a file or blob source is specified - valid_blob_source = container and blob and not share and not path and not file_snapshot - valid_file_source = share and path and not container and not blob and not snapshot - - if not valid_blob_source and not valid_file_source: - raise ValueError(usage_string.format('Neither a valid blob or file source is specified')) - if valid_blob_source and valid_file_source: - raise ValueError(usage_string.format('Ambiguous parameters, both blob and file sources are ' - 'specified')) - - validate_client_parameters(cmd, namespace) # must run first to resolve storage account - - if not source_account_name: - if source_account_key: - raise ValueError(usage_string.format('Source account key is given but account name is not')) - # assume that user intends to copy blob in the same account - source_account_name = ns.get('account_name', None) - - # determine if the copy will happen in the same storage account - same_account = False - - if not source_account_key and not source_sas: - if source_account_name == ns.get('account_name', None): - same_account = True - source_account_key = ns.get('account_key', None) - source_sas = ns.get('sas_token', None) - else: - # the source account is different from destination account but the key is missing try to query one. - try: - source_account_key = _query_account_key(cmd.cli_ctx, source_account_name) - except ValueError: - raise ValueError('Source storage account {} not found.'.format(source_account_name)) - - # Both source account name and either key or sas (or both) are now available - if not source_sas: - # generate a sas token even in the same account when the source and destination are not the same kind. - if valid_file_source and (ns.get('container_name', None) or not same_account): - import os - dir_name, file_name = os.path.split(path) if path else (None, '') - source_sas = create_short_lived_file_sas(cmd, source_account_name, source_account_key, share, - dir_name, file_name) - elif valid_blob_source and (ns.get('share_name', None) or not same_account): - source_sas = create_short_lived_blob_sas(cmd, source_account_name, source_account_key, container, blob) - - query_params = [] - if source_sas: - query_params.append(source_sas.lstrip('?')) - if snapshot: - query_params.append('snapshot={}'.format(snapshot)) - if file_snapshot: - query_params.append('sharesnapshot={}'.format(file_snapshot)) - - uri = 'https://{0}.{1}.{6}/{2}/{3}{4}{5}'.format( - source_account_name, - 'blob' if valid_blob_source else 'file', - container if valid_blob_source else share, - encode_for_url(blob if valid_blob_source else path), - '?' if query_params else '', - '&'.join(query_params), - cmd.cli_ctx.cloud.suffixes.storage_endpoint) - - namespace.copy_source = uri - - -def validate_blob_type(namespace): - if not namespace.blob_type: - namespace.blob_type = 'page' if namespace.file_path.endswith('.vhd') else 'block' - - -def validate_storage_data_plane_list(namespace): - if namespace.num_results == '*': - namespace.num_results = None - else: - namespace.num_results = int(namespace.num_results) - - -def get_content_setting_validator(settings_class, update, guess_from_file=None): - def _class_name(class_type): - return class_type.__module__ + "." + class_type.__class__.__name__ - - def validator(cmd, namespace): - t_base_blob_service, t_file_service, t_blob_content_settings, t_file_content_settings = cmd.get_models( - 'blob.baseblobservice#BaseBlobService', - 'file#FileService', - 'blob.models#ContentSettings', - 'file.models#ContentSettings') - - # must run certain validators first for an update - if update: - validate_client_parameters(cmd, namespace) - if update and _class_name(settings_class) == _class_name(t_file_content_settings): - get_file_path_validator()(namespace) - ns = vars(namespace) - clear_content_settings = ns.pop('clear_content_settings', False) - - # retrieve the existing object properties for an update - props = None - if update and not clear_content_settings: - account = ns.get('account_name') - key = ns.get('account_key') - cs = ns.get('connection_string') - sas = ns.get('sas_token') - token_credential = ns.get('token_credential') - if _class_name(settings_class) == _class_name(t_blob_content_settings): - client = get_storage_data_service_client(cmd.cli_ctx, - service=t_base_blob_service, - name=account, - key=key, connection_string=cs, sas_token=sas, - token_credential=token_credential) - container = ns.get('container_name') - blob = ns.get('blob_name') - lease_id = ns.get('lease_id') - props = client.get_blob_properties(container, blob, lease_id=lease_id).properties.content_settings - elif _class_name(settings_class) == _class_name(t_file_content_settings): - client = get_storage_data_service_client(cmd.cli_ctx, t_file_service, account, key, cs, sas) - share = ns.get('share_name') - directory = ns.get('directory_name') - filename = ns.get('file_name') - props = client.get_file_properties(share, directory, filename).properties.content_settings - - # create new properties - new_props = settings_class( - content_type=ns.pop('content_type', None), - content_disposition=ns.pop('content_disposition', None), - content_encoding=ns.pop('content_encoding', None), - content_language=ns.pop('content_language', None), - content_md5=ns.pop('content_md5', None), - cache_control=ns.pop('content_cache_control', None) - ) - - # if update, fill in any None values with existing - if update: - if not clear_content_settings: - for attr in ['content_type', 'content_disposition', 'content_encoding', 'content_language', - 'content_md5', 'cache_control']: - if getattr(new_props, attr) is None: - setattr(new_props, attr, getattr(props, attr)) - else: - if guess_from_file: - new_props = guess_content_type(ns[guess_from_file], new_props, settings_class) - - ns['content_settings'] = new_props - - return validator - - -def validate_custom_domain(namespace): - if namespace.use_subdomain and not namespace.custom_domain: - raise ValueError('usage error: --custom-domain DOMAIN [--use-subdomain]') - - -def validate_encryption_services(cmd, namespace): - """ - Builds up the encryption services object for storage account operations based on the list of services passed in. - """ - if namespace.encryption_services: - t_encryption_services, t_encryption_service = get_sdk(cmd.cli_ctx, ResourceType.MGMT_STORAGE, - 'EncryptionServices', 'EncryptionService', mod='models') - services = {service: t_encryption_service(enabled=True) for service in namespace.encryption_services} - - namespace.encryption_services = t_encryption_services(**services) - - -def validate_encryption_source(namespace): - if namespace.encryption_key_source == 'Microsoft.Keyvault' and \ - not (namespace.encryption_key_name and namespace.encryption_key_vault): - raise ValueError('--encryption-key-name and --encryption-key-vault are required ' - 'when --encryption-key-source=Microsoft.Keyvault is specified.') - - if namespace.encryption_key_name or namespace.encryption_key_version is not None or namespace.encryption_key_vault: - if namespace.encryption_key_source and namespace.encryption_key_source != 'Microsoft.Keyvault': - raise ValueError('--encryption-key-name, --encryption-key-vault, and --encryption-key-version are not ' - 'applicable without Microsoft.Keyvault key-source.') - - -def validate_entity(namespace): - """ Converts a list of key value pairs into a dictionary. Ensures that required - RowKey and PartitionKey are converted to the correct case and included. """ - values = dict(x.split('=', 1) for x in namespace.entity) - keys = values.keys() - for key in list(keys): - if key.lower() == 'rowkey': - val = values[key] - del values[key] - values['RowKey'] = val - elif key.lower() == 'partitionkey': - val = values[key] - del values[key] - values['PartitionKey'] = val - keys = values.keys() - missing_keys = 'RowKey ' if 'RowKey' not in keys else '' - missing_keys = '{}PartitionKey'.format(missing_keys) \ - if 'PartitionKey' not in keys else missing_keys - if missing_keys: - raise argparse.ArgumentError( - None, 'incorrect usage: entity requires: {}'.format(missing_keys)) - - def cast_val(key, val): - """ Attempts to cast numeric values (except RowKey and PartitionKey) to numbers so they - can be queried correctly. """ - if key in ['PartitionKey', 'RowKey']: - return val - - def try_cast(to_type): - try: - return to_type(val) - except ValueError: - return None - - return try_cast(int) or try_cast(float) or val - - # ensure numbers are converted from strings so querying will work correctly - values = {key: cast_val(key, val) for key, val in values.items()} - namespace.entity = values - - -def validate_marker(namespace): - """ Converts a list of key value pairs into a dictionary. Ensures that required - nextrowkey and nextpartitionkey are included. """ - if not namespace.marker: - return - marker = dict(x.split('=', 1) for x in namespace.marker) - expected_keys = {'nextrowkey', 'nextpartitionkey'} - - for key in list(marker.keys()): - new_key = key.lower() - if new_key in expected_keys: - expected_keys.remove(key.lower()) - val = marker[key] - del marker[key] - marker[new_key] = val - if expected_keys: - raise argparse.ArgumentError( - None, 'incorrect usage: marker requires: {}'.format(' '.join(expected_keys))) - - namespace.marker = marker - - -def get_file_path_validator(default_file_param=None): - """ Creates a namespace validator that splits out 'path' into 'directory_name' and 'file_name'. - Allows another path-type parameter to be named which can supply a default filename. """ - - def validator(namespace): - import os - if not hasattr(namespace, 'path'): - return - - path = namespace.path - dir_name, file_name = os.path.split(path) if path else (None, '') - - if default_file_param and '.' not in file_name: - dir_name = path - file_name = os.path.split(getattr(namespace, default_file_param))[1] - dir_name = None if dir_name in ('', '.') else dir_name - namespace.directory_name = dir_name - namespace.file_name = file_name - del namespace.path - - return validator - - -def validate_included_datasets(cmd, namespace): - if namespace.include: - include = namespace.include - if set(include) - set('cmsd'): - help_string = '(c)opy-info (m)etadata (s)napshots (d)eleted' - raise ValueError('valid values are {} or a combination thereof.'.format(help_string)) - t_blob_include = cmd.get_models('blob#Include') - namespace.include = t_blob_include('s' in include, 'm' in include, False, 'c' in include, 'd' in include) - - -def validate_key_name(namespace): - key_options = {'primary': '1', 'secondary': '2'} - if hasattr(namespace, 'key_type') and namespace.key_type: - namespace.key_name = namespace.key_type + key_options[namespace.key_name] - else: - namespace.key_name = storage_account_key_options[namespace.key_name] - if hasattr(namespace, 'key_type'): - del namespace.key_type - - -def validate_metadata(namespace): - if namespace.metadata: - namespace.metadata = dict(x.split('=', 1) for x in namespace.metadata) - - -def get_permission_help_string(permission_class): - allowed_values = [x.lower() for x in dir(permission_class) if not x.startswith('__')] - return ' '.join(['({}){}'.format(x[0], x[1:]) for x in allowed_values]) - - -def get_permission_validator(permission_class): - allowed_values = [x.lower() for x in dir(permission_class) if not x.startswith('__')] - allowed_string = ''.join(x[0] for x in allowed_values) - - def validator(namespace): - if namespace.permission: - if set(namespace.permission) - set(allowed_string): - help_string = get_permission_help_string(permission_class) - raise ValueError( - 'valid values are {} or a combination thereof.'.format(help_string)) - namespace.permission = permission_class(_str=namespace.permission) - - return validator - - -def table_permission_validator(cmd, namespace): - """ A special case for table because the SDK associates the QUERY permission with 'r' """ - t_table_permissions = get_table_data_type(cmd.cli_ctx, 'table', 'TablePermissions') - if namespace.permission: - if set(namespace.permission) - set('raud'): - help_string = '(r)ead/query (a)dd (u)pdate (d)elete' - raise ValueError('valid values are {} or a combination thereof.'.format(help_string)) - namespace.permission = t_table_permissions(_str=namespace.permission) - - -def validate_container_public_access(cmd, namespace): - from .sdkutil import get_container_access_type - t_base_blob_svc = cmd.get_models('blob.baseblobservice#BaseBlobService') - - if namespace.public_access: - namespace.public_access = get_container_access_type(cmd.cli_ctx, namespace.public_access.lower()) - - if hasattr(namespace, 'signed_identifiers'): - # must retrieve the existing ACL to simulate a patch operation because these calls - # are needlessly conflated - ns = vars(namespace) - validate_client_parameters(cmd, namespace) - account = ns.get('account_name') - key = ns.get('account_key') - cs = ns.get('connection_string') - sas = ns.get('sas_token') - client = get_storage_data_service_client(cmd.cli_ctx, t_base_blob_svc, account, key, cs, sas) - container = ns.get('container_name') - lease_id = ns.get('lease_id') - ns['signed_identifiers'] = client.get_container_acl(container, lease_id=lease_id) - - -def validate_fs_public_access(cmd, namespace): - from .sdkutil import get_fs_access_type - - if namespace.public_access: - namespace.public_access = get_fs_access_type(cmd.cli_ctx, namespace.public_access.lower()) - - -def validate_select(namespace): - if namespace.select: - namespace.select = ','.join(namespace.select) - - -# pylint: disable=too-many-statements -def get_source_file_or_blob_service_client(cmd, namespace): - """ - Create the second file service or blob service client for batch copy command, which is used to - list the source files or blobs. If both the source account and source URI are omitted, it - indicates that user want to copy files or blobs in the same storage account, therefore the - destination client will be set None hence the command will use destination client. - """ - t_file_svc, t_block_blob_svc = cmd.get_models('file#FileService', 'blob.blockblobservice#BlockBlobService') - usage_string = 'invalid usage: supply only one of the following argument sets:' + \ - '\n\t --source-uri [--source-sas]' + \ - '\n\tOR --source-container' + \ - '\n\tOR --source-container --source-account-name --source-account-key' + \ - '\n\tOR --source-container --source-account-name --source-sas' + \ - '\n\tOR --source-share --source-account-name --source-account-key' + \ - '\n\tOR --source-share --source-account-name --source-account-sas' - - ns = vars(namespace) - source_account = ns.pop('source_account_name', None) - source_key = ns.pop('source_account_key', None) - source_uri = ns.pop('source_uri', None) - source_sas = ns.get('source_sas', None) - source_container = ns.get('source_container', None) - source_share = ns.get('source_share', None) - - if source_uri and source_account: - raise ValueError(usage_string) - if not source_uri and bool(source_container) == bool(source_share): # must be container or share - raise ValueError(usage_string) - - if (not source_account) and (not source_uri): - # Set the source_client to None if neither source_account or source_uri is given. This - # indicates the command that the source files share or blob container is in the same storage - # account as the destination file share or blob container. - # - # The command itself should create the source service client since the validator can't - # access the destination client through the namespace. - # - # A few arguments check will be made as well so as not to cause ambiguity. - if source_key or source_sas: - raise ValueError('invalid usage: --source-account-name is missing; the source account is assumed to be the' - ' same as the destination account. Do not provide --source-sas or --source-account-key') - ns['source_client'] = None - - if 'token_credential' not in ns: # not using oauth - return - # oauth is only possible through destination, must still get source creds - source_account, source_key, source_sas = ns['account_name'], ns['account_key'], ns['sas_token'] - - if source_account: - if not (source_key or source_sas): - # when neither storage account key or SAS is given, try to fetch the key in the current - # subscription - source_key = _query_account_key(cmd.cli_ctx, source_account) - - if source_container: - ns['source_client'] = get_storage_data_service_client( - cmd.cli_ctx, t_block_blob_svc, name=source_account, key=source_key, sas_token=source_sas) - elif source_share: - ns['source_client'] = get_storage_data_service_client( - cmd.cli_ctx, t_file_svc, name=source_account, key=source_key, sas_token=source_sas) - elif source_uri: - if source_key or source_container or source_share: - raise ValueError(usage_string) - - from .storage_url_helpers import StorageResourceIdentifier - if source_sas: - source_uri = '{}{}{}'.format(source_uri, '?', source_sas.lstrip('?')) - identifier = StorageResourceIdentifier(cmd.cli_ctx.cloud, source_uri) - nor_container_or_share = not identifier.container and not identifier.share - if not identifier.is_url(): - raise ValueError('incorrect usage: --source-uri expects a URI') - if identifier.blob or identifier.directory or identifier.filename or nor_container_or_share: - raise ValueError('incorrect usage: --source-uri has to be blob container or file share') - - if identifier.sas_token: - ns['source_sas'] = identifier.sas_token - else: - source_key = _query_account_key(cmd.cli_ctx, identifier.account_name) - - if identifier.container: - ns['source_container'] = identifier.container - if identifier.account_name != ns.get('account_name'): - ns['source_client'] = get_storage_data_service_client( - cmd.cli_ctx, t_block_blob_svc, name=identifier.account_name, key=source_key, - sas_token=identifier.sas_token) - elif identifier.share: - ns['source_share'] = identifier.share - if identifier.account_name != ns.get('account_name'): - ns['source_client'] = get_storage_data_service_client( - cmd.cli_ctx, t_file_svc, name=identifier.account_name, key=source_key, - sas_token=identifier.sas_token) - - -def add_progress_callback(cmd, namespace): - def _update_progress(current, total): - message = getattr(_update_progress, 'message', 'Alive') - reuse = getattr(_update_progress, 'reuse', False) - - if total: - hook.add(message=message, value=current, total_val=total) - if total == current and not reuse: - hook.end() - - hook = cmd.cli_ctx.get_progress_controller(det=True) - _update_progress.hook = hook - - if not namespace.no_progress: - namespace.progress_callback = _update_progress - del namespace.no_progress - - -def process_container_delete_parameters(cmd, namespace): - """Process the parameters for storage container delete command""" - # check whether to use mgmt or data-plane - if namespace.bypass_immutability_policy: - # use management-plane - namespace.processed_account_name = namespace.account_name - namespace.processed_resource_group, namespace.mgmt_client = _query_account_rg( - cmd.cli_ctx, namespace.account_name) - del namespace.auth_mode - else: - # use data-plane, like before - validate_client_parameters(cmd, namespace) - - -def process_blob_download_batch_parameters(cmd, namespace): - """Process the parameters for storage blob download command""" - import os - - # 1. quick check - if not os.path.exists(namespace.destination) or not os.path.isdir(namespace.destination): - raise ValueError('incorrect usage: destination must be an existing directory') - - # 2. try to extract account name and container name from source string - _process_blob_batch_container_parameters(cmd, namespace) - - # 3. Call validators - add_progress_callback(cmd, namespace) - - -def process_blob_upload_batch_parameters(cmd, namespace): - """Process the source and destination of storage blob upload command""" - import os - - # 1. quick check - if not os.path.exists(namespace.source) or not os.path.isdir(namespace.source): - raise ValueError('incorrect usage: source must be an existing directory') - - # 2. try to extract account name and container name from destination string - _process_blob_batch_container_parameters(cmd, namespace, source=False) - - # 3. collect the files to be uploaded - namespace.source = os.path.realpath(namespace.source) - namespace.source_files = list(glob_files_locally(namespace.source, namespace.pattern)) - - # 4. determine blob type - if namespace.blob_type is None: - vhd_files = [f for f in namespace.source_files if f[0].endswith('.vhd')] - if any(vhd_files) and len(vhd_files) == len(namespace.source_files): - # when all the listed files are vhd files use page - namespace.blob_type = 'page' - elif any(vhd_files): - # source files contain vhd files but not all of them - raise CLIError("""Fail to guess the required blob type. Type of the files to be - uploaded are not consistent. Default blob type for .vhd files is "page", while - others are "block". You can solve this problem by either explicitly set the blob - type or ensure the pattern matches a correct set of files.""") - else: - namespace.blob_type = 'block' - - # 5. call other validators - validate_metadata(namespace) - t_blob_content_settings = cmd.loader.get_sdk('blob.models#ContentSettings') - get_content_setting_validator(t_blob_content_settings, update=False)(cmd, namespace) - add_progress_callback(cmd, namespace) - - -def process_blob_delete_batch_parameters(cmd, namespace): - _process_blob_batch_container_parameters(cmd, namespace) - - -def _process_blob_batch_container_parameters(cmd, namespace, source=True): - """Process the container parameters for storage blob batch commands before populating args from environment.""" - if source: - container_arg, container_name_arg = 'source', 'source_container_name' - else: - # destination - container_arg, container_name_arg = 'destination', 'destination_container_name' - - # try to extract account name and container name from source string - from .storage_url_helpers import StorageResourceIdentifier - container_arg_val = getattr(namespace, container_arg) # either a url or name - identifier = StorageResourceIdentifier(cmd.cli_ctx.cloud, container_arg_val) - - if not identifier.is_url(): - setattr(namespace, container_name_arg, container_arg_val) - elif identifier.blob: - raise ValueError('incorrect usage: {} should be either a container URL or name'.format(container_arg)) - else: - setattr(namespace, container_name_arg, identifier.container) - if namespace.account_name is None: - namespace.account_name = identifier.account_name - elif namespace.account_name != identifier.account_name: - raise ValueError('The given storage account name is not consistent with the ' - 'account name in the destination URL') - - # if no sas-token is given and the container url contains one, use it - if not namespace.sas_token and identifier.sas_token: - namespace.sas_token = identifier.sas_token - - # Finally, grab missing storage connection parameters from environment variables - validate_client_parameters(cmd, namespace) - - -def process_file_upload_batch_parameters(cmd, namespace): - """Process the parameters of storage file batch upload command""" - import os - - # 1. quick check - if not os.path.exists(namespace.source): - raise ValueError('incorrect usage: source {} does not exist'.format(namespace.source)) - - if not os.path.isdir(namespace.source): - raise ValueError('incorrect usage: source must be a directory') - - # 2. try to extract account name and container name from destination string - from .storage_url_helpers import StorageResourceIdentifier - identifier = StorageResourceIdentifier(cmd.cli_ctx.cloud, namespace.destination) - if identifier.is_url(): - if identifier.filename or identifier.directory: - raise ValueError('incorrect usage: destination must be a file share url') - - namespace.destination = identifier.share - - if not namespace.account_name: - namespace.account_name = identifier.account_name - - namespace.source = os.path.realpath(namespace.source) - - -def process_file_download_batch_parameters(cmd, namespace): - """Process the parameters for storage file batch download command""" - import os - - # 1. quick check - if not os.path.exists(namespace.destination) or not os.path.isdir(namespace.destination): - raise ValueError('incorrect usage: destination must be an existing directory') - - # 2. try to extract account name and share name from source string - process_file_batch_source_parameters(cmd, namespace) - - -def process_file_batch_source_parameters(cmd, namespace): - from .storage_url_helpers import StorageResourceIdentifier - identifier = StorageResourceIdentifier(cmd.cli_ctx.cloud, namespace.source) - if identifier.is_url(): - if identifier.filename or identifier.directory: - raise ValueError('incorrect usage: source should be either share URL or name') - - namespace.source = identifier.share - - if not namespace.account_name: - namespace.account_name = identifier.account_name - - -def process_file_download_namespace(namespace): - import os - - get_file_path_validator()(namespace) - - dest = namespace.file_path - if not dest or os.path.isdir(dest): - namespace.file_path = os.path.join(dest, namespace.file_name) \ - if dest else namespace.file_name - - -def process_metric_update_namespace(namespace): - namespace.hour = namespace.hour == 'true' - namespace.minute = namespace.minute == 'true' - namespace.api = namespace.api == 'true' if namespace.api else None - if namespace.hour is None and namespace.minute is None: - raise argparse.ArgumentError( - None, 'incorrect usage: must specify --hour and/or --minute') - if (namespace.hour or namespace.minute) and namespace.api is None: - raise argparse.ArgumentError( - None, 'incorrect usage: specify --api when hour or minute metrics are enabled') - - -def validate_subnet(cmd, namespace): - from azure.mgmt.core.tools import resource_id, is_valid_resource_id - from azure.cli.core.commands.client_factory import get_subscription_id - - subnet = namespace.subnet - subnet_is_id = is_valid_resource_id(subnet) - vnet = namespace.vnet_name - - if (subnet_is_id and not vnet) or (not subnet and not vnet): - return - if subnet and not subnet_is_id and vnet: - namespace.subnet = resource_id( - subscription=get_subscription_id(cmd.cli_ctx), - resource_group=namespace.resource_group_name, - namespace='Microsoft.Network', - type='virtualNetworks', - name=vnet, - child_type_1='subnets', - child_name_1=subnet) - else: - raise CLIError('incorrect usage: [--subnet ID | --subnet NAME --vnet-name NAME]') - - -def get_datetime_type(to_string): - """ Validates UTC datetime. Examples of accepted forms: - 2017-12-31T01:11:59Z,2017-12-31T01:11Z or 2017-12-31T01Z or 2017-12-31 """ - from datetime import datetime - - def datetime_type(string): - """ Validates UTC datetime. Examples of accepted forms: - 2017-12-31T01:11:59Z,2017-12-31T01:11Z or 2017-12-31T01Z or 2017-12-31 """ - accepted_date_formats = ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%MZ', - '%Y-%m-%dT%HZ', '%Y-%m-%d'] - for form in accepted_date_formats: - try: - if to_string: - return datetime.strptime(string, form).strftime(form) - - return datetime.strptime(string, form) - except ValueError: - continue - raise ValueError("Input '{}' not valid. Valid example: 2000-12-31T12:59:59Z".format(string)) - - return datetime_type - - -def ipv4_range_type(string): - """ Validates an IPv4 address or address range. """ - import re - ip_format = r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}' - if not re.match("^{}$".format(ip_format), string): - if not re.match("^{ip_format}-{ip_format}$".format(ip_format=ip_format), string): - raise ValueError - return string - - -def resource_type_type(loader): - """ Returns a function which validates that resource types string contains only a combination of service, - container, and object. Their shorthand representations are s, c, and o. """ - - def impl(string): - t_resources = loader.get_models('common.models#ResourceTypes') - if set(string) - set("sco"): - raise ValueError - return t_resources(_str=''.join(set(string))) - - return impl - - -def services_type(loader): - """ Returns a function which validates that services string contains only a combination of blob, queue, table, - and file. Their shorthand representations are b, q, t, and f. """ - - def impl(string): - t_services = loader.get_models('common.models#Services') - if set(string) - set("bqtf"): - raise ValueError - return t_services(_str=''.join(set(string))) - - return impl - - -def get_char_options_validator(types, property_name): - def _validator(namespace): - service_types = set(getattr(namespace, property_name, [])) - - if not service_types: - raise ValueError('Missing options --{}.'.format(property_name.replace('_', '-'))) - - if service_types - set(types): - raise ValueError( - '--{}: only valid values are: {}.'.format(property_name.replace('_', '-'), ', '.join(types))) - - setattr(namespace, property_name, service_types) - - return _validator - - -def page_blob_tier_validator(cmd, namespace): - if not namespace.tier: - return - - if namespace.blob_type != 'page' and namespace.tier: - raise ValueError('Blob tier is only applicable to page blobs on premium storage accounts.') - - try: - namespace.tier = getattr(cmd.get_models('blob.models#PremiumPageBlobTier'), namespace.tier) - except AttributeError: - from azure.cli.command_modules.storage.sdkutil import get_blob_tier_names - raise ValueError('Unknown premium page blob tier name. Choose among {}'.format(', '.join( - get_blob_tier_names(cmd.cli_ctx, 'PremiumPageBlobTier')))) - - -def block_blob_tier_validator(cmd, namespace): - if not namespace.tier: - return - - if namespace.blob_type != 'block' and namespace.tier: - raise ValueError('Blob tier is only applicable to block blobs on standard storage accounts.') - - try: - namespace.tier = getattr(cmd.get_models('blob.models#StandardBlobTier'), namespace.tier) - except AttributeError: - from azure.cli.command_modules.storage.sdkutil import get_blob_tier_names - raise ValueError('Unknown block blob tier name. Choose among {}'.format(', '.join( - get_blob_tier_names(cmd.cli_ctx, 'StandardBlobTier')))) - - -def blob_tier_validator(cmd, namespace): - if namespace.blob_type == 'page': - page_blob_tier_validator(cmd, namespace) - elif namespace.blob_type == 'block': - block_blob_tier_validator(cmd, namespace) - else: - raise ValueError('Blob tier is only applicable to block or page blob.') - - -def blob_rehydrate_priority_validator(namespace): - if namespace.blob_type == 'page' and namespace.rehydrate_priority: - raise ValueError('--rehydrate-priority is only applicable to block blob.') - if namespace.tier == 'Archive' and namespace.rehydrate_priority: - raise ValueError('--rehydrate-priority is only applicable to rehydrate blob data from the archive tier.') - if namespace.rehydrate_priority is None: - namespace.rehydrate_priority = 'Standard' - - -def validate_azcopy_upload_destination_url(cmd, namespace): - client = blob_data_service_factory(cmd.cli_ctx, { - 'account_name': namespace.account_name, 'connection_string': namespace.connection_string}) - destination_path = namespace.destination_path - if not destination_path: - destination_path = '' - url = client.make_blob_url(namespace.destination_container, destination_path) - namespace.destination = url - del namespace.destination_container - del namespace.destination_path - - -def validate_azcopy_remove_arguments(cmd, namespace): - usage_string = \ - 'Invalid usage: {}. Supply only one of the following argument sets to specify source:' \ - '\n\t --container-name [--name]' \ - '\n\tOR --share-name [--path]' - - ns = vars(namespace) - - # source as blob - container = ns.pop('container_name', None) - blob = ns.pop('blob_name', None) - - # source as file - share = ns.pop('share_name', None) - path = ns.pop('path', None) - - # ensure either a file or blob source is specified - valid_blob = container and not share - valid_file = share and not container - - if not valid_blob and not valid_file: - raise ValueError(usage_string.format('Neither a valid blob or file source is specified')) - if valid_blob and valid_file: - raise ValueError(usage_string.format('Ambiguous parameters, both blob and file sources are ' - 'specified')) - if valid_blob: - client = blob_data_service_factory(cmd.cli_ctx, { - 'account_name': namespace.account_name, 'connection_string': namespace.connection_string}) - if not blob: - blob = '' - url = client.make_blob_url(container, blob) - namespace.service = 'blob' - namespace.target = url - - if valid_file: - import os - client = file_data_service_factory(cmd.cli_ctx, { - 'account_name': namespace.account_name, - 'account_key': namespace.account_key}) - dir_name, file_name = os.path.split(path) if path else (None, '') - dir_name = None if dir_name in ('', '.') else dir_name - url = client.make_file_url(share, dir_name, file_name) - namespace.service = 'file' - namespace.target = url - - -def as_user_validator(namespace): - if hasattr(namespace, 'token_credential') and not namespace.as_user: - raise CLIError('incorrect usage: specify --as-user when --auth-mode login is used to get user delegation key.') - if namespace.as_user: - if namespace.expiry is None: - raise argparse.ArgumentError( - None, 'incorrect usage: specify --expiry when as-user is enabled') - - expiry = get_datetime_type(False)(namespace.expiry) - - from datetime import datetime, timedelta - if expiry > datetime.utcnow() + timedelta(days=7): - raise argparse.ArgumentError( - None, 'incorrect usage: --expiry should be within 7 days from now') - - if ((not hasattr(namespace, 'token_credential') or namespace.token_credential is None) and - (not hasattr(namespace, 'auth_mode') or namespace.auth_mode != 'login')): - raise argparse.ArgumentError( - None, "incorrect usage: specify '--auth-mode login' when as-user is enabled") - - -def validator_delete_retention_days(namespace): - if namespace.enable_delete_retention is True and namespace.delete_retention_days is None: - raise ValueError( - "incorrect usage: you have to provide value for '--delete-retention-days' when '--enable-delete-retention' " - "is set to true") - - if namespace.enable_delete_retention is False and namespace.delete_retention_days is not None: - raise ValueError( - "incorrect usage: '--delete-retention-days' is invalid when '--enable-delete-retention' is set to false") - - if namespace.enable_delete_retention is None and namespace.delete_retention_days is not None: - raise ValueError( - "incorrect usage: please specify '--enable-delete-retention true' if you want to set the value for " - "'--delete-retention-days'") - - if namespace.delete_retention_days or namespace.delete_retention_days == 0: - if namespace.delete_retention_days < 1: - raise ValueError( - "incorrect usage: '--delete-retention-days' must be greater than or equal to 1") - if namespace.delete_retention_days > 365: - raise ValueError( - "incorrect usage: '--delete-retention-days' must be less than or equal to 365") - - -def validate_delete_retention_days(namespace): - if namespace.enable_delete_retention is True and namespace.delete_retention_days is None: - raise ValueError( - "incorrect usage: you have to provide value for '--delete-retention-days' when '--enable-delete-retention' " - "is set to true") - - if namespace.enable_delete_retention is False and namespace.delete_retention_days is not None: - raise ValueError( - "incorrect usage: '--delete-retention-days' is invalid when '--enable-delete-retention' is set to false") - - -# pylint: disable=too-few-public-methods -class BlobRangeAddAction(argparse._AppendAction): - def __call__(self, parser, namespace, values, option_string=None): - if not namespace.blob_ranges: - namespace.blob_ranges = [] - if isinstance(values, list): - values = ' '.join(values) - BlobRange = namespace._cmd.get_models('BlobRestoreRange', resource_type=ResourceType.MGMT_STORAGE) - try: - start_range, end_range = values.split(' ') - except (ValueError, TypeError): - raise CLIError('usage error: --blob-range VARIABLE OPERATOR VALUE') - namespace.blob_ranges.append(BlobRange( - start_range=start_range, - end_range=end_range - )) - - -def validate_private_endpoint_connection_id(cmd, namespace): - if namespace.connection_id: - from azure.cli.core.util import parse_proxy_resource_id - result = parse_proxy_resource_id(namespace.connection_id) - namespace.resource_group_name = result['resource_group'] - namespace.account_name = result['name'] - namespace.private_endpoint_connection_name = result['child_name_1'] - - if namespace.account_name and not namespace.resource_group_name: - namespace.resource_group_name = _query_account_rg(cmd.cli_ctx, namespace.account_name)[0] - - if not all([namespace.account_name, namespace.resource_group_name, namespace.private_endpoint_connection_name]): - raise CLIError('incorrect usage: [--id ID | --name NAME --account-name NAME]') - - del namespace.connection_id - - -def pop_data_client_auth(ns): - del ns.auth_mode - del ns.account_key - del ns.connection_string - del ns.sas_token - - -def validate_client_auth_parameter(cmd, ns): - from .sdkutil import get_container_access_type - if ns.public_access: - ns.public_access = get_container_access_type(cmd.cli_ctx, ns.public_access.lower()) - if ns.default_encryption_scope and ns.prevent_encryption_scope_override is not None: - # simply try to retrieve the remaining variables from environment variables - if not ns.account_name: - ns.account_name = get_config_value(cmd, 'storage', 'account', None) - if ns.account_name and not ns.resource_group_name: - ns.resource_group_name = _query_account_rg(cmd.cli_ctx, account_name=ns.account_name)[0] - pop_data_client_auth(ns) - elif (ns.default_encryption_scope and ns.prevent_encryption_scope_override is None) or \ - (not ns.default_encryption_scope and ns.prevent_encryption_scope_override is not None): - raise CLIError("usage error: You need to specify both --default-encryption-scope and " - "--prevent-encryption-scope-override to set encryption scope information " - "when creating container.") - else: - validate_client_parameters(cmd, ns) - - -def validate_encryption_scope_client_params(ns): - if ns.encryption_scope: - # will use track2 client and socket_timeout is unused - del ns.socket_timeout - - -def validate_access_control(namespace): - if namespace.acl and namespace.permissions: - raise CLIError('usage error: invalid when specifying both --acl and --permissions.') - - -def validate_service_type(services, service_type): - if service_type == 'table': - return 't' in services - if service_type == 'blob': - return 'b' in services - if service_type == 'queue': - return 'q' in services - - -def validate_logging_version(namespace): - if validate_service_type(namespace.services, 'table') and namespace.version != 1.0: - raise CLIError( - 'incorrect usage: for table service, the supported version for logging is `1.0`. For more information, ' - 'please refer to https://learn.microsoft.com/rest/api/storageservices/storage-analytics-log-format.') diff --git a/src/azure-cli/azure/cli/command_modules/storage/commands_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/commands_azure_stack.py deleted file mode 100644 index aa42d093c58..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/commands_azure_stack.py +++ /dev/null @@ -1,638 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from azure.cli.command_modules.storage._client_factory_azure_stack \ - import (cf_sa, cf_blob_container_mgmt, blob_data_service_factory, - page_blob_service_factory, file_data_service_factory, - queue_data_service_factory, table_data_service_factory, - cloud_storage_account_service_factory, multi_service_properties_factory, - cf_mgmt_policy, cf_blob_data_gen_update, cf_sa_for_keys, - cf_mgmt_blob_services, cf_mgmt_file_shares, - cf_private_link, cf_private_endpoint, - cf_mgmt_encryption_scope, cf_mgmt_file_services) -from azure.cli.command_modules.storage.sdkutil import cosmosdb_table_exists -from azure.cli.core.commands import CliCommandType -from azure.cli.core.commands.arm import show_exception_handler -from azure.cli.core.profiles import ResourceType - - -def load_command_table(self, _): # pylint: disable=too-many-locals, too-many-statements - storage_account_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#StorageAccountsOperations.{}', - client_factory=cf_sa, - resource_type=ResourceType.MGMT_STORAGE - ) - - blob_service_mgmt_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#BlobServicesOperations.{}', - client_factory=cf_mgmt_blob_services, - resource_type=ResourceType.MGMT_STORAGE - ) - - file_service_mgmt_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#FileServicesOperations.{}', - client_factory=cf_mgmt_file_services, - resource_type=ResourceType.MGMT_STORAGE - ) - - file_shares_mgmt_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#FileSharesOperations.{}', - client_factory=cf_mgmt_file_shares, - resource_type=ResourceType.MGMT_STORAGE - ) - - storage_account_sdk_keys = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#StorageAccountsOperations.{}', - client_factory=cf_sa_for_keys, - resource_type=ResourceType.MGMT_STORAGE - ) - - private_link_resource_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#PrivateLinkResourcesOperations.{}', - client_factory=cf_private_link, - resource_type=ResourceType.MGMT_STORAGE - ) - - private_endpoint_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#PrivateEndpointConnectionsOperations.{}', - client_factory=cf_private_endpoint, - resource_type=ResourceType.MGMT_STORAGE - ) - - private_endpoint_custom_type = CliCommandType( - operations_tmpl='azure.cli.command_modules.storage.operations.account#{}', - client_factory=cf_private_endpoint, - resource_type=ResourceType.MGMT_STORAGE) - - storage_account_custom_type = CliCommandType( - operations_tmpl='azure.cli.command_modules.storage.operations.account#{}', - client_factory=cf_sa) - - cloud_data_plane_sdk = CliCommandType( - operations_tmpl='azure.multiapi.storage.common#CloudStorageAccount.{}', - client_factory=cloud_storage_account_service_factory - ) - - block_blob_sdk = CliCommandType( - operations_tmpl='azure.multiapi.storage.blob.blockblobservice#BlockBlobService.{}', - client_factory=blob_data_service_factory, - resource_type=ResourceType.DATA_STORAGE) - - def get_custom_sdk(custom_module, client_factory, resource_type=ResourceType.DATA_STORAGE): - """Returns a CliCommandType instance with specified operation template based on the given custom module name. - This is useful when the command is not defined in the default 'custom' module but instead in a module under - 'operations' package.""" - return CliCommandType( - operations_tmpl='azure.cli.command_modules.storage.operations.{}#'.format( - custom_module) + '{}', - client_factory=client_factory, - resource_type=resource_type - ) - - with self.command_group('storage', command_type=block_blob_sdk, - custom_command_type=get_custom_sdk('azcopy', blob_data_service_factory)) as g: - g.storage_custom_command('remove', 'storage_remove', is_preview=True) - - with self.command_group('storage', custom_command_type=get_custom_sdk('azcopy', None)) as g: - g.custom_command('copy', 'storage_copy', is_preview=True) - - with self.command_group('storage account', storage_account_sdk, resource_type=ResourceType.MGMT_STORAGE, - custom_command_type=storage_account_custom_type) as g: - g.custom_command('check-name', 'check_name_availability') - g.custom_command('create', 'create_storage_account') - g.command('delete', 'delete', confirmation=True) - g.show_command('show', 'get_properties') - g.custom_command('list', 'list_storage_accounts') - g.custom_command( - 'show-usage', 'show_storage_account_usage', min_api='2018-02-01') - g.custom_command( - 'show-usage', 'show_storage_account_usage_no_location', max_api='2017-10-01') - g.custom_command('show-connection-string', - 'show_storage_account_connection_string') - g.generic_update_command('update', getter_name='get_properties', setter_name='update', - custom_func_name='update_storage_account', min_api='2016-12-01') - failover_confirmation = """ - The secondary cluster will become the primary cluster after failover. Please understand the following impact to your storage account before you initiate the failover: - 1. Please check the Last Sync Time using `az storage account show` with `--expand geoReplicationStats` and check the "geoReplicationStats" property. This is the data you may lose if you initiate the failover. - 2. After the failover, your storage account type will be converted to locally redundant storage (LRS). You can convert your account to use geo-redundant storage (GRS). - 3. Once you re-enable GRS/GZRS for your storage account, Microsoft will replicate data to your new secondary region. Replication time is dependent on the amount of data to replicate. Please note that there are bandwidth charges for the bootstrap. Please refer to doc: https://azure.microsoft.com/pricing/details/bandwidth/ - """ - g.command('failover', 'begin_failover', supports_no_wait=True, is_preview=True, min_api='2018-07-01', - confirmation=failover_confirmation) - - with self.command_group('storage account', storage_account_sdk_keys, resource_type=ResourceType.MGMT_STORAGE, - custom_command_type=storage_account_custom_type) as g: - from ._validators_azure_stack import validate_key_name - g.custom_command('keys renew', 'regenerate_key', validator=validate_key_name, - transform=lambda x: getattr(x, 'keys', x)) - g.command('keys list', 'list_keys', - transform=lambda x: getattr(x, 'keys', x)) - g.command('revoke-delegation-keys', 'revoke_user_delegation_keys', min_api='2019-04-01') - - with self.command_group('storage account', cloud_data_plane_sdk) as g: - g.storage_command('generate-sas', 'generate_shared_access_signature') - - encryption_scope_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#EncryptionScopesOperations.{}', - client_factory=cf_mgmt_encryption_scope, - resource_type=ResourceType.MGMT_STORAGE - ) - - encryption_scope_custom_type = CliCommandType( - operations_tmpl='azure.cli.command_modules.storage.operations.account#{}', - client_factory=cf_mgmt_encryption_scope, - resource_type=ResourceType.MGMT_STORAGE - ) - - with self.command_group('storage account encryption-scope', encryption_scope_sdk, - custom_command_type=encryption_scope_custom_type, is_preview=True, - resource_type=ResourceType.MGMT_STORAGE, min_api='2019-06-01') as g: - - g.custom_command('create', 'create_encryption_scope') - g.show_command('show', 'get') - g.command('list', 'list') - g.custom_command('update', 'update_encryption_scope') - - management_policy_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#ManagementPoliciesOperations.{}', - client_factory=cf_mgmt_policy, - resource_type=ResourceType.MGMT_STORAGE - ) - - management_policy_custom_type = CliCommandType( - operations_tmpl='azure.cli.command_modules.storage.operations.account#{}', - client_factory=cf_mgmt_policy) - - storage_blob_custom_type = CliCommandType( - operations_tmpl='azure.cli.command_modules.storage.operations.blob_azure_stack#{}', - client_factory=cf_sa, - resource_type=ResourceType.MGMT_STORAGE) - - with self.command_group('storage account management-policy', management_policy_sdk, - resource_type=ResourceType.MGMT_STORAGE, min_api='2018-11-01', - custom_command_type=management_policy_custom_type) as g: - g.custom_show_command('show', 'get_management_policy') - g.custom_command('create', 'create_management_policies') - g.generic_update_command('update', getter_name='get_management_policy', - getter_type=management_policy_custom_type, - setter_name='update_management_policies', - setter_type=management_policy_custom_type) - g.custom_command('delete', 'delete_management_policy') - - with self.command_group('storage account network-rule', storage_account_sdk, - custom_command_type=storage_account_custom_type, - resource_type=ResourceType.MGMT_STORAGE, min_api='2017-06-01') as g: - g.custom_command('add', 'add_network_rule') - g.custom_command('list', 'list_network_rules') - g.custom_command('remove', 'remove_network_rule') - - with self.command_group('storage account private-endpoint-connection', private_endpoint_sdk, - custom_command_type=private_endpoint_custom_type, is_preview=True, - resource_type=ResourceType.MGMT_STORAGE, min_api='2019-06-01') as g: - from ._validators import validate_private_endpoint_connection_id - g.command('delete', 'delete', confirmation=True, validator=validate_private_endpoint_connection_id) - g.show_command('show', 'get', validator=validate_private_endpoint_connection_id) - g.custom_command('approve', 'approve_private_endpoint_connection', - validator=validate_private_endpoint_connection_id) - g.custom_command('reject', 'reject_private_endpoint_connection', - validator=validate_private_endpoint_connection_id) - - with self.command_group('storage account private-link-resource', private_link_resource_sdk, - resource_type=ResourceType.MGMT_STORAGE) as g: - from azure.cli.core.commands.transform import gen_dict_to_list_transform - g.command('list', 'list_by_storage_account', is_preview=True, min_api='2019-06-01', - transform=gen_dict_to_list_transform(key="value")) - - with self.command_group('storage account blob-service-properties', blob_service_mgmt_sdk, - custom_command_type=storage_account_custom_type, - resource_type=ResourceType.MGMT_STORAGE, min_api='2018-07-01', is_preview=True) as g: - g.show_command('show', 'get_service_properties') - g.generic_update_command('update', - getter_name='get_service_properties', - setter_name='set_service_properties', - custom_func_name='update_blob_service_properties') - - with self.command_group('storage account file-service-properties', file_service_mgmt_sdk, - custom_command_type=get_custom_sdk('account', client_factory=cf_mgmt_file_services, - resource_type=ResourceType.MGMT_STORAGE), - resource_type=ResourceType.MGMT_STORAGE, min_api='2019-06-01', is_preview=True) as g: - g.show_command('show', 'get_service_properties') - g.custom_command('update', 'update_file_service_properties') - - with self.command_group('storage logging', get_custom_sdk('logging', multi_service_properties_factory)) as g: - from ._transformers_azure_stack import transform_logging_list_output - g.storage_command('update', 'set_logging') - g.storage_command('show', 'get_logging', - table_transformer=transform_logging_list_output, - exception_handler=show_exception_handler) - g.storage_command('off', 'disable_logging', is_preview=True) - - with self.command_group('storage metrics', get_custom_sdk('metrics', multi_service_properties_factory)) as g: - from ._transformers_azure_stack import transform_metrics_list_output - g.storage_command('update', 'set_metrics') - g.storage_command('show', 'get_metrics', - table_transformer=transform_metrics_list_output, - exception_handler=show_exception_handler) - - base_blob_sdk = CliCommandType( - operations_tmpl='azure.multiapi.storage.blob.baseblobservice#BaseBlobService.{}', - client_factory=blob_data_service_factory, - resource_type=ResourceType.DATA_STORAGE) - - with self.command_group('storage blob', command_type=block_blob_sdk, - custom_command_type=get_custom_sdk('blob_azure_stack', blob_data_service_factory)) as g: - from ._format import transform_boolean_for_table, transform_blob_output - from ._transformers_azure_stack import (transform_storage_list_output, transform_url, - create_boolean_result_output_transformer) - from ._validators_azure_stack import (process_blob_download_batch_parameters, - process_blob_delete_batch_parameters, - process_blob_upload_batch_parameters) - - g.storage_command_oauth('list', 'list_blobs', transform=transform_storage_list_output, - table_transformer=transform_blob_output) - g.storage_command_oauth( - 'download', 'get_blob_to_path', table_transformer=transform_blob_output) - g.storage_custom_command_oauth('generate-sas', 'generate_sas_blob_uri') - g.storage_custom_command_oauth( - 'url', 'create_blob_url', transform=transform_url) - g.storage_command_oauth('snapshot', 'snapshot_blob') - g.storage_command_oauth('update', 'set_blob_properties') - g.storage_command_oauth( - 'exists', 'exists', transform=create_boolean_result_output_transformer('exists')) - g.storage_command_oauth('delete', 'delete_blob') - g.storage_command_oauth('undelete', 'undelete_blob', - transform=create_boolean_result_output_transformer( - 'undeleted'), - table_transformer=transform_boolean_for_table, min_api='2017-07-29') - - g.storage_custom_command_oauth('set-tier', 'set_blob_tier') - g.storage_custom_command_oauth('upload', 'upload_blob', - doc_string_source='blob#BlockBlobService.create_blob_from_path') - g.storage_custom_command_oauth('upload-batch', 'storage_blob_upload_batch', - validator=process_blob_upload_batch_parameters) - g.storage_custom_command_oauth('download-batch', 'storage_blob_download_batch', - validator=process_blob_download_batch_parameters) - g.storage_custom_command_oauth('delete-batch', 'storage_blob_delete_batch', - validator=process_blob_delete_batch_parameters) - g.storage_custom_command_oauth('show', 'show_blob', table_transformer=transform_blob_output, - client_factory=page_blob_service_factory, - doc_string_source='blob#PageBlobService.get_blob_properties', - exception_handler=show_exception_handler) - - g.storage_command_oauth( - 'metadata show', 'get_blob_metadata', exception_handler=show_exception_handler) - g.storage_command_oauth('metadata update', 'set_blob_metadata') - - g.storage_command_oauth('lease acquire', 'acquire_blob_lease') - g.storage_command_oauth('lease renew', 'renew_blob_lease') - g.storage_command_oauth('lease release', 'release_blob_lease') - g.storage_command_oauth('lease change', 'change_blob_lease') - g.storage_command_oauth('lease break', 'break_blob_lease') - - g.storage_command_oauth('copy start', 'copy_blob') - g.storage_command_oauth('copy cancel', 'abort_copy_blob') - g.storage_custom_command_oauth( - 'copy start-batch', 'storage_blob_copy_batch') - - with self.command_group('storage blob', storage_account_sdk, resource_type=ResourceType.MGMT_STORAGE, - custom_command_type=storage_blob_custom_type) as g: - g.custom_command('restore', 'restore_blob_ranges', min_api='2019-06-01', is_preview=True, supports_no_wait=True) - - with self.command_group('storage blob incremental-copy', - operations_tmpl='azure.multiapi.storage.blob.pageblobservice#PageBlobService.{}', - client_factory=page_blob_service_factory, - resource_type=ResourceType.DATA_STORAGE, - min_api='2016-05-31') as g: - g.storage_command_oauth('start', 'incremental_copy_blob') - - with self.command_group('storage blob incremental-copy', - operations_tmpl='azure.multiapi.storage.blob.blockblobservice#BlockBlobService.{}', - client_factory=page_blob_service_factory, - resource_type=ResourceType.DATA_STORAGE, - min_api='2016-05-31') as g: - g.storage_command_oauth('cancel', 'abort_copy_blob') - - with self.command_group('storage blob service-properties delete-policy', command_type=base_blob_sdk, - min_api='2017-07-29', - custom_command_type=get_custom_sdk('blob_azure_stack', blob_data_service_factory)) as g: - g.storage_command_oauth('show', 'get_blob_service_properties', - transform=lambda x: getattr( - x, 'delete_retention_policy', x), - exception_handler=show_exception_handler) - g.storage_custom_command_oauth('update', 'set_delete_policy') - - with self.command_group('storage blob service-properties', command_type=base_blob_sdk) as g: - g.storage_command_oauth( - 'show', 'get_blob_service_properties', exception_handler=show_exception_handler) - g.storage_command_oauth('update', generic_update=True, getter_name='get_blob_service_properties', - setter_type=get_custom_sdk( - 'blob_azure_stack', cf_blob_data_gen_update), - setter_name='set_service_properties', - client_factory=cf_blob_data_gen_update) - - with self.command_group('storage blob', command_type=block_blob_sdk, - custom_command_type=get_custom_sdk('azcopy', blob_data_service_factory)) as g: - g.storage_custom_command_oauth('sync', 'storage_blob_sync', is_preview=True) - - with self.command_group('storage container', command_type=block_blob_sdk, - custom_command_type=get_custom_sdk('blob_azure_stack', blob_data_service_factory)) as g: - from azure.cli.command_modules.storage._transformers_azure_stack import (transform_storage_list_output, - transform_container_permission_output, - transform_acl_list_output) - from azure.cli.command_modules.storage._format import (transform_container_list, transform_boolean_for_table, - transform_container_show) - from ._validators_azure_stack import process_container_delete_parameters, validate_client_auth_parameter - - g.storage_command_oauth('list', 'list_containers', transform=transform_storage_list_output, - table_transformer=transform_container_list) - g.storage_custom_command_oauth('delete', 'delete_container', validator=process_container_delete_parameters, - transform=create_boolean_result_output_transformer( - 'deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command_oauth('show', 'get_container_properties', table_transformer=transform_container_show, - exception_handler=show_exception_handler) - g.storage_custom_command_oauth('create', 'create_container', validator=validate_client_auth_parameter, - client_factory=None, - transform=create_boolean_result_output_transformer('created'), - table_transformer=transform_boolean_for_table) - g.storage_custom_command_oauth('generate-sas', 'generate_container_shared_access_signature', - min_api='2018-11-09') - g.storage_command_oauth( - 'generate-sas', 'generate_container_shared_access_signature', max_api='2018-03-28') - g.storage_command_oauth('exists', 'exists', transform=create_boolean_result_output_transformer('exists'), - table_transformer=transform_boolean_for_table) - g.storage_command_oauth('set-permission', 'set_container_acl') - g.storage_command_oauth('show-permission', 'get_container_acl', - transform=transform_container_permission_output) - g.storage_command_oauth('metadata update', 'set_container_metadata') - g.storage_command_oauth( - 'metadata show', 'get_container_metadata', exception_handler=show_exception_handler) - - g.storage_command_oauth('lease acquire', 'acquire_container_lease') - g.storage_command_oauth('lease renew', 'renew_container_lease') - g.storage_command_oauth('lease release', 'release_container_lease') - g.storage_command_oauth('lease change', 'change_container_lease') - g.storage_command_oauth('lease break', 'break_container_lease') - - with self.command_group('storage container', command_type=block_blob_sdk, - custom_command_type=get_custom_sdk('acl', blob_data_service_factory)) as g: - g.storage_custom_command_oauth('policy create', 'create_acl_policy') - g.storage_custom_command_oauth('policy delete', 'delete_acl_policy') - g.storage_custom_command_oauth( - 'policy update', 'set_acl_policy', min_api='2017-04-17') - g.storage_custom_command_oauth( - 'policy show', 'get_acl_policy', exception_handler=show_exception_handler) - g.storage_custom_command_oauth( - 'policy list', 'list_acl_policies', table_transformer=transform_acl_list_output) - - blob_container_mgmt_sdk = CliCommandType( - operations_tmpl='azure.mgmt.storage.operations#BlobContainersOperations.{}', - client_factory=cf_blob_container_mgmt, - resource_type=ResourceType.MGMT_STORAGE - ) - - with self.command_group('storage container immutability-policy', command_type=blob_container_mgmt_sdk, - custom_command_type=get_custom_sdk('blob', cf_blob_container_mgmt, - resource_type=ResourceType.MGMT_STORAGE), - min_api='2018-02-01') as g: - from azure.cli.command_modules.storage._transformers import transform_immutability_policy - g.show_command('show', 'get_immutability_policy', - transform=transform_immutability_policy) - g.custom_command('create', 'create_or_update_immutability_policy') - g.command('delete', 'delete_immutability_policy', - transform=lambda x: None) - g.command('lock', 'lock_immutability_policy') - g.custom_command('extend', 'extend_immutability_policy') - - with self.command_group('storage container legal-hold', command_type=blob_container_mgmt_sdk, - custom_command_type=get_custom_sdk('blob', cf_blob_container_mgmt, - resource_type=ResourceType.MGMT_STORAGE), - min_api='2018-02-01') as g: - g.custom_command('set', 'set_legal_hold') - g.custom_command('clear', 'clear_legal_hold') - g.show_command( - 'show', 'get', transform=lambda x: getattr(x, 'legal_hold', x)) - - file_sdk = CliCommandType( - operations_tmpl='azure.multiapi.storage.file.fileservice#FileService.{}', - client_factory=file_data_service_factory, - resource_type=ResourceType.DATA_STORAGE) - - with self.command_group('storage share-rm', command_type=file_shares_mgmt_sdk, - custom_command_type=get_custom_sdk('file_azure_stack', - cf_mgmt_file_shares, - resource_type=ResourceType.MGMT_STORAGE), - resource_type=ResourceType.MGMT_STORAGE, min_api='2019-04-01', is_preview=True) as g: - g.command('create', 'create') - g.command('delete', 'delete', confirmation=True) - g.custom_command('exists', '_file_share_exists', transform=create_boolean_result_output_transformer('exists')) - g.command('list', 'list') - g.show_command('show', 'get') - g.command('update', 'update') - - with self.command_group('storage share', command_type=file_sdk, - custom_command_type=get_custom_sdk('file_azure_stack', file_data_service_factory)) as g: - from ._format import (transform_share_list, - transform_boolean_for_table) - g.storage_command('list', 'list_shares', transform=transform_storage_list_output, - table_transformer=transform_share_list) - g.storage_command('create', 'create_share', transform=create_boolean_result_output_transformer('created'), - table_transformer=transform_boolean_for_table) - g.storage_command('delete', 'delete_share', transform=create_boolean_result_output_transformer('deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command( - 'generate-sas', 'generate_share_shared_access_signature') - g.storage_command('stats', 'get_share_stats') - g.storage_command('show', 'get_share_properties', - exception_handler=show_exception_handler) - g.storage_command('update', 'set_share_properties') - g.storage_command('snapshot', 'snapshot_share', min_api='2017-04-17') - g.storage_command( - 'exists', 'exists', transform=create_boolean_result_output_transformer('exists')) - g.storage_custom_command( - 'url', 'create_share_url', transform=transform_url) - - g.storage_command('metadata show', 'get_share_metadata', - exception_handler=show_exception_handler) - g.storage_command('metadata update', 'set_share_metadata') - - with self.command_group('storage share policy', command_type=file_sdk, - custom_command_type=get_custom_sdk('acl', file_data_service_factory)) as g: - g.storage_custom_command('create', 'create_acl_policy') - g.storage_custom_command('delete', 'delete_acl_policy') - g.storage_custom_command( - 'show', 'get_acl_policy', exception_handler=show_exception_handler) - g.storage_custom_command( - 'list', 'list_acl_policies', table_transformer=transform_acl_list_output) - g.storage_custom_command('update', 'set_acl_policy') - - with self.command_group('storage directory', command_type=file_sdk, - custom_command_type=get_custom_sdk('directory_azure_stack', - file_data_service_factory)) as g: - from ._format import transform_file_output - from ._format_azure_stack import transform_file_directory_result - - g.storage_command('create', 'create_directory', transform=create_boolean_result_output_transformer('created'), - table_transformer=transform_boolean_for_table) - g.storage_command('delete', 'delete_directory', transform=create_boolean_result_output_transformer('deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command('show', 'get_directory_properties', table_transformer=transform_file_output, - exception_handler=show_exception_handler) - g.storage_command( - 'exists', 'exists', transform=create_boolean_result_output_transformer('exists')) - g.storage_command('metadata show', 'get_directory_metadata', - exception_handler=show_exception_handler) - g.storage_command('metadata update', 'set_directory_metadata') - g.storage_custom_command('list', 'list_share_directories', - transform=transform_file_directory_result( - self.cli_ctx), - table_transformer=transform_file_output, - doc_string_source='file#FileService.list_directories_and_files') - - with self.command_group('storage file', command_type=file_sdk, - custom_command_type=get_custom_sdk('file_azure_stack', file_data_service_factory)) as g: - from ._format import transform_boolean_for_table, transform_file_output - from ._transformers_azure_stack import transform_url - g.storage_custom_command('list', 'list_share_files', transform=transform_file_directory_result(self.cli_ctx), - table_transformer=transform_file_output, - doc_string_source='file#FileService.list_directories_and_files') - g.storage_command('delete', 'delete_file', transform=create_boolean_result_output_transformer('deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command('resize', 'resize_file') - g.storage_custom_command( - 'url', 'create_file_url', transform=transform_url) - g.storage_command( - 'generate-sas', 'generate_file_shared_access_signature') - g.storage_command('show', 'get_file_properties', table_transformer=transform_file_output, - exception_handler=show_exception_handler) - g.storage_command('update', 'set_file_properties') - g.storage_command( - 'exists', 'exists', transform=create_boolean_result_output_transformer('exists')) - g.storage_command('download', 'get_file_to_path') - g.storage_command('upload', 'create_file_from_path') - g.storage_command('metadata show', 'get_file_metadata', - exception_handler=show_exception_handler) - g.storage_command('metadata update', 'set_file_metadata') - g.storage_command('copy start', 'copy_file') - g.storage_command('copy cancel', 'abort_copy_file') - g.storage_custom_command('upload-batch', 'storage_file_upload_batch') - g.storage_custom_command( - 'download-batch', 'storage_file_download_batch') - g.storage_custom_command('delete-batch', 'storage_file_delete_batch') - g.storage_custom_command('copy start-batch', 'storage_file_copy_batch') - - with self.command_group('storage cors', get_custom_sdk('cors_azure_stack', multi_service_properties_factory)) as g: - from ._transformers_azure_stack import transform_cors_list_output - - g.storage_command('add', 'add_cors') - g.storage_command('clear', 'clear_cors') - g.storage_command('list', 'list_cors', - transform=transform_cors_list_output) - - queue_sdk = CliCommandType(operations_tmpl='azure.multiapi.storage.queue.queueservice#QueueService.{}', - client_factory=queue_data_service_factory, - resource_type=ResourceType.DATA_STORAGE) - - with self.command_group('storage queue', queue_sdk, - custom_command_type=get_custom_sdk('acl', queue_data_service_factory)) as g: - from ._format import transform_boolean_for_table - from ._transformers import create_boolean_result_output_transformer - - g.storage_command_oauth('list', 'list_queues', - transform=transform_storage_list_output) - g.storage_command_oauth('create', 'create_queue', transform=create_boolean_result_output_transformer('created'), - table_transformer=transform_boolean_for_table) - g.storage_command_oauth('delete', 'delete_queue', transform=create_boolean_result_output_transformer('deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command_oauth( - 'generate-sas', 'generate_queue_shared_access_signature') - g.storage_command_oauth( - 'stats', 'get_queue_service_stats', min_api='2016-05-31') - g.storage_command_oauth( - 'exists', 'exists', transform=create_boolean_result_output_transformer('exists')) - - g.storage_command_oauth( - 'metadata show', 'get_queue_metadata', exception_handler=show_exception_handler) - g.storage_command_oauth('metadata update', 'set_queue_metadata') - - g.storage_custom_command_oauth('policy create', 'create_acl_policy') - g.storage_custom_command_oauth('policy delete', 'delete_acl_policy') - g.storage_custom_command_oauth( - 'policy show', 'get_acl_policy', exception_handler=show_exception_handler) - g.storage_custom_command_oauth( - 'policy list', 'list_acl_policies', table_transformer=transform_acl_list_output) - g.storage_custom_command_oauth('policy update', 'set_acl_policy') - - with self.command_group('storage message', queue_sdk) as g: - from ._transformers import create_boolean_result_output_transformer - from ._format import transform_message_show - - g.storage_command_oauth('put', 'put_message') - g.storage_command_oauth('get', 'get_messages', - table_transformer=transform_message_show) - g.storage_command_oauth('peek', 'peek_messages', - table_transformer=transform_message_show) - g.storage_command_oauth('delete', 'delete_message', - transform=create_boolean_result_output_transformer( - 'deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command_oauth('clear', 'clear_messages') - g.storage_command_oauth('update', 'update_message') - - if cosmosdb_table_exists(self.cli_ctx): - table_sdk = CliCommandType(operations_tmpl='azure.multiapi.cosmosdb.table.tableservice#TableService.{}', - client_factory=table_data_service_factory, - resource_type=ResourceType.DATA_COSMOS_TABLE) - else: - table_sdk = CliCommandType(operations_tmpl='azure.multiapi.storage.table.tableservice#TableService.{}', - client_factory=table_data_service_factory, - resource_type=ResourceType.DATA_COSMOS_TABLE) - - with self.command_group('storage table', table_sdk, - custom_command_type=get_custom_sdk('acl', table_data_service_factory)) as g: - from ._format import transform_boolean_for_table - from ._transformers import create_boolean_result_output_transformer - - g.storage_command('create', 'create_table', transform=create_boolean_result_output_transformer('created'), - table_transformer=transform_boolean_for_table) - g.storage_command('delete', 'delete_table', transform=create_boolean_result_output_transformer('deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command( - 'exists', 'exists', transform=create_boolean_result_output_transformer('exists')) - g.storage_command( - 'generate-sas', 'generate_table_shared_access_signature') - g.storage_command('list', 'list_tables', - transform=transform_storage_list_output) - g.storage_command('stats', 'get_table_service_stats', - min_api='2016-05-31') - - g.storage_custom_command('policy create', 'create_acl_policy') - g.storage_custom_command('policy delete', 'delete_acl_policy') - g.storage_custom_command( - 'policy show', 'get_acl_policy', exception_handler=show_exception_handler) - g.storage_custom_command( - 'policy list', 'list_acl_policies', table_transformer=transform_acl_list_output) - g.storage_custom_command('policy update', 'set_acl_policy') - - with self.command_group('storage entity', table_sdk, - custom_command_type=get_custom_sdk('table', table_data_service_factory)) as g: - from ._format import transform_boolean_for_table, transform_entity_show - from ._transformers import (create_boolean_result_output_transformer, - transform_entity_query_output, - transform_entities_result, - transform_entity_result) - - g.storage_command('query', - 'query_entities', - table_transformer=transform_entity_query_output, - transform=transform_entities_result) - g.storage_command('replace', 'update_entity') - g.storage_command('merge', 'merge_entity') - g.storage_command('delete', 'delete_entity', transform=create_boolean_result_output_transformer('deleted'), - table_transformer=transform_boolean_for_table) - g.storage_command('show', 'get_entity', table_transformer=transform_entity_show, - exception_handler=show_exception_handler, - transform=transform_entity_result) - g.storage_custom_command('insert', 'insert_table_entity') diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/blob_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/operations/blob_azure_stack.py deleted file mode 100644 index 54ef462fbe5..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/blob_azure_stack.py +++ /dev/null @@ -1,679 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import os -from datetime import datetime - -from azure.cli.core.profiles import ResourceType -from azure.cli.core.util import sdk_no_wait -from azure.cli.command_modules.storage.url_quote_util import encode_for_url, make_encoded_file_url_and_params -from azure.cli.command_modules.storage.util import (create_blob_service_from_storage_client, - create_file_share_from_storage_client, - create_short_lived_share_sas, - create_short_lived_container_sas, - filter_none, collect_blobs, collect_blob_objects, collect_files, - mkdir_p, guess_content_type, normalize_blob_file_path, - check_precondition_success) -from azure.core.exceptions import HttpResponseError -from knack.log import get_logger -from knack.util import CLIError - -logger = get_logger(__name__) - - -def set_legal_hold(cmd, client, container_name, account_name, tags, resource_group_name=None): - LegalHold = cmd.get_models('LegalHold', resource_type=ResourceType.MGMT_STORAGE) - legal_hold = LegalHold(tags=tags) - return client.set_legal_hold(resource_group_name, account_name, container_name, legal_hold) - - -def clear_legal_hold(cmd, client, container_name, account_name, tags, resource_group_name=None): - LegalHold = cmd.get_models('LegalHold', resource_type=ResourceType.MGMT_STORAGE) - legal_hold = LegalHold(tags=tags) - return client.clear_legal_hold(resource_group_name, account_name, container_name, legal_hold) - - -def create_or_update_immutability_policy(cmd, client, container_name, account_name, - resource_group_name=None, allow_protected_append_writes=None, - period=None, if_match=None): - ImmutabilityPolicy = cmd.get_models('ImmutabilityPolicy', resource_type=ResourceType.MGMT_STORAGE) - immutability_policy = ImmutabilityPolicy(immutability_period_since_creation_in_days=period, - allow_protected_append_writes=allow_protected_append_writes) - return client.create_or_update_immutability_policy(resource_group_name, account_name, container_name, - if_match, immutability_policy) - - -def extend_immutability_policy(cmd, client, container_name, account_name, - resource_group_name=None, allow_protected_append_writes=None, - period=None, if_match=None): - ImmutabilityPolicy = cmd.get_models('ImmutabilityPolicy', resource_type=ResourceType.MGMT_STORAGE) - immutability_policy = ImmutabilityPolicy(immutability_period_since_creation_in_days=period, - allow_protected_append_writes=allow_protected_append_writes) - return client.extend_immutability_policy(resource_group_name, account_name, container_name, - if_match, immutability_policy) - - -def create_container(cmd, container_name, resource_group_name=None, account_name=None, - metadata=None, public_access=None, fail_on_exist=False, timeout=None, - default_encryption_scope=None, prevent_encryption_scope_override=None, **kwargs): - if default_encryption_scope is not None or prevent_encryption_scope_override is not None: - from .._client_factory import storage_client_factory - client = storage_client_factory(cmd.cli_ctx).blob_containers - BlobContainer = cmd.get_models('BlobContainer', resource_type=ResourceType.MGMT_STORAGE) - blob_container = BlobContainer(default_encryption_scope=default_encryption_scope, - deny_encryption_scope_override=prevent_encryption_scope_override) - container = client.create(resource_group_name=resource_group_name, account_name=account_name, - container_name=container_name, blob_container=blob_container) - return container is not None - - from .._client_factory import blob_data_service_factory - kwargs['account_name'] = account_name - client = blob_data_service_factory(cmd.cli_ctx, kwargs) - return client.create_container(container_name, metadata=metadata, public_access=public_access, - fail_on_exist=fail_on_exist, timeout=timeout) - - -def delete_container(client, container_name, fail_not_exist=False, lease_id=None, if_modified_since=None, - if_unmodified_since=None, timeout=None, bypass_immutability_policy=False, - processed_resource_group=None, processed_account_name=None, mgmt_client=None): - if bypass_immutability_policy: - return mgmt_client.blob_containers.delete(processed_resource_group, processed_account_name, container_name) - return client.delete_container( - container_name, fail_not_exist=fail_not_exist, lease_id=lease_id, if_modified_since=if_modified_since, - if_unmodified_since=if_unmodified_since, timeout=timeout) - - -def restore_blob_ranges(cmd, client, resource_group_name, account_name, time_to_restore, blob_ranges=None, - no_wait=False): - - if blob_ranges is None: - BlobRestoreRange = cmd.get_models("BlobRestoreRange") - blob_ranges = [BlobRestoreRange(start_range="", end_range="")] - restore_parameters = cmd.get_models("BlobRestoreParameters")(time_to_restore=time_to_restore, - blob_ranges=blob_ranges) - return sdk_no_wait(no_wait, client.begin_restore_blob_ranges, resource_group_name=resource_group_name, - account_name=account_name, parameters=restore_parameters) - - -def set_blob_tier(client, container_name, blob_name, tier, blob_type='block', timeout=None): - if blob_type == 'block': - return client.set_standard_blob_tier(container_name=container_name, blob_name=blob_name, - standard_blob_tier=tier, timeout=timeout) - if blob_type == 'page': - return client.set_premium_page_blob_tier(container_name=container_name, blob_name=blob_name, - premium_page_blob_tier=tier, timeout=timeout) - raise ValueError('Blob tier is only applicable to block or page blob.') - - -def set_delete_policy(client, enable=None, days_retained=None): - policy = client.get_blob_service_properties().delete_retention_policy - - if enable is not None: - policy.enabled = enable == 'true' - if days_retained is not None: - policy.days = days_retained - - if policy.enabled and not policy.days: - raise CLIError("must specify days-retained") - - client.set_blob_service_properties(delete_retention_policy=policy) - return client.get_blob_service_properties().delete_retention_policy - - -def set_service_properties(client, parameters, delete_retention=None, delete_retention_period=None, - static_website=None, index_document=None, error_document_404_path=None): - # update - kwargs = {} - if hasattr(parameters, 'delete_retention_policy'): - kwargs['delete_retention_policy'] = parameters.delete_retention_policy - if delete_retention is not None: - parameters.delete_retention_policy.enabled = delete_retention - if delete_retention_period is not None: - parameters.delete_retention_policy.days = delete_retention_period - - if hasattr(parameters, 'static_website'): - kwargs['static_website'] = parameters.static_website - elif any(param is not None for param in [static_website, index_document, error_document_404_path]): - raise CLIError('Static websites are only supported for StorageV2 (general-purpose v2) accounts.') - if static_website is not None: - parameters.static_website.enabled = static_website - if index_document is not None: - parameters.static_website.index_document = index_document - if error_document_404_path is not None: - parameters.static_website.error_document_404_path = error_document_404_path - if hasattr(parameters, 'hour_metrics'): - kwargs['hour_metrics'] = parameters.hour_metrics - if hasattr(parameters, 'logging'): - kwargs['logging'] = parameters.logging - if hasattr(parameters, 'minute_metrics'): - kwargs['minute_metrics'] = parameters.minute_metrics - if hasattr(parameters, 'cors'): - kwargs['cors'] = parameters.cors - - # checks - policy = kwargs.get('delete_retention_policy', None) - if policy and policy.enabled and not policy.days: - raise CLIError("must specify days-retained") - - client.set_blob_service_properties(**kwargs) - return client.get_blob_service_properties() - - -def storage_blob_copy_batch(cmd, client, source_client, container_name=None, - destination_path=None, source_container=None, source_share=None, - source_sas=None, pattern=None, dryrun=False): - """Copy a group of blob or files to a blob container.""" - if dryrun: - logger.warning('copy files or blobs to blob container') - logger.warning(' account %s', client.account_name) - logger.warning(' container %s', container_name) - logger.warning(' source %s', source_container or source_share) - logger.warning('source type %s', 'blob' if source_container else 'file') - logger.warning(' pattern %s', pattern) - logger.warning(' operations') - - if source_container: - # copy blobs for blob container - - # if the source client is None, recreate one from the destination client. - source_client = source_client or create_blob_service_from_storage_client(cmd, client) - if not source_sas: - source_sas = create_short_lived_container_sas(cmd, source_client.account_name, source_client.account_key, - source_container) - - # pylint: disable=inconsistent-return-statements - def action_blob_copy(blob_name): - if dryrun: - logger.warning(' - copy blob %s', blob_name) - else: - return _copy_blob_to_blob_container(client, source_client, container_name, destination_path, - source_container, source_sas, blob_name) - - return list(filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, - source_container, - pattern))) - - if source_share: - # copy blob from file share - - # if the source client is None, recreate one from the destination client. - source_client = source_client or create_file_share_from_storage_client(cmd, client) - - if not source_sas: - source_sas = create_short_lived_share_sas(cmd, source_client.account_name, source_client.account_key, - source_share) - - # pylint: disable=inconsistent-return-statements - def action_file_copy(file_info): - dir_name, file_name = file_info - if dryrun: - logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) - else: - return _copy_file_to_blob_container(client, source_client, container_name, destination_path, - source_share, source_sas, dir_name, file_name) - - return list(filter_none(action_file_copy(file) for file in collect_files(cmd, - source_client, - source_share, - pattern))) - raise ValueError('Fail to find source. Neither blob container or file share is specified') - - -# pylint: disable=unused-argument -def storage_blob_download_batch(client, source, destination, source_container_name, pattern=None, dryrun=False, - progress_callback=None, max_connections=2): - - def _download_blob(blob_service, container, destination_folder, normalized_blob_name, blob_name): - # TODO: try catch IO exception - destination_path = os.path.join(destination_folder, normalized_blob_name) - destination_folder = os.path.dirname(destination_path) - if not os.path.exists(destination_folder): - mkdir_p(destination_folder) - - blob = blob_service.get_blob_to_path(container, blob_name, destination_path, max_connections=max_connections, - progress_callback=progress_callback) - return blob.name - - source_blobs = collect_blobs(client, source_container_name, pattern) - blobs_to_download = {} - for blob_name in source_blobs: - # remove starting path seperator and normalize - normalized_blob_name = normalize_blob_file_path(None, blob_name) - if normalized_blob_name in blobs_to_download: - raise CLIError('Multiple blobs with download path: `{}`. As a solution, use the `--pattern` parameter ' - 'to select for a subset of blobs to download OR utilize the `storage blob download` ' - 'command instead to download individual blobs.'.format(normalized_blob_name)) - blobs_to_download[normalized_blob_name] = blob_name - - if dryrun: - logger.warning('download action: from %s to %s', source, destination) - logger.warning(' pattern %s', pattern) - logger.warning(' container %s', source_container_name) - logger.warning(' total %d', len(source_blobs)) - logger.warning(' operations') - for b in source_blobs: - logger.warning(' - %s', b) - return [] - - # Tell progress reporter to reuse the same hook - if progress_callback: - progress_callback.reuse = True - - results = [] - for index, blob_normed in enumerate(blobs_to_download): - # add blob name and number to progress message - if progress_callback: - progress_callback.message = '{}/{}: "{}"'.format( - index + 1, len(blobs_to_download), blobs_to_download[blob_normed]) - results.append(_download_blob( - client, source_container_name, destination, blob_normed, blobs_to_download[blob_normed])) - - # end progress hook - if progress_callback: - progress_callback.hook.end() - - return results - - -def storage_blob_upload_batch(cmd, client, source, destination, pattern=None, # pylint: disable=too-many-locals - source_files=None, destination_path=None, - destination_container_name=None, blob_type=None, - content_settings=None, metadata=None, validate_content=False, - maxsize_condition=None, max_connections=2, lease_id=None, progress_callback=None, - if_modified_since=None, if_unmodified_since=None, if_match=None, - if_none_match=None, timeout=None, dryrun=False): - def _create_return_result(blob_name, blob_content_settings, upload_result=None): - blob_name = normalize_blob_file_path(destination_path, blob_name) - return { - 'Blob': client.make_blob_url(destination_container_name, blob_name), - 'Type': blob_content_settings.content_type, - 'Last Modified': upload_result.last_modified if upload_result else None, - 'eTag': upload_result.etag if upload_result else None} - - source_files = source_files or [] - t_content_settings = cmd.get_models('blob.models#ContentSettings') - - results = [] - if dryrun: - logger.info('upload action: from %s to %s', source, destination) - logger.info(' pattern %s', pattern) - logger.info(' container %s', destination_container_name) - logger.info(' type %s', blob_type) - logger.info(' total %d', len(source_files)) - results = [] - for src, dst in source_files: - results.append(_create_return_result(dst, guess_content_type(src, content_settings, t_content_settings))) - else: - @check_precondition_success - def _upload_blob(*args, **kwargs): - return upload_blob(*args, **kwargs) - - # Tell progress reporter to reuse the same hook - if progress_callback: - progress_callback.reuse = True - - for index, source_file in enumerate(source_files): - src, dst = source_file - # logger.warning('uploading %s', src) - guessed_content_settings = guess_content_type(src, content_settings, t_content_settings) - - # add blob name and number to progress message - if progress_callback: - progress_callback.message = '{}/{}: "{}"'.format( - index + 1, len(source_files), normalize_blob_file_path(destination_path, dst)) - - include, result = _upload_blob(cmd, client, destination_container_name, - normalize_blob_file_path(destination_path, dst), src, - blob_type=blob_type, content_settings=guessed_content_settings, - metadata=metadata, validate_content=validate_content, - maxsize_condition=maxsize_condition, max_connections=max_connections, - lease_id=lease_id, progress_callback=progress_callback, - if_modified_since=if_modified_since, - if_unmodified_since=if_unmodified_since, if_match=if_match, - if_none_match=if_none_match, timeout=timeout) - if include: - results.append(_create_return_result(dst, guessed_content_settings, result)) - # end progress hook - if progress_callback: - progress_callback.hook.end() - num_failures = len(source_files) - len(results) - if num_failures: - logger.warning('%s of %s files not uploaded due to "Failed Precondition"', num_failures, len(source_files)) - return results - - -def transform_blob_type(cmd, blob_type): - """ - get_blob_types() will get ['block', 'page', 'append'] - transform it to BlobType in track2 - """ - BlobType = cmd.get_models('_models#BlobType', resource_type=ResourceType.DATA_STORAGE_BLOB) - if blob_type == 'block': - return BlobType.BlockBlob - if blob_type == 'page': - return BlobType.PageBlob - if blob_type == 'append': - return BlobType.AppendBlob - return None - - -# pylint: disable=too-many-locals -def upload_blob(cmd, client, container_name, blob_name, file_path, blob_type=None, content_settings=None, metadata=None, - validate_content=False, maxsize_condition=None, max_connections=2, lease_id=None, tier=None, - if_modified_since=None, if_unmodified_since=None, if_match=None, if_none_match=None, timeout=None, - progress_callback=None, encryption_scope=None): - """Upload a blob to a container.""" - - if encryption_scope: - count = os.path.getsize(file_path) - with open(file_path, 'rb') as stream: - data = stream.read(count) - from azure.core import MatchConditions - upload_args = { - 'content_settings': content_settings, - 'metadata': metadata, - 'timeout': timeout, - 'if_modified_since': if_modified_since, - 'if_unmodified_since': if_unmodified_since, - 'blob_type': transform_blob_type(cmd, blob_type), - 'validate_content': validate_content, - 'lease': lease_id, - 'max_concurrency': max_connections, - } - - if cmd.supported_api_version(min_api='2017-04-17') and tier: - upload_args['premium_page_blob_tier'] = tier - if maxsize_condition: - upload_args['maxsize_condition'] = maxsize_condition - if cmd.supported_api_version(min_api='2016-05-31'): - upload_args['validate_content'] = validate_content - - # Precondition Check - if if_match: - if if_match == '*': - upload_args['match_condition'] = MatchConditions.IfPresent - else: - upload_args['etag'] = if_match - upload_args['match_condition'] = MatchConditions.IfNotModified - - if if_none_match: - upload_args['etag'] = if_none_match - upload_args['match_condition'] = MatchConditions.IfModified - response = client.upload_blob(data=data, length=count, encryption_scope=encryption_scope, **upload_args) - if response['content_md5'] is not None: - from msrest import Serializer - response['content_md5'] = Serializer.serialize_bytearray(response['content_md5']) - return response - - t_content_settings = cmd.get_models('blob.models#ContentSettings') - content_settings = guess_content_type(file_path, content_settings, t_content_settings) - - def upload_append_blob(): - check_blob_args = { - 'container_name': container_name, - 'blob_name': blob_name, - 'lease_id': lease_id, - 'if_modified_since': if_modified_since, - 'if_unmodified_since': if_unmodified_since, - 'if_match': if_match, - 'if_none_match': if_none_match, - 'timeout': timeout - } - - if client.exists(container_name, blob_name): - # used to check for the preconditions as append_blob_from_path() cannot - client.get_blob_properties(**check_blob_args) - else: - client.create_blob(content_settings=content_settings, metadata=metadata, **check_blob_args) - - append_blob_args = { - 'container_name': container_name, - 'blob_name': blob_name, - 'file_path': file_path, - 'progress_callback': progress_callback, - 'maxsize_condition': maxsize_condition, - 'lease_id': lease_id, - 'timeout': timeout - } - - if cmd.supported_api_version(min_api='2016-05-31'): - append_blob_args['validate_content'] = validate_content - - return client.append_blob_from_path(**append_blob_args) - - def upload_block_blob(): - # increase the block size to 100MB when the block list will contain more than 50,000 blocks - if os.path.isfile(file_path) and os.stat(file_path).st_size > 50000 * 4 * 1024 * 1024: - client.MAX_BLOCK_SIZE = 100 * 1024 * 1024 - client.MAX_SINGLE_PUT_SIZE = 256 * 1024 * 1024 - - create_blob_args = { - 'container_name': container_name, - 'blob_name': blob_name, - 'file_path': file_path, - 'progress_callback': progress_callback, - 'content_settings': content_settings, - 'metadata': metadata, - 'max_connections': max_connections, - 'lease_id': lease_id, - 'if_modified_since': if_modified_since, - 'if_unmodified_since': if_unmodified_since, - 'if_match': if_match, - 'if_none_match': if_none_match, - 'timeout': timeout - } - - if cmd.supported_api_version(min_api='2017-04-17') and tier: - create_blob_args['premium_page_blob_tier'] = tier - - if cmd.supported_api_version(min_api='2016-05-31'): - create_blob_args['validate_content'] = validate_content - - return client.create_blob_from_path(**create_blob_args) - - type_func = { - 'append': upload_append_blob, - 'block': upload_block_blob, - 'page': upload_block_blob # same implementation - } - return type_func[blob_type]() - - -def show_blob(cmd, client, container_name, blob_name, snapshot=None, lease_id=None, - if_modified_since=None, if_unmodified_since=None, if_match=None, - if_none_match=None, timeout=None): - blob = client.get_blob_properties( - container_name, blob_name, snapshot=snapshot, lease_id=lease_id, - if_modified_since=if_modified_since, if_unmodified_since=if_unmodified_since, if_match=if_match, - if_none_match=if_none_match, timeout=timeout) - - try: - page_ranges = None - if blob.properties.blob_type == cmd.get_models('blob.models#_BlobTypes').PageBlob: - page_ranges = client.get_page_ranges( - container_name, blob_name, snapshot=snapshot, lease_id=lease_id, if_modified_since=if_modified_since, - if_unmodified_since=if_unmodified_since, if_match=if_match, if_none_match=if_none_match, - timeout=timeout) - - blob.properties.page_ranges = page_ranges - except HttpResponseError as ex: - logger.warning("GetPageRanges failed with status code: %d, message: %s", ex.status_code, ex.message) - - return blob - - -def storage_blob_delete_batch(client, source, source_container_name, pattern=None, lease_id=None, - delete_snapshots=None, if_modified_since=None, if_unmodified_since=None, if_match=None, - if_none_match=None, timeout=None, dryrun=False): - @check_precondition_success - def _delete_blob(blob_name): - delete_blob_args = { - 'container_name': source_container_name, - 'blob_name': blob_name, - 'lease_id': lease_id, - 'delete_snapshots': delete_snapshots, - 'if_modified_since': if_modified_since, - 'if_unmodified_since': if_unmodified_since, - 'if_match': if_match, - 'if_none_match': if_none_match, - 'timeout': timeout - } - return client.delete_blob(**delete_blob_args) - - source_blobs = list(collect_blob_objects(client, source_container_name, pattern)) - - if dryrun: - from datetime import timezone - delete_blobs = [] - if_modified_since_utc = if_modified_since.replace(tzinfo=timezone.utc) if if_modified_since else None - if_unmodified_since_utc = if_unmodified_since.replace(tzinfo=timezone.utc) if if_unmodified_since else None - for blob in source_blobs: - if not if_modified_since or blob[1].properties.last_modified >= if_modified_since_utc: - if not if_unmodified_since or blob[1].properties.last_modified <= if_unmodified_since_utc: - delete_blobs.append(blob[0]) - logger.warning('delete action: from %s', source) - logger.warning(' pattern %s', pattern) - logger.warning(' container %s', source_container_name) - logger.warning(' total %d', len(delete_blobs)) - logger.warning(' operations') - for blob in delete_blobs: - logger.warning(' - %s', blob) - return [] - - results = [result for include, result in (_delete_blob(blob[0]) for blob in source_blobs) if include] - num_failures = len(source_blobs) - len(results) - if num_failures: - logger.warning('%s of %s blobs not deleted due to "Failed Precondition"', num_failures, len(source_blobs)) - - -def generate_sas_blob_uri(client, container_name, blob_name, permission=None, - expiry=None, start=None, id=None, ip=None, # pylint: disable=redefined-builtin - protocol=None, cache_control=None, content_disposition=None, - content_encoding=None, content_language=None, - content_type=None, full_uri=False, as_user=False): - if as_user: - user_delegation_key = client.get_user_delegation_key( - _get_datetime_from_string(start) if start else datetime.utcnow(), _get_datetime_from_string(expiry)) - sas_token = client.generate_blob_shared_access_signature( - container_name, blob_name, permission=permission, expiry=expiry, start=start, id=id, ip=ip, - protocol=protocol, cache_control=cache_control, content_disposition=content_disposition, - content_encoding=content_encoding, content_language=content_language, content_type=content_type, - user_delegation_key=user_delegation_key) - else: - sas_token = client.generate_blob_shared_access_signature( - container_name, blob_name, permission=permission, expiry=expiry, start=start, id=id, ip=ip, - protocol=protocol, cache_control=cache_control, content_disposition=content_disposition, - content_encoding=content_encoding, content_language=content_language, content_type=content_type) - if full_uri: - from ..url_quote_util import encode_url_path - return encode_url_path(client.make_blob_url(container_name, blob_name, protocol=protocol, sas_token=sas_token)) - return sas_token - - -def generate_container_shared_access_signature(client, container_name, permission=None, - expiry=None, start=None, id=None, ip=None, # pylint: disable=redefined-builtin - protocol=None, cache_control=None, content_disposition=None, - content_encoding=None, content_language=None, - content_type=None, as_user=False): - user_delegation_key = None - if as_user: - user_delegation_key = client.get_user_delegation_key( - _get_datetime_from_string(start) if start else datetime.utcnow(), _get_datetime_from_string(expiry)) - - return client.generate_container_shared_access_signature( - container_name, permission=permission, expiry=expiry, start=start, id=id, ip=ip, - protocol=protocol, cache_control=cache_control, content_disposition=content_disposition, - content_encoding=content_encoding, content_language=content_language, content_type=content_type, - user_delegation_key=user_delegation_key) - - -def create_blob_url(client, container_name, blob_name, protocol=None, snapshot=None): - return client.make_blob_url( - container_name, blob_name, protocol=protocol, snapshot=snapshot, sas_token=client.sas_token) - - -def _copy_blob_to_blob_container(blob_service, source_blob_service, destination_container, destination_path, - source_container, source_sas, source_blob_name): - from azure.common import AzureException - source_blob_url = source_blob_service.make_blob_url(source_container, encode_for_url(source_blob_name), - sas_token=source_sas) - destination_blob_name = normalize_blob_file_path(destination_path, source_blob_name) - try: - blob_service.copy_blob(destination_container, destination_blob_name, source_blob_url) - return blob_service.make_blob_url(destination_container, destination_blob_name) - except AzureException: - error_template = 'Failed to copy blob {} to container {}.' - raise CLIError(error_template.format(source_blob_name, destination_container)) - - -def _copy_file_to_blob_container(blob_service, source_file_service, destination_container, destination_path, - source_share, source_sas, source_file_dir, source_file_name): - from azure.common import AzureException - file_url, source_file_dir, source_file_name = \ - make_encoded_file_url_and_params(source_file_service, source_share, source_file_dir, - source_file_name, source_sas) - - source_path = os.path.join(source_file_dir, source_file_name) if source_file_dir else source_file_name - destination_blob_name = normalize_blob_file_path(destination_path, source_path) - - try: - blob_service.copy_blob(destination_container, destination_blob_name, file_url) - return blob_service.make_blob_url(destination_container, destination_blob_name) - except AzureException as ex: - error_template = 'Failed to copy file {} to container {}. {}' - raise CLIError(error_template.format(source_file_name, destination_container, ex)) - - -def _get_datetime_from_string(dt_str): - accepted_date_formats = ['%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%MZ', - '%Y-%m-%dT%HZ', '%Y-%m-%d'] - for form in accepted_date_formats: - try: - return datetime.strptime(dt_str, form) - except ValueError: - continue - raise ValueError("datetime string '{}' not valid. Valid example: 2000-12-31T12:59:59Z".format(dt_str)) - - -# ------------------Track2 Support----------------------- -def show_blob_v2(cmd, client, container_name, blob_name, snapshot=None, lease_id=None, - if_modified_since=None, if_unmodified_since=None, if_match=None, - if_none_match=None, timeout=None): - property_kwargs = { - 'lease': lease_id, - 'if_modified_since': if_modified_since, - 'if_unmodified_since': if_unmodified_since, - 'timeout': timeout - } - - # Precondition Check - from ..track2_util import _if_match, _if_none_match - if if_match: - property_kwargs = _if_match(if_match, **property_kwargs) - - if if_none_match: - property_kwargs = _if_none_match(if_none_match, **property_kwargs) - - blob = client.get_blob_properties(**property_kwargs) - - try: - page_ranges = None - if blob.blob_type == cmd.get_models('_models#BlobType', resource_type=ResourceType.DATA_STORAGE_BLOB).PageBlob: - page_ranges = client.get_page_ranges(**property_kwargs) - - blob.page_ranges = page_ranges - except HttpResponseError as ex: - logger.warning("GetPageRanges failed with status code: %d, message: %s", ex.status_code, ex.message) - - return blob - - -def set_blob_tier_v2(client, container_name, blob_name, tier, blob_type='block', rehydrate_priority=None, timeout=None): - if blob_type == 'block': - return client.set_standard_blob_tier(standard_blob_tier=tier, rehydrate_priority=rehydrate_priority, - timeout=timeout) - if blob_type == 'page': - return client.set_premium_page_blob_tier(premium_page_blob_tier=tier, timeout=timeout) - raise ValueError('Blob tier is only applicable to block or page blob.') diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/cors_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/operations/cors_azure_stack.py deleted file mode 100644 index 4a0def38685..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/cors_azure_stack.py +++ /dev/null @@ -1,21 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - - -def list_cors(client, timeout=None): - results = {} - for s in client: - results[s.name] = s.get_cors(timeout) - return results - - -def add_cors(client, origins, methods, max_age=0, exposed_headers=None, allowed_headers=None, timeout=None): - for s in client: - s.add_cors(origins, methods, max_age, exposed_headers, allowed_headers, timeout) - - -def clear_cors(client, timeout=None): - for s in client: - s.clear_cors(timeout) diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/directory_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/operations/directory_azure_stack.py deleted file mode 100644 index 4edb64865da..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/directory_azure_stack.py +++ /dev/null @@ -1,11 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - - -def list_share_directories(cmd, client, share_name, directory_name=None, timeout=None): - t_dir_properties = cmd.get_models('file.models#DirectoryProperties') - - generator = client.list_directories_and_files(share_name, directory_name, timeout=timeout) - return list(f for f in generator if isinstance(f.properties, t_dir_properties)) diff --git a/src/azure-cli/azure/cli/command_modules/storage/operations/file_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/operations/file_azure_stack.py deleted file mode 100644 index 96780c9cef9..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/operations/file_azure_stack.py +++ /dev/null @@ -1,367 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -""" -Commands for storage file share operations -""" - -import os -from knack.log import get_logger - -from azure.cli.command_modules.storage.util import (filter_none, collect_blobs, collect_files, - create_blob_service_from_storage_client, - create_short_lived_container_sas, create_short_lived_share_sas, - guess_content_type) -from azure.cli.command_modules.storage.url_quote_util import encode_for_url, make_encoded_file_url_and_params -from azure.cli.core.profiles import ResourceType - - -def create_share_rm(cmd, client, resource_group_name, account_name, share_name, metadata=None, share_quota=None, - enabled_protocols=None, root_squash=None): - - FileShare = cmd.get_models('FileShare', resource_type=ResourceType.MGMT_STORAGE) - - file_share = FileShare() - if share_quota is not None: - file_share.share_quota = share_quota - if enabled_protocols is not None: - file_share.enabled_protocols = enabled_protocols - if root_squash is not None: - file_share.root_squash = root_squash - if metadata is not None: - file_share.metadata = metadata - - return client.create(resource_group_name=resource_group_name, account_name=account_name, share_name=share_name, - file_share=file_share) - - -def update_share_rm(cmd, instance, metadata=None, share_quota=None, root_squash=None): - - FileShare = cmd.get_models('FileShare', resource_type=ResourceType.MGMT_STORAGE) - - params = FileShare( - share_quota=share_quota if share_quota is not None else instance.share_quota, - root_squash=root_squash if root_squash is not None else instance.root_squash, - metadata=metadata if metadata is not None else instance.metadata, - enabled_protocols=instance.enabled_protocols, - ) - - return params - - -def create_share_url(client, share_name, unc=None, protocol=None): - url = client.make_file_url(share_name, None, '', protocol=protocol).rstrip('/') - if unc: - url = ':'.join(url.split(':')[1:]) - return url - - -def create_file_url(client, share_name, directory_name, file_name, protocol=None): - return client.make_file_url( - share_name, directory_name, file_name, protocol=protocol, sas_token=client.sas_token) - - -def list_share_files(cmd, client, share_name, directory_name=None, timeout=None, exclude_dir=False, snapshot=None, - num_results=None, marker=None): - if cmd.supported_api_version(min_api='2017-04-17'): - generator = client.list_directories_and_files( - share_name, directory_name, timeout=timeout, num_results=num_results, marker=marker, snapshot=snapshot) - else: - generator = client.list_directories_and_files( - share_name, directory_name, timeout=timeout, num_results=num_results, marker=marker) - - if exclude_dir: - t_file_properties = cmd.get_models('file.models#FileProperties') - - return list(f for f in generator if isinstance(f.properties, t_file_properties)) - - return generator - - -def storage_file_upload_batch(cmd, client, destination, source, destination_path=None, pattern=None, dryrun=False, - validate_content=False, content_settings=None, max_connections=1, metadata=None, - progress_callback=None): - """ Upload local files to Azure Storage File Share in batch """ - - from azure.cli.command_modules.storage.util import glob_files_locally, normalize_blob_file_path - - source_files = list(glob_files_locally(source, pattern)) - logger = get_logger(__name__) - settings_class = cmd.get_models('file.models#ContentSettings') - - if dryrun: - logger.info('upload files to file share') - logger.info(' account %s', client.account_name) - logger.info(' share %s', destination) - logger.info(' total %d', len(source_files)) - return [{'File': client.make_file_url(destination, os.path.dirname(dst) or None, os.path.basename(dst)), - 'Type': guess_content_type(src, content_settings, settings_class).content_type} for src, dst in - source_files] - - # TODO: Performance improvement - # 1. Upload files in parallel - def _upload_action(src, dst): - dst = normalize_blob_file_path(destination_path, dst) - dir_name = os.path.dirname(dst) - file_name = os.path.basename(dst) - - _make_directory_in_files_share(client, destination, dir_name) - create_file_args = {'share_name': destination, 'directory_name': dir_name, 'file_name': file_name, - 'local_file_path': src, 'progress_callback': progress_callback, - 'content_settings': guess_content_type(src, content_settings, settings_class), - 'metadata': metadata, 'max_connections': max_connections} - - if cmd.supported_api_version(min_api='2016-05-31'): - create_file_args['validate_content'] = validate_content - - logger.warning('uploading %s', src) - client.create_file_from_path(**create_file_args) - - return client.make_file_url(destination, dir_name, file_name) - - return [_upload_action(src, dst) for src, dst in source_files] - - -def storage_file_download_batch(cmd, client, source, destination, pattern=None, dryrun=False, validate_content=False, - max_connections=1, progress_callback=None, snapshot=None): - """ - Download files from file share to local directory in batch - """ - - from azure.cli.command_modules.storage.util import glob_files_remotely, mkdir_p - - source_files = glob_files_remotely(cmd, client, source, pattern) - - if dryrun: - source_files_list = list(source_files) - - logger = get_logger(__name__) - logger.warning('download files from file share') - logger.warning(' account %s', client.account_name) - logger.warning(' share %s', source) - logger.warning('destination %s', destination) - logger.warning(' pattern %s', pattern) - logger.warning(' total %d', len(source_files_list)) - logger.warning(' operations') - for f in source_files_list: - logger.warning(' - %s/%s => %s', f[0], f[1], os.path.join(destination, *f)) - - return [] - - def _download_action(pair): - destination_dir = os.path.join(destination, pair[0]) - mkdir_p(destination_dir) - - get_file_args = {'share_name': source, 'directory_name': pair[0], 'file_name': pair[1], - 'file_path': os.path.join(destination, *pair), 'max_connections': max_connections, - 'progress_callback': progress_callback, 'snapshot': snapshot} - - if cmd.supported_api_version(min_api='2016-05-31'): - get_file_args['validate_content'] = validate_content - - client.get_file_to_path(**get_file_args) - return client.make_file_url(source, *pair) - - return [_download_action(f) for f in source_files] - - -def storage_file_copy_batch(cmd, client, source_client, destination_share=None, destination_path=None, - source_container=None, source_share=None, source_sas=None, pattern=None, dryrun=False, - metadata=None, timeout=None): - """ - Copy a group of files asynchronously - """ - logger = None - if dryrun: - logger = get_logger(__name__) - logger.warning('copy files or blobs to file share') - logger.warning(' account %s', client.account_name) - logger.warning(' share %s', destination_share) - logger.warning(' path %s', destination_path) - logger.warning(' source %s', source_container or source_share) - logger.warning('source type %s', 'blob' if source_container else 'file') - logger.warning(' pattern %s', pattern) - logger.warning(' operations') - - if source_container: - # copy blobs to file share - - # if the source client is None, recreate one from the destination client. - source_client = source_client or create_blob_service_from_storage_client(cmd, client) - - # the cache of existing directories in the destination file share. the cache helps to avoid - # repeatedly create existing directory so as to optimize the performance. - existing_dirs = set() - - if not source_sas: - source_sas = create_short_lived_container_sas(cmd, source_client.account_name, source_client.account_key, - source_container) - - # pylint: disable=inconsistent-return-statements - def action_blob_copy(blob_name): - if dryrun: - logger.warning(' - copy blob %s', blob_name) - else: - return _create_file_and_directory_from_blob(client, source_client, destination_share, source_container, - source_sas, blob_name, destination_dir=destination_path, - metadata=metadata, timeout=timeout, - existing_dirs=existing_dirs) - - return list( - filter_none(action_blob_copy(blob) for blob in collect_blobs(source_client, source_container, pattern))) - - if source_share: - # copy files from share to share - - # if the source client is None, assume the file share is in the same storage account as - # destination, therefore client is reused. - source_client = source_client or client - - # the cache of existing directories in the destination file share. the cache helps to avoid - # repeatedly create existing directory so as to optimize the performance. - existing_dirs = set() - - if not source_sas: - source_sas = create_short_lived_share_sas(cmd, source_client.account_name, source_client.account_key, - source_share) - - # pylint: disable=inconsistent-return-statements - def action_file_copy(file_info): - dir_name, file_name = file_info - if dryrun: - logger.warning(' - copy file %s', os.path.join(dir_name, file_name)) - else: - return _create_file_and_directory_from_file(client, source_client, destination_share, source_share, - source_sas, dir_name, file_name, - destination_dir=destination_path, metadata=metadata, - timeout=timeout, existing_dirs=existing_dirs) - - return list(filter_none( - action_file_copy(file) for file in collect_files(cmd, source_client, source_share, pattern))) - # won't happen, the validator should ensure either source_container or source_share is set - raise ValueError('Fail to find source. Neither blob container or file share is specified.') - - -def storage_file_delete_batch(cmd, client, source, pattern=None, dryrun=False, timeout=None): - """ - Delete files from file share in batch - """ - - def delete_action(file_pair): - delete_file_args = {'share_name': source, 'directory_name': file_pair[0], 'file_name': file_pair[1], - 'timeout': timeout} - - return client.delete_file(**delete_file_args) - - from azure.cli.command_modules.storage.util import glob_files_remotely - source_files = list(glob_files_remotely(cmd, client, source, pattern)) - - if dryrun: - logger = get_logger(__name__) - logger.warning('delete files from %s', source) - logger.warning(' pattern %s', pattern) - logger.warning(' share %s', source) - logger.warning(' total %d', len(source_files)) - logger.warning(' operations') - for f in source_files: - logger.warning(' - %s/%s', f[0], f[1]) - return [] - - for f in source_files: - delete_action(f) - - -def _create_file_and_directory_from_blob(file_service, blob_service, share, container, sas, blob_name, - destination_dir=None, metadata=None, timeout=None, existing_dirs=None): - """ - Copy a blob to file share and create the directory if needed. - """ - from azure.common import AzureException - from azure.cli.command_modules.storage.util import normalize_blob_file_path - - blob_url = blob_service.make_blob_url(container, encode_for_url(blob_name), sas_token=sas) - full_path = normalize_blob_file_path(destination_dir, blob_name) - file_name = os.path.basename(full_path) - dir_name = os.path.dirname(full_path) - _make_directory_in_files_share(file_service, share, dir_name, existing_dirs) - - try: - file_service.copy_file(share, dir_name, file_name, blob_url, metadata, timeout) - return file_service.make_file_url(share, dir_name, file_name) - except AzureException: - error_template = 'Failed to copy blob {} to file share {}. Please check if you have permission to read ' \ - 'source or set a correct sas token.' - from knack.util import CLIError - raise CLIError(error_template.format(blob_name, share)) - - -def _create_file_and_directory_from_file(file_service, source_file_service, share, source_share, sas, source_file_dir, - source_file_name, destination_dir=None, metadata=None, timeout=None, - existing_dirs=None): - """ - Copy a file from one file share to another - """ - from azure.common import AzureException - from azure.cli.command_modules.storage.util import normalize_blob_file_path - - file_url, source_file_dir, source_file_name = make_encoded_file_url_and_params(source_file_service, source_share, - source_file_dir, source_file_name, - sas_token=sas) - - full_path = normalize_blob_file_path(destination_dir, os.path.join(source_file_dir, source_file_name)) - file_name = os.path.basename(full_path) - dir_name = os.path.dirname(full_path) - _make_directory_in_files_share(file_service, share, dir_name, existing_dirs) - - try: - file_service.copy_file(share, dir_name, file_name, file_url, metadata, timeout) - return file_service.make_file_url(share, dir_name or None, file_name) - except AzureException: - error_template = 'Failed to copy file {} from share {} to file share {}. Please check if ' \ - 'you have right permission to read source or set a correct sas token.' - from knack.util import CLIError - raise CLIError(error_template.format(file_name, source_share, share)) - - -def _make_directory_in_files_share(file_service, file_share, directory_path, existing_dirs=None): - """ - Create directories recursively. - - This method accept a existing_dirs set which serves as the cache of existing directory. If the - parameter is given, the method will search the set first to avoid repeatedly create directory - which already exists. - """ - from azure.common import AzureHttpError - - if not directory_path: - return - - parents = [directory_path] - p = os.path.dirname(directory_path) - while p: - parents.append(p) - p = os.path.dirname(p) - - for dir_name in reversed(parents): - if existing_dirs and (dir_name in existing_dirs): - continue - - try: - file_service.create_directory(share_name=file_share, directory_name=dir_name, fail_on_exist=False) - except AzureHttpError: - from knack.util import CLIError - raise CLIError('Failed to create directory {}'.format(dir_name)) - - if existing_dirs: - existing_dirs.add(directory_path) - - -def _file_share_exists(client, resource_group_name, account_name, share_name): - from azure.core.exceptions import HttpResponseError - try: - file_share = client.get(resource_group_name, account_name, share_name, expand=None) - return file_share is not None - except HttpResponseError: - return False diff --git a/src/azure-cli/azure/cli/command_modules/storage/services_wrapper_azure_stack.py b/src/azure-cli/azure/cli/command_modules/storage/services_wrapper_azure_stack.py deleted file mode 100644 index 30253894fe8..00000000000 --- a/src/azure-cli/azure/cli/command_modules/storage/services_wrapper_azure_stack.py +++ /dev/null @@ -1,86 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -from azure.cli.core.profiles import get_sdk, ResourceType - -from ._client_factory import generic_data_service_factory - - -class ServiceProperties: - def __init__(self, cli_ctx, name, service, account_name=None, account_key=None, connection_string=None, - sas_token=None): - self.cli_ctx = cli_ctx - self.name = name - self.client = generic_data_service_factory(cli_ctx, service, name=account_name, key=account_key, - connection_string=connection_string, sas_token=sas_token) - if not self.client: - from knack.util import CLIError - raise CLIError('Failed to initialize data client.') - - def get_service_properties(self): - return getattr(self.client, 'get_{}_service_properties'.format(self.name)) - - def set_service_properties(self): - return getattr(self.client, 'set_{}_service_properties'.format(self.name)) - - def get_logging(self, timeout=None): - return self.get_service_properties()(timeout=timeout).__dict__['logging'] - - def set_logging(self, read, write, delete, retention, timeout=None, version=None): - t_logging, t_retention_policy = get_sdk(self.cli_ctx, ResourceType.DATA_STORAGE, 'Logging', 'RetentionPolicy', - mod='common.models') - - retention_policy = t_retention_policy(enabled=retention != 0, days=retention) - logging = t_logging(delete, read, write, retention_policy) - if version: - logging.version = str(version) - return self.set_service_properties()(logging=logging, timeout=timeout) - - def disable_logging(self, timeout=None): - return self.set_logging(read=False, write=False, delete=False, retention=0, timeout=timeout) - - def get_cors(self, timeout=None): - return self.get_service_properties()(timeout=timeout).__dict__['cors'] - - def add_cors(self, origins, methods, max_age, exposed_headers=None, allowed_headers=None, timeout=None): - from azure.common import AzureHttpError - - t_cors_rule = get_sdk(self.cli_ctx, ResourceType.DATA_STORAGE, 'CorsRule', mod='common.models') - cors = self.get_cors(timeout) - new_rule = t_cors_rule(origins, methods, max_age, exposed_headers, allowed_headers) - cors.append(new_rule) - try: - return self.set_service_properties()(cors=cors, timeout=timeout) - except AzureHttpError as ex: - # The service issue: https://msazure.visualstudio.com/DefaultCollection/One/_workitems/edit/1247479. - # This workaround can be removed once the service is updated. - if ex.status_code == 400 and len(cors) > 5: - from knack.util import CLIError - raise CLIError('Failed to add CORS rules. No more than 5 CORS rule can be added.') - - raise ex - - def clear_cors(self, timeout=None): - return self.set_service_properties()(cors=[], timeout=timeout) - - def get_metrics(self, interval, timeout=None): - props = self.get_service_properties()(timeout=timeout) - metrics = {} - if interval == 'both': - metrics['hour'] = props.__dict__['hour_metrics'] - metrics['minute'] = props.__dict__['minute_metrics'] - else: - metrics[interval] = props.__dict__['{}_metrics'.format(interval)] - return metrics - - def set_metrics(self, retention, hour, minute, api=None, timeout=None): - t_metrics, t_retention_policy = get_sdk(self.cli_ctx, ResourceType.DATA_STORAGE, 'Metrics', 'RetentionPolicy', - mod='common.models') - - retention_policy = t_retention_policy(enabled=retention != 0, days=retention) - hour_metrics = t_metrics(hour, api, retention_policy) if hour is not None else None - minute_metrics = t_metrics(minute, api, retention_policy) if minute is not None else None - return self.set_service_properties()( - hour_metrics=hour_metrics, minute_metrics=minute_metrics, timeout=timeout)