data_transfer.py

from google.cloud import storage_transfer

def transfer_between_posix(
        project_id: str, description: str, source_agent_pool_name: str,
        sink_agent_pool_name: str, root_directory: str,
        destination_directory: str, intermediate_bucket: str):
    """Creates a transfer between POSIX file systems."""

    client = storage_transfer.StorageTransferServiceClient()

    # GCP Project ID
    # project_id = 'my-project-id'

    # A useful description for your transfer job
    # description = 'My transfer job'

    # The agent pool associated with the POSIX data source.
    # Defaults to 'projects/{project_id}/agentPools/transfer_service_default'
    # source_agent_pool_name = 'projects/my-project/agentPools/my-agent'

    # The agent pool associated with the POSIX data sink.
    # Defaults to 'projects/{project_id}/agentPools/transfer_service_default'
    # sink_agent_pool_name = 'projects/my-project/agentPools/my-agent'

    # The root directory path on the source filesystem
    # root_directory = '/directory/to/transfer/source'

    # The root directory path on the destination filesystem
    # destination_directory = '/directory/to/transfer/sink'

    # The Google Cloud Storage bucket for intermediate storage
    # intermediate_bucket = 'my-intermediate-bucket'

    transfer_job_request = storage_transfer.CreateTransferJobRequest({
        'transfer_job': {
            'project_id': project_id,
            'description': description,
            'status': storage_transfer.TransferJob.Status.ENABLED,
            'transfer_spec': {
                'source_agent_pool_name': source_agent_pool_name,
                'sink_agent_pool_name': sink_agent_pool_name,
                'posix_data_source': {
                    'root_directory': root_directory,
                },
                'posix_data_sink': {
                    'root_directory': destination_directory,
                },
                'gcs_intermediate_data_location': {
                    'bucket_name': intermediate_bucket
                },
            }
        }
    })

    result = client.create_transfer_job(transfer_job_request)
    print(f'Created transferJob: {result.name}')