Migrate gsutil usage to gcloud storage

Author: bhandarivijay-png

dataflow/conftest.py

@@ -85,7 +85,7 @@ def bucket_name(test_name: str, location: str, unique_id: str) -> Iterator[str]:
     # Try to remove all files before deleting the bucket.
     # Deleting a bucket with too many files results in an error.
     try:
-        run_cmd("gsutil", "-m", "rm", "-rf", f"gs://{bucket_name}/*")
+        run_cmd("gcloud", "storage", "rm", "--recursive", "--continue-on-error", f"gs://{bucket_name}/*")
     except RuntimeError:
         # If no files were found and it fails, ignore the error.
         pass

dataflow/encryption-keys/README.md

@@ -25,7 +25,7 @@ Additionally, for this sample you need the following:
 
    ```sh
    export BUCKET=your-gcs-bucket
-   gsutil mb gs://$BUCKET
+   gcloud storage buckets create gs://$BUCKET
    ```
 
 1. [Create a symmetric key ring](https://cloud.google.com/kms/docs/creating-keys).
@@ -174,10 +174,10 @@ To avoid incurring charges to your GCP account for the resources used:
 
 ```sh
 # Remove only the files created by this sample.
-gsutil -m rm -rf "gs://$BUCKET/samples/dataflow/kms"
+gcloud storage rm --recursive --continue-on-error "gs://$BUCKET/samples/dataflow/kms"
 
 # [optional] Remove the Cloud Storage bucket.
-gsutil rb gs://$BUCKET
+gcloud storage buckets delete gs://$BUCKET
 
 # Remove the BigQuery table.
 bq rm -f -t $PROJECT:$DATASET.$TABLE

dataflow/flex-templates/getting_started/README.md

@@ -9,7 +9,7 @@ Make sure you have followed the
 
 ```sh
 export BUCKET="your--bucket"
-gsutil mb gs://$BUCKET
+gcloud storage buckets create gs://$BUCKET
 ```
 
 ## create an Artifact Registry repository
@@ -51,4 +51,3 @@ gcloud dataflow flex-template run "flex-`date +%Y%m%d-%H%M%S`" \
 
 For more information about building and running flex templates, see
 📝 [Use Flex Templates](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates).
-

dataflow/flex-templates/pipeline_with_dependencies/README.md

@@ -73,7 +73,7 @@ rules. It is optional.
 export PROJECT="project-id"
 export BUCKET="your-bucket"
 export REGION="us-central1"
-gsutil mb -p $PROJECT gs://$BUCKET
+gcloud storage buckets create gs://$BUCKET --project=$PROJECT
 ```
 
 ## Create an Artifact Registry repository
@@ -165,7 +165,7 @@ gcloud dataflow flex-template run "flex-`date +%Y%m%d-%H%M%S`" \
 After the pipeline finishes, use the following command to inspect the output:
 
 ```bash
-gsutil cat gs://$BUCKET/output*
+gcloud storage cat gs://$BUCKET/output*
 ```
 
 ## Optional: Update the dependencies in the requirements file and rebuild the Docker images

dataflow/flex-templates/streaming_beam/README.md

@@ -25,7 +25,7 @@ Additionally, for this sample you need the following:
 
     ```sh
     export BUCKET="your-gcs-bucket"
-    gsutil mb gs://$BUCKET
+    gcloud storage buckets create gs://$BUCKET
     ```
 
 1. Create a
@@ -231,7 +231,7 @@ The following sections describe how to delete or turn off these resources.
 1. Delete the template spec file from Cloud Storage.
 
     ```sh
-    gsutil rm $TEMPLATE_PATH
+    gcloud storage rm $TEMPLATE_PATH
     ```
 
 1. Delete the Flex Template container image from Container Registry.
@@ -277,7 +277,7 @@ The following sections describe how to delete or turn off these resources.
     > These objects cannot be recovered.
     >
     > ```sh
-    > gsutil rm -r gs://$BUCKET
+    > gcloud storage rm --recursive gs://$BUCKET
     > ```
 
 ## Limitations

dataflow/gemma-flex-template/README.md

@@ -43,7 +43,7 @@ Click [here to create a GCS bucket](https://console.cloud.google.com/storage/cre
 
 ```sh
 export GCS_BUCKET="your--bucket"
-gsutil mb gs://$GCS_BUCKET
+gcloud storage buckets create gs://$GCS_BUCKET
 ```
 
 Make sure your GCS bucket name does __not__ include the `gs://` prefix

dataflow/gemma-flex-template/e2e_test.py

@@ -92,7 +92,7 @@ def responses_subscription(
 
 @pytest.fixture(scope="session")
 def flex_template_image(utils: Utils) -> str:
-    conftest.run_cmd("gsutil", "cp", "-r", GEMMA_GCS, ".")
+    conftest.run_cmd("gcloud", "storage", "cp", "--recursive", GEMMA_GCS, ".")
     yield from utils.cloud_build_submit(NAME)
 
 

dataflow/gemma/e2e_test.py

@@ -60,7 +60,7 @@ def test_name() -> str:
 @pytest.fixture(scope="session")
 def container_image(utils: Utils) -> str:
     # Copy Gemma onto the local environment
-    conftest.run_cmd("gsutil", "cp", "-r", GEMMA_GCS, ".")
+    conftest.run_cmd("gcloud", "storage", "cp", "--recursive", GEMMA_GCS, ".")
     yield from utils.cloud_build_submit(NAME)
 
 

dataflow/run-inference/tests/e2e_test.py

@@ -95,7 +95,7 @@ def dataflow_job(
 ) -> Iterator[str]:
     # Upload the state dict to Cloud Storage.
     state_dict_gcs = f"gs://{bucket_name}/temp/state_dict.pt"
-    conftest.run_cmd("gsutil", "cp", "-n", state_dict_path, state_dict_gcs)
+    conftest.run_cmd("gcloud", "storage", "cp", "--no-clobber", state_dict_path, state_dict_gcs)
 
     # Launch the streaming Dataflow pipeline.
     conftest.run_cmd(

dataflow/run_template/README.md

@@ -29,7 +29,7 @@ Additionally, for this sample you need the following:
 
    ```sh
    export BUCKET=your-gcs-bucket
-   gsutil mb gs://$BUCKET
+   gcloud storage buckets create gs://$BUCKET
    ```
 
 1. Clone the `python-docs-samples` repository.