Merge pull request #13618 from bhandarivijay-png:ai-gsutil-migration-1f4c60eec73c4db7a4ef14ed8519d40f

PiperOrigin-RevId: 884853554

Author: TF Object Detection Team

official/legacy/bert/bert_cloud_tpu.md

@@ -14,7 +14,7 @@ gcloud config set project ${PROJECT_ID}
 ```
 4. Create a Cloud Storage bucket using the following command:
 ```
-gsutil mb -p ${PROJECT_ID} -c standard -l europe-west4 -b on gs://your-bucket-name
+gcloud storage buckets create gs://your-bucket-name --project ${PROJECT_ID} --default-storage-class standard --location europe-west4 --uniform-bucket-level-access
 ```
 This Cloud Storage bucket stores the data you use to train your model and the training results.
 5. Launch a Compute Engine VM and Cloud TPU using the ctpu up command.
@@ -98,9 +98,9 @@ $ ctpu delete --zone=your-zone
 ```
 $ ctpu status --zone=your-zone
 ```
-4. Run gsutil as shown, replacing your-bucket with the name of the Cloud Storage bucket you created for this tutorial:
+4. Run gcloud storage as shown, replacing your-bucket with the name of the Cloud Storage bucket you created for this tutorial:
 ```
-$ gsutil rm -r gs://your-bucket
+$ gcloud storage rm --recursive gs://your-bucket
 ```
 
 

official/projects/edgetpu/vision/serving/inference_visualization_tool.ipynb

@@ -57,7 +57,7 @@
         "  sandbox_path = web_path.split('/')[-1]\n",
         "  !rm -f {sandbox_path}\n",
         "  if web_path[:2] == \"gs\":\n",
-        "    !gsutil cp {web_path} {sandbox_path}\n",
+        "    !gcloud storage cp {web_path} {sandbox_path}\n",
         "  else:\n",
         "    !wget -v {web_path} --no-check-certificate\n",
         "  return sandbox_path\n"
@@ -122,7 +122,7 @@
       },
       "source": [
         "MODEL_HOME='gs://tf_model_garden/models/edgetpu/checkpoint_and_tflite/vision/segmentation-edgetpu/tflite/default_argmax'\n",
-        "!gsutil ls {MODEL_HOME}"
+        "!gcloud storage ls {MODEL_HOME}"
       ],
       "execution_count": null,
       "outputs": []

official/projects/longformer/README.md

@@ -21,7 +21,7 @@ pytorch longformer tokenized data to tf_records.
 Option 1. Use our saved checkpoint of `allenai/longformer-base-4096` stored in cloud storage
 
 ```bash
-gsutil cp -r gs://model-garden-ucsd-zihan/longformer-4096 .
+gcloud storage cp --recursive gs://model-garden-ucsd-zihan/longformer-4096 .
 ```
 Option 2. Create it directly
 

official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops.py

@@ -131,7 +131,7 @@ def upload_image_results_to_storage_bucket(
     try:
       commands = [
           f"rm -r {os.path.basename(input_directory)}",
-          f"gsutil -m cp -r {prediction_folder} {output_directory}",
+          f"gcloud storage cp --recursive {prediction_folder} {output_directory}",
           f"rm -r {prediction_folder}",
       ]
       subprocess.run(" && ".join(commands), shell=True, check=True)

official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/big_query_ops_test.py

@@ -198,7 +198,7 @@ def test_upload_image_results_to_storage_bucket_success(self):
 
     self.mock_subprocess_run.assert_called_once()
     args, _ = self.mock_subprocess_run.call_args
-    self.assertIn(f"gsutil -m cp -r {pred_dir} {output_dir}", args[0])
+    self.assertIn(f"gcloud storage cp --recursive {pred_dir} {output_dir}", args[0])
 
   def test_upload_image_results_to_storage_bucket_failure(self):
     input_dir = "/tmp/input"

official/projects/waste_identification_ml/Deploy/detr_cloud_deployment/client/utils.py

@@ -94,7 +94,7 @@ def setup_logger_and_directories(input_dir):
   """
 
   input_directory = (input_dir).rstrip('/\\')
-  command = f'gsutil -m cp -r {input_directory} .'
+  command = f'gcloud storage cp --recursive {input_directory} .'
   subprocess.run(command, shell=True, check=True)
   prediction_folder = os.path.basename(input_directory) + '_prediction'
   os.makedirs(prediction_folder, exist_ok=True)

official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/README.md

@@ -100,7 +100,7 @@ images to the source GCS bucket created by the script. The script output will
 provide the name of the bucket.
 
 ```bash
-gsutil cp your-local-image.jpg gs://<source-bucket-name>/
+gcloud storage cp your-local-image.jpg gs://<source-bucket-name>/
 ```
 ---
 

official/projects/waste_identification_ml/llm_applications/milk_pouch_detection/src/run_pipeline.sh

@@ -29,14 +29,14 @@ cd milk_pouch_project
 # NOTE: Adjust the grep pattern if other image types are expected.
 echo "=== DEBUGGING START ==="
 echo "DEBUG: gcs_path variable is: '${gcs_path}'"
-echo "DEBUG: Running 'gsutil ls \"${gcs_path}\"' to check accessibility:"
-gsutil ls "${gcs_path}" || echo "❌ gsutil ls failed"
-echo "DEBUG: Running 'gsutil ls -r \"${gcs_path}\" | head -n 10' to check content:"
-gsutil ls -r "${gcs_path}" | head -n 10 || echo "❌ gsutil recursive ls failed"
+echo "DEBUG: Running 'gcloud storage ls "${gcs_path}"' to check accessibility:"
+gcloud storage ls "${gcs_path}" || echo "❌ gsutil ls failed"
+echo "DEBUG: Running 'gcloud storage ls --recursive "${gcs_path}" | head -n 10' to check content:"
+gcloud storage ls --recursive "${gcs_path}" | head -n 10 || echo "❌ gsutil recursive ls failed"
 echo "=== DEBUGGING END ==="
 
 echo "🖨️ Listing image files from GCS bucket: $gcs_path"
-mapfile -t all_gcs_files < <(gsutil ls -r "${gcs_path}" | grep -iE '\.(png|jpg|jpeg)$' | grep -v "/predictions/" | grep -v "/processed/")
+mapfile -t all_gcs_files < <(gcloud storage ls --recursive "${gcs_path}" | grep -iE '\.(png|jpg|jpeg)$' | grep -v "/predictions/" | grep -v "/processed/")
 num_files=${#all_gcs_files[@]}
 
 if (( num_files == 0 )); then
@@ -77,7 +77,7 @@ for (( i=0; i<num_files; i+=batch_size )); do
 
   # Copy current batch files from GCS
   echo "🖨️ Copying $num_in_batch files from GCS to input_images/..."
-  gsutil -m cp "${current_batch[@]}" input_images/
+  gcloud storage cp "${current_batch[@]}" input_images/
 
   # Extract objects
   echo "🔎 Extracting objects from images..."
@@ -96,7 +96,7 @@ for (( i=0; i<num_files; i+=batch_size )); do
   # Move predictions back to GCS
   if [ -d "predictions" ] && [ -n "$(find predictions -type f -print -quit)" ]; then
     echo "🖨️ Moving predictions for this batch back to GCS bucket: $gcs_path"
-    gsutil -m cp -r predictions/ "$gcs_path"
+    gcloud storage cp --recursive predictions/ "$gcs_path"
   else
     echo "⚠️ No predictions generated for this batch."
   fi
@@ -109,7 +109,7 @@ for (( i=0; i<num_files; i+=batch_size )); do
 
   target_root="${clean_gcs_path}processed/"
 
-  # Group files by their destination directory to optimize gsutil calls
+  # Group files by their destination directory to optimize gcloud storage calls
   declare -a current_move_batch
   current_move_dir=""
 
@@ -128,7 +128,7 @@ for (( i=0; i<num_files; i+=batch_size )); do
     # If the destination directory changes, flush the current batch
     if [[ "$dest_dir" != "$current_move_dir" ]]; then
       if (( ${#current_move_batch[@]} > 0 )); then
-        gsutil -m mv "${current_move_batch[@]}" "$current_move_dir"
+        gcloud storage mv "${current_move_batch[@]}" "$current_move_dir"
         current_move_batch=()
       fi
       current_move_dir="$dest_dir"
@@ -138,7 +138,7 @@ for (( i=0; i<num_files; i+=batch_size )); do
 
   # Flush any remaining files
   if (( ${#current_move_batch[@]} > 0 )); then
-    gsutil -m mv "${current_move_batch[@]}" "$current_move_dir"
+    gcloud storage mv "${current_move_batch[@]}" "$current_move_dir"
   fi
 
   unset current_move_batch

research/object_detection/dockerfiles/tf2_ai_platform/Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y \
     python3-opencv \
     wget
 
-# Installs google cloud sdk, this is mostly for using gsutil to export model.
+# Installs google cloud sdk, this is mostly for using gcloud storage to export model.
 RUN wget -nv \
     https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz && \
     mkdir /root/tools && \
@@ -29,7 +29,7 @@ RUN wget -nv \
 
 # Path configuration
 ENV PATH $PATH:/root/tools/google-cloud-sdk/bin
-# Make sure gsutil will use the default service account
+# Make sure gcloud storage will use the default service account
 RUN echo '[GoogleCompute]\nservice_account = default' > /etc/boto.cfg
 
 WORKDIR /home/tensorflow

research/object_detection/g3doc/oid_inference_and_evaluation.md

@@ -58,7 +58,7 @@ access to the cloud bucket with the images. Then run:
 # From tensorflow/models/research/oid
 SPLIT=validation  # Set SPLIT to "test" to download the images in the test set
 mkdir raw_images_${SPLIT}
-gsutil -m rsync -r gs://open-images-dataset/$SPLIT raw_images_${SPLIT}
+gcloud storage rsync --recursive gs://open-images-dataset/$SPLIT raw_images_${SPLIT}
 ```
 
 Another option for downloading the images is to follow the URLs contained in the