chore: Migrate gsutil usage to gcloud storage (#192)

Author: bhandarivijay-png

ai-infrastructure/tpu-training-on-gke/environment/cloudbuild.destroy.yaml

@@ -20,10 +20,10 @@ steps:
     args:
       - -c
       - |
-        gsutil cp gs://${_AUTOMATION_BUCKET}/providers/* .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/providers/* .
         sed -i "s/<FOLDER-NAME>/${_ENV_NAME}\/tfstate\/gke_config/g" backend.tf
-        gsutil cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
-        gsutil cp gs://${_AUTOMATION_BUCKET}/${_ENV_NAME}/tfvars/1-base-infra.auto.tfvars.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/${_ENV_NAME}/tfvars/1-base-infra.auto.tfvars.json .
         echo '{"env_name":"${_ENV_NAME}"}' > env-name.auto.tfvars.json
         
   - id: 'Destroy GKE config - tf init'
@@ -47,9 +47,9 @@ steps:
     args:
       - -c
       - |
-        gsutil cp gs://${_AUTOMATION_BUCKET}/providers/* .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/providers/* .
         sed -i "s/<FOLDER-NAME>/${_ENV_NAME}\/tfstate\/base_infra/g" backend.tf
-        gsutil cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
         echo '{"env_name":"${_ENV_NAME}"}' > env-name.auto.tfvars.json
 
   - id: 'Destroy base infra - tf init'

ai-infrastructure/tpu-training-on-gke/environment/cloudbuild.provision.yaml

@@ -20,8 +20,8 @@ steps:
     args:
       - -c
       - |
-        gsutil cp gs://${_AUTOMATION_BUCKET}/providers/* .
-        gsutil cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/providers/* .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
         sed -i "s/<FOLDER-NAME>/${_ENV_NAME}\/tfstate\/base_infra/g" backend.tf
         echo '{"env_name":"${_ENV_NAME}"}' > env-name.auto.tfvars.json
         
@@ -49,7 +49,7 @@ steps:
         ## This is a mitigation to address issues with using Terraform to install CRDs.
         wget -O jq https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-amd64
         chmod 755 jq
-        gsutil cp gs://${_AUTOMATION_BUCKET}/${_ENV_NAME}/settings/1-base-infra-settings.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/${_ENV_NAME}/settings/1-base-infra-settings.json .
         CLUSTER_NAME=$(cat 1-base-infra-settings.json | ./jq -r '.cluster_name') 
         CLUSTER_LOCATION=$(cat 1-base-infra-settings.json | ./jq -r '.cluster_location') 
         gcloud config set auth/impersonate_service_account $_AUTOMATION_ACCOUNT
@@ -58,9 +58,9 @@ steps:
         kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/$_KUEUE_API_VERSION/manifests.yaml"
 
         # Prepare tfvars and providers for the Terraform gke-config stage
-        gsutil cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
-        gsutil cp gs://${_AUTOMATION_BUCKET}/providers/* .
-        gsutil cp gs://${_AUTOMATION_BUCKET}/${_ENV_NAME}/tfvars/1-base-infra.auto.tfvars.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/tfvars/0-bootstrap.auto.tfvars.json .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/providers/* .
+        gcloud storage cp gs://${_AUTOMATION_BUCKET}/${_ENV_NAME}/tfvars/1-base-infra.auto.tfvars.json .
         sed -i "s/<FOLDER-NAME>/${_ENV_NAME}\/tfstate\/gke_config/g" backend.tf
         echo '{"env_name":"${_ENV_NAME}"}' > env-name.auto.tfvars.json
 

genai-on-vertex-ai/agents/reasoning_engine/crewai/CrewAI_on_VertexAI_Reasoning_Engine.ipynb

@@ -263,7 +263,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "!set -x && gsutil mb -p $PROJECT_ID -l $LOCATION $STAGING_BUCKET"
+    "!set -x && gcloud storage buckets create --project $PROJECT_ID --location $LOCATION $STAGING_BUCKET"
    ]
   },
   {

genai-on-vertex-ai/gemini/prompting_recipes/spatial_reasoning/spatial_reasoning_app_for_gemini2.ipynb

@@ -461,7 +461,7 @@
     "    if not os.path.exists(local_path):\n",
     "        print(f'Local path to image file does not exist: {local_path}')\n",
     "        print(f'Downloading sample image from GCS...')\n",
-    "        !gsutil cp \"{gcs_uri}\" \"{local_path}\"\n",
+    "        !gcloud storage cp \"{gcs_uri}\" \"{local_path}\"\n",
     "    else:\n",
     "        print(f'Image already exists at: {local_path}')\n"
    ]

genai-on-vertex-ai/retrieval_augmented_generation/diy_rag_with_vertexai_apis/build_grounded_rag_app_with_vertex.ipynb

@@ -2877,8 +2877,7 @@
    "outputs": [],
    "source": [
     "if DELETE_BUCKET:\n",
-    "    ! gsutil -m rm -r $STAGING_BUCKET_URI"
-   ]
+    "    ! gcloud storage rm --recursive $STAGING_BUCKET_URI"   ]
   },
   {
    "cell_type": "markdown",