Merge pull request #3519 from CIeNET-International:maxtext/user/dora/add_tools

PiperOrigin-RevId: 892695602

Author: Google-ML-Automation

tools/setup/setup_gcsfuse.sh src/dependencies/scripts/setup_gcsfuse.shtools/setup/setup_gcsfuse.sh renamed to src/dependencies/scripts/setup_gcsfuse.sh

@@ -34,7 +34,7 @@ then
     EVAL_METRICS=grain_checkpoint_save_restore
     echo "Using c4-array_record dataset type"
     echo "Mounting $DATASET_PATH to /tmp/gcsfuse/"
-    bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
+    bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
     DATASET_PATH=/tmp/gcsfuse/
     CMD_DATA=" dataset_type=c4-array_record dataset_name=array-record/c4/en/3.0.1 eval_dataset_name=array-record/c4/en/3.0.1"
 fi

tests/end_to_end/gpu/a3/test_convergence_125m_params.sh

@@ -34,7 +34,7 @@ then
     EVAL_METRICS=grain_checkpoint_save_restore
     echo "Using c4-array_record dataset type"
     echo "Mounting $DATASET_PATH to /tmp/gcsfuse/"
-    bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
+    bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
     DATASET_PATH=/tmp/gcsfuse/
     CMD_DATA=" dataset_type=c4-array_record dataset_name=array-record/c4/en/3.0.1 eval_dataset_name=array-record/c4/en/3.0.1"
 fi

tests/end_to_end/gpu/a3/test_convergence_1b_params.sh

@@ -16,7 +16,7 @@ fi
 model_params=" base_emb_dim=384 base_num_query_heads=8 base_num_kv_heads=8 base_mlp_dim=192 base_num_decoder_layers=8 head_dim=128"
 
 echo "Mounting $DATASET_PATH to /tmp/gcsfuse/"
-bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
+bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
 
 echo "Run_1: Starting the first run using the grain input pipeline"
 

tests/end_to_end/test_checkpoint_compatibility.sh

@@ -30,7 +30,7 @@ then
     eval_metrics=grain_checkpoint_save_restore
     echo "Using grain dataset type"
     echo "Mounting $DATASET_PATH to /tmp/gcsfuse/"
-    bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
+    bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
     DATASET_PATH=/tmp/gcsfuse/
     CMD_DATA=" grain_worker_count=0 dataset_type=grain grain_train_files=/tmp/gcsfuse/array-record/c4/en/3.0.1/c4-train.array_record*"
 fi

tests/end_to_end/test_checkpointing.sh

@@ -38,7 +38,7 @@ then
     EVAL_METRICS=grain_checkpoint_save_restore
     echo "dataset_type is grain"
     echo "Mounting $DATASET_PATH to /tmp/gcsfuse/"
-    bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
+    bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=$DATASET_PATH MOUNT_PATH=/tmp/gcsfuse/
     DATASET_PATH=/tmp/gcsfuse/
     CMD_DATA=" grain_worker_count=2 \
             grain_train_files=/tmp/gcsfuse/array-record/c4/en/3.0.1/c4-train.array_record* \

tests/end_to_end/tpu/test_convergence_1b_params.sh

@@ -21,7 +21,7 @@
 The tests confirm restoration by checking the starting step of the resumed runs.
 
 Note: Make sure to run
-  `bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=gs://maxtext-dataset MOUNT_PATH=/tmp/gcsfuse/`
+  `bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=gs://maxtext-dataset MOUNT_PATH=/tmp/gcsfuse/`
 before running tests locally.
 """
 

tests/integration/checkpoint_compatibility_test.py

@@ -20,7 +20,7 @@
 continue from that saved checkpoint.
 
 Note: Make sure to run
-  `bash tools/setup/setup_gcsfuse.sh DATASET_GCS_BUCKET=gs://maxtext-dataset MOUNT_PATH=/tmp/gcsfuse/`
+  `bash src/dependencies/scripts/setup_gcsfuse.sh DATASET_GCS_BUCKET=gs://maxtext-dataset MOUNT_PATH=/tmp/gcsfuse/`
 before running tests locally.
 """