blog

Author: TaoShuchang

.gitignore

@@ -145,7 +145,7 @@ tutorial/example_math_agent/math_agent_debug.yaml
 saved_experiments
 tests/temp
 vsdb.py
-appworld_pack_v2.tar*
+appworld_pack_v3.tar*
 saved_checkpoints
 data
 datasets
@@ -160,8 +160,11 @@ tutorial/example_deep_finance/scripts/*
 flash_attn-2.8.*.whl
 tutorial/example_deep_finance/prepare_data/*
 tutorial/example_deep_finance/judge/analytical_sufficiency/*
+<<<<<<< HEAD
 tutorial/example_deep_finance/output_report/*
 dataset_gsm8k/*
+=======
+>>>>>>> origin/main
 
 .dockerignore
 benchmark_datasets

ajet/utils/metric_helper/reward_metric_helper.py

@@ -83,10 +83,14 @@ def compute_reward_metrics(reward_stats_list: List[Dict[str, Any]], prefix: str
         openjudge_graders = [
             "presentation_quality",
             "grounding",
+<<<<<<< HEAD
             "planning",
             "audit",
             "traceability",
             "cgcv"
+=======
+            "planning"
+>>>>>>> origin/main
         ]
 
         for grader_name in openjudge_graders:

docs/en/example_app_world.md

@@ -24,10 +24,10 @@ export APPWORLD_PATH="${base_path}/pack_all_in_one"
 export APPWORLD_SCRIPT="bash EnvService/env_sandbox/appworld.sh"
 
 rm -rf "${APPWORLD_PATH}"
-rm -f ./appworld_pack_v2.tar.gz
+rm -f ./appworld_pack_v3.tar.gz
 
-wget -q "https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/astuner_archive/appworld_pack_v2.tar.gz" -O appworld_pack_v2.tar.gz
-tar -xzf ./appworld_pack_v2.tar.gz -C "${base_path}"
+wget -q "https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/astuner_archive/appworld_pack_v3.tar.gz" -O appworld_pack_v3.tar.gz
+tar -xzf ./appworld_pack_v3.tar.gz -C "${base_path}"
 ```
 
 Then export the environment variables (re-run in every new shell):

tests/bench/benchmark_appworld/execute_benchmark_appworld.py

@@ -75,25 +75,25 @@ def clear_system_processes(self):
 
     def install_appworld(self):
         # run:
-        # `rm -rf /tmp/pack_all_in_one & wget https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/astuner_archive/appworld_pack_v2.tar.gz  &&   tar   -xzf   ./appworld_pack_v2.tar.gz  -C /tmp`
+        # `rm -rf /tmp/pack_all_in_one & wget https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/astuner_archive/appworld_pack_v3.tar.gz  &&   tar   -xzf   ./appworld_pack_v3.tar.gz  -C /tmp`
         self.clear_system_processes()
         import shutil
 
         if os.path.exists("/tmp/pack_all_in_one"):
             shutil.rmtree("/tmp/pack_all_in_one")
-        if os.path.exists("./appworld_pack_v2.tar.gz"):
-            os.remove("./appworld_pack_v2.tar.gz")
+        if os.path.exists("./appworld_pack_v3.tar.gz"):
+            os.remove("./appworld_pack_v3.tar.gz")
         subprocess.run(
             [
                 "wget",
-                "https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/astuner_archive/appworld_pack_v2.tar.gz",
+                "https://dail-wlcb.oss-cn-wulanchabu.aliyuncs.com/astuner_archive/appworld_pack_v3.tar.gz",
             ]
         )
         subprocess.run(
             [
                 "tar",
                 "-xzf",
-                "./appworld_pack_v2.tar.gz",
+                "./appworld_pack_v3.tar.gz",
                 "-C",
                 "/tmp",
             ]

tests/bench/benchmark_frozenlake/benchmark_frozenlake.py

@@ -9,7 +9,7 @@
 class TestProbe(BenchmarkProbe):
     def __init__(self):
         # fmt: off
-        self.expected_train_time = 3600 * 24  # 24 hours budget for frozenlake easy benchmark
+        self.expected_train_time = 3600 * 48  # 48 hours budget for frozenlake easy benchmark
         self.begin_time = time.time()
         self.reward_array = []
         self.reward_expectation_avg_window = 20