update benchmark test robot

Author: binary-husky

tests/bench/README.md

@@ -14,13 +14,14 @@ Note: `tests/bench` source code is for test robot only, therefore `yaml` configu
 source .venv/bin/activate
 
 python -m pytest -s tests/bench/benchmark_math/execute_benchmark_math.py
+python -m pytest -s tests/bench/benchmark_appworld/execute_benchmark_appworld.py
+python -m pytest -s tests/bench/benchmark_countdown/execute_benchmark_countdown.py
 python -m pytest -s tests/bench/benchmark_learn2ask/execute_benchmark_learn2ask.py
 python -m pytest -s tests/bench/benchmark_frozenlake/execute_benchmark_frozenlake.py
-python -m pytest -s tests/bench/benchmark_countdown/execute_benchmark_countdown.py
-python -m pytest -s tests/bench/benchmark_appworld/execute_benchmark_appworld.py
 
-python -m pytest -s tests/bench/benchmark_countdown/execute_benchmark_countdown.py::TestBenchmarkCountdown::test_01_begin_verl
-python -m pytest -s tests/bench/benchmark_math/execute_benchmark_math.py::TestBenchmarkMath::test_01_begin_verl
-python -m pytest -s tests/bench/benchmark_appworld/execute_benchmark_appworld.py::TestBenchmarkAppworld::test_01_begin_verl
-python -m pytest -s tests/bench/benchmark_learn2ask/execute_benchmark_learn2ask.py::TestBenchmarkLearnToAsk::test_02_begin_verl
+VERL_PYTHON="./.venv/bin/python" python -m pytest -s tests/bench/benchmark_math/execute_benchmark_math.py::TestBenchmarkMath::test_01_begin_verl
+VERL_PYTHON="./.venv/bin/python" python -m pytest -s tests/bench/benchmark_appworld/execute_benchmark_appworld.py::TestBenchmarkAppworld::test_01_begin_verl
+VERL_PYTHON="./.venv/bin/python" python -m pytest -s tests/bench/benchmark_countdown/execute_benchmark_countdown.py::TestBenchmarkCountdown::test_01_begin_verl
+VERL_PYTHON="./.venv/bin/python" python -m pytest -s tests/bench/benchmark_learn2ask/execute_benchmark_learn2ask.py::TestBenchmarkLearnToAsk::test_01_begin_verl
+VERL_PYTHON="./.venv/bin/python" python -m pytest -s tests/bench/benchmark_frozenlake/execute_benchmark_frozenlake.py::TestBenchmarkFrozenLake::test_01_begin_verl
 ```

tests/bench/benchmark_appworld/execute_benchmark_appworld.py

@@ -8,6 +8,7 @@
 
 
 class TestBenchmarkAppworld(BenchmarkTestCase):
+
     def test_01_begin_verl(self):
         # get probe target, so as to get timeout settings
         BACKBONE = "verl"
@@ -16,7 +17,7 @@ def test_01_begin_verl(self):
         # tests/bench/benchmark_appworld/benchmark_appworld.py
         # tests/bench/benchmark_appworld/benchmark_appworld.yaml
         TARGET_NAME = f"benchmark_appworld_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".verl/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("VERL_PYTHON", ".verl/bin/python")
         multi_nodes = False
 
         self.execute_benchmark(
@@ -37,7 +38,7 @@ def test_02_begin_trinity(self):
         TEST_TARGET = "tests/bench/benchmark_appworld/benchmark_appworld_2nodes.yaml"
         PROBE_TARGET = "tests/bench/benchmark_appworld/benchmark_appworld.py->TestProbe"
         TARGET_NAME = f"benchmark_appworld_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".venv/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("TRINITY_PYTHON", ".venv/bin/python")
         multi_nodes = True
 
         self.execute_benchmark(

tests/bench/benchmark_countdown/execute_benchmark_countdown.py

@@ -1,15 +1,20 @@
+import os
 import unittest
 
 from tests.bench.benchmark_base import BenchmarkTestCase
 
 
+
+
 class TestBenchmarkCountdown(BenchmarkTestCase, unittest.TestCase):
+
     def test_01_begin_verl(self):
         BACKBONE = "verl"
         TEST_TARGET = "tests/bench/benchmark_countdown/benchmark_countdown.yaml"
         PROBE_TARGET = "tests/bench/benchmark_countdown/benchmark_countdown.py->TestProbe"
         TARGET_NAME = f"benchmark_countdown_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".verl/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("VERL_PYTHON", ".verl/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,
@@ -23,7 +28,8 @@ def test_02_begin_trinity(self):
         TEST_TARGET = "tests/bench/benchmark_countdown/benchmark_countdown.yaml"
         PROBE_TARGET = "tests/bench/benchmark_countdown/benchmark_countdown.py->TestProbe"
         TARGET_NAME = f"benchmark_countdown_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".venv/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("TRINITY_PYTHON", ".venv/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,

tests/bench/benchmark_frozenlake/execute_benchmark_frozenlake.py

@@ -1,15 +1,17 @@
-import unittest
+import os
 
 from tests.bench.benchmark_base import BenchmarkTestCase
 
 
 class TestBenchmarkFrozenLake(BenchmarkTestCase):
-    def test_01_begin_trinity(self):
-        BACKBONE = "trinity"
+
+    def test_01_begin_verl(self):
+        BACKBONE = "verl"
         TEST_TARGET = "tests/bench/benchmark_frozenlake/benchmark_frozenlake.yaml"
         PROBE_TARGET = "tests/bench/benchmark_frozenlake/benchmark_frozenlake.py->TestProbe"
         TARGET_NAME = f"benchmark_frozenlake_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".venv/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("VERL_PYTHON", ".verl/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,
@@ -18,12 +20,13 @@ def test_01_begin_trinity(self):
             python_executable=PYTHON_EXECUTABLE,
         )
 
-    def test_02_begin_verl(self):
-        BACKBONE = "verl"
+    def test_02_begin_trinity(self):
+        BACKBONE = "trinity"
         TEST_TARGET = "tests/bench/benchmark_frozenlake/benchmark_frozenlake.yaml"
         PROBE_TARGET = "tests/bench/benchmark_frozenlake/benchmark_frozenlake.py->TestProbe"
         TARGET_NAME = f"benchmark_frozenlake_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".verl/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("TRINITY_PYTHON", ".venv/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,

tests/bench/benchmark_learn2ask/execute_benchmark_learn2ask.py

@@ -1,17 +1,18 @@
-import unittest
+import os
 
 from tests.bench.benchmark_base import BenchmarkTestCase
 
 
 class TestBenchmarkLearnToAsk(BenchmarkTestCase):
-    def test_01_begin_trinity(self):
+
+    def test_01_begin_verl(self):
         # get probe target, so as to get timeout settings
-        BACKBONE = "trinity"
+        BACKBONE = "verl"
         TEST_TARGET = "tests/bench/benchmark_learn2ask/benchmark_learn2ask.yaml"
         PROBE_TARGET = "tests/bench/benchmark_learn2ask/benchmark_learn2ask.py->TestProbe"
         TARGET_NAME = f"benchmark_learn2ask_{BACKBONE}"
-        # PYTHON_EXECUTABLE = "python"
-        PYTHON_EXECUTABLE = ".venv/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("VERL_PYTHON", ".verl/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,
@@ -20,14 +21,14 @@ def test_01_begin_trinity(self):
             python_executable=PYTHON_EXECUTABLE,
         )
 
-    def test_02_begin_verl(self):
+    def test_02_begin_trinity(self):
         # get probe target, so as to get timeout settings
-        BACKBONE = "verl"
+        BACKBONE = "trinity"
         TEST_TARGET = "tests/bench/benchmark_learn2ask/benchmark_learn2ask.yaml"
         PROBE_TARGET = "tests/bench/benchmark_learn2ask/benchmark_learn2ask.py->TestProbe"
         TARGET_NAME = f"benchmark_learn2ask_{BACKBONE}"
-        # PYTHON_EXECUTABLE = "python"
-        PYTHON_EXECUTABLE = ".verl/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("TRINITY_PYTHON", ".venv/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,

tests/bench/benchmark_math/execute_benchmark_math.py

@@ -1,14 +1,17 @@
+import os
 from tests.bench.benchmark_base import BenchmarkTestCase
 
 
 class TestBenchmarkMath(BenchmarkTestCase):
-    def test_02_begin_trinity(self):
+
+    def test_01_begin_verl(self):
         # get probe target, so as to get timeout settings
-        BACKBONE = "trinity"
+        BACKBONE = "verl"
         TEST_TARGET = "tests/bench/benchmark_math/benchmark_math.yaml"
         PROBE_TARGET = "tests/bench/benchmark_math/benchmark_math.py->TestProbe"
         TARGET_NAME = f"benchmark_math_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".venv/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("VERL_PYTHON", ".verl/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,
@@ -17,13 +20,14 @@ def test_02_begin_trinity(self):
             python_executable=PYTHON_EXECUTABLE,
         )
 
-    def test_01_begin_verl(self):
+    def test_02_begin_trinity(self):
         # get probe target, so as to get timeout settings
-        BACKBONE = "verl"
+        BACKBONE = "trinity"
         TEST_TARGET = "tests/bench/benchmark_math/benchmark_math.yaml"
         PROBE_TARGET = "tests/bench/benchmark_math/benchmark_math.py->TestProbe"
         TARGET_NAME = f"benchmark_math_{BACKBONE}"
-        PYTHON_EXECUTABLE = ".verl/bin/python"
+        PYTHON_EXECUTABLE = os.environ.get("TRINITY_PYTHON", ".venv/bin/python")
+
         self.execute_benchmark(
             backbone=BACKBONE,
             test_target=TEST_TARGET,