Add checkpoint tests

Author: MDUYN

tests/app/backtesting/test_run_backtest.py

@@ -29,20 +29,11 @@ class Test(TestCase):
     Collection of tests for backtest report operations
     """
     def setUp(self) -> None:
-        self.resource_directory = os.path.abspath(
-            os.path.join(
-                os.path.join(
-                    os.path.join(
-                        os.path.join(
-                            os.path.realpath(__file__),
-                            os.pardir
-                        ),
-                        os.pardir
-                    ),
-                    os.pardir
-                ),
-                "resources"
-            )
+        # RESOURCE_DIRECTORY should always point to the parent directory/resources
+        # Resource directory should point to /tests/resources
+        # Resource directory is two levels up from the current file
+        self.resource_directory = os.path.join(
+            os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'resources'
         )
 
     def tearDown(self) -> None:

tests/app/reporting/test_backtest_report.py

@@ -82,22 +82,23 @@ def test_save_without_algorithm(self):
         # Check if the report was saved correctly
         self.assertTrue(os.path.exists(output_path))
 
-        # Check if the strategy directory was created
-        strategy_dir = os.path.join(output_path, "strategies")
-        self.assertTrue(os.path.exists(strategy_dir))
-
-        # Check if the strategy.py file exists within the strategy directory
-        strategy_file_path = os.path.join(strategy_dir, "strategy_one.py")
-        self.assertTrue(os.path.exists(strategy_file_path))
-
-        # Check if the metrics JSON file exists
-        metrics_json_path = os.path.join(output_path, "metrics.json")
-        self.assertTrue(os.path.exists(metrics_json_path))
+        # Check if the runs directory exists
+        runs_dir = os.path.join(output_path, "runs")
+        self.assertTrue(os.path.exists(runs_dir))
 
+        # Check if the backtest run directory exists
+        backtest_run_dir = os.path.join(
+            runs_dir, "backtest_EUR_20230807_20231201"
+        )
+        self.assertTrue(os.path.exists(backtest_run_dir))
 
         # Check if the results were saved correctly
-        self.assertTrue(os.path.exists(os.path.join(output_path, "results.json")))
-        self.assertTrue(os.path.exists(os.path.join(output_path, "strategies")))
+        self.assertTrue(
+            os.path.exists(os.path.join(backtest_run_dir, "run.json"))
+        )
+        self.assertTrue(
+            os.path.exists(os.path.join(backtest_run_dir, "metrics.json"))
+        )
 
     def test_save_with_strategies_directory(self):
         """
@@ -156,10 +157,6 @@ def test_save_with_strategies_directory(self):
                 max_daily_drawdown=0.05
             )
         )
-        data_files = [
-            "tests/resources/market_data_sources_for_testing/OHLCV_BTC-EUR_BINANCE_2h_2023-08-07-07-59_2023-12-02-00-00.csv",
-            "tests/resources/market_data_sources_for_testing/OHLCV_BTC-EUR_BINANCE_15m_2023-12-14-22-00_2023-12-25-00-00.csv",
-        ]
 
         backtest = Backtest(
             algorithm_id="alg-025",
@@ -169,21 +166,26 @@ def test_save_with_strategies_directory(self):
         output_path = os.path.join(self.resource_dir, "backtest_report")
         backtest.save(output_path)
 
+        print(output_path)
+
         # Check if the report was saved correctly
         self.assertTrue(os.path.exists(output_path))
 
-        # Check if the strategy directory was created
-        strategy_dir = os.path.join(output_path, "strategies")
-        self.assertTrue(os.path.exists(strategy_dir))
+        # Check if the runs directory exists
+        runs_dir = os.path.join(output_path, "runs")
+        self.assertTrue(os.path.exists(runs_dir))
 
-        # Check if the strategy.py file exists within the strategy directory
-        strategy_file_path = os.path.join(strategy_dir, "strategy_one.py")
-        self.assertTrue(os.path.exists(strategy_file_path))
-
-        # Check if the metrics JSON file exists
-        metrics_json_path = os.path.join(output_path, "metrics.json")
-        self.assertTrue(os.path.exists(metrics_json_path))
+        # Check if the backtest run directory exists
+        backtest_run_dir = os.path.join(
+            runs_dir, "backtest_EUR_20230807_20231201"
+        )
+        self.assertTrue(os.path.exists(backtest_run_dir))
 
         # Check if the results were saved correctly
-        self.assertTrue(os.path.exists(os.path.join(output_path, "results.json")))
-        self.assertTrue(os.path.exists(os.path.join(output_path, "strategies")))
+        self.assertTrue(
+            os.path.exists(os.path.join(backtest_run_dir, "run.json"))
+        )
+        self.assertTrue(
+            os.path.exists(os.path.join(backtest_run_dir, "metrics.json"))
+        )
+

tests/scenarios/vectorized_backtests/test_use_backtest_storage_directory.py

@@ -249,8 +249,6 @@ def test_run_with_backtest_storage_directory(self):
             dict(zip(param_options.keys(), values))
             for values in product(*param_options.values())
         ]
-        print(
-            f"Total parameter combinations to evaluate: {len(param_variations)}")
 
         # RESOURCE_DIRECTORY should always point to the parent directory/resources
         # Resource directory should point to /tests/resources

tests/scenarios/vectorized_backtests/test_use_checkpoints.py

@@ -241,7 +241,8 @@ def test_run_with_checkpoints_multiple_backtest_ranges(self):
         # Resource directory should point to /tests/resources
         # Resource directory is two levels up from the current file
         resource_directory = os.path.join(
-            os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'resources'
+            os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
+            'resources'
         )
         config = {RESOURCE_DIRECTORY: resource_directory}
         app = create_app(name="GoldenCrossStrategy", config=config)

tests/scenarios/vectorized_backtests/test_with_show_progress.py

@@ -340,9 +340,6 @@ def test_run_with_show_output(self):
         # Get the captured output as a string
         output = captured_output.getvalue()
 
-        # Print the captured output (optional - for debugging)
-        print(output)
-
         # Verify that checkpointing message appears
         self.assertTrue(
             "Using checkpoints to skip completed backtests" in output or
@@ -368,9 +365,3 @@ def test_run_with_show_output(self):
         self.assertTrue(
             "Applying final filter function" in output
         )
-
-        # Verify that filtered-out backtests are marked in metadata
-        # (not deleted from storage)
-        if "Marked backtest" in output and "as filtered out" in output:
-            print("✓ Filtered-out backtests were marked in metadata "
-                  "(preserved in storage)")