Merge pull request #378 from coding-kitties/fix/376-refactor-tests-scenarios

fix: refactor tests/scenarios for deterministic offline execution (#376)

Author: MDUYN

.github/workflows/test.yml

@@ -102,6 +102,11 @@ jobs:
         run: |
           source $VENV
           coverage run -m unittest discover -s tests
+      - name: Generate coverage report
+        if: always()
+        continue-on-error: true
+        run: |
+          source $VENV
           coverage xml
       #----------------------------------------------
       #             upload coverage stats

tests/__init__.py

@@ -0,0 +1 @@
+

tests/app/reporting/test_backtest_report.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 from datetime import datetime, timezone
 from unittest import TestCase
 
@@ -23,6 +24,11 @@ def setUp(self):
                 "resources"
             )
         )
+        self.output_path = os.path.join(self.resource_dir, "backtest_report")
+
+    def tearDown(self):
+        if os.path.exists(self.output_path):
+            shutil.rmtree(self.output_path)
 
     def test_save_without_algorithm(self):
         """
@@ -47,8 +53,8 @@ def test_save_without_algorithm(self):
             )
         ]
         backtest_date_range = BacktestDateRange(
-            start_date="2023-08-07 07:00:00",
-            end_date="2023-12-02 00:00:00",
+            start_date=datetime(2023, 8, 7, 7, 0, tzinfo=timezone.utc),
+            end_date=datetime(2023, 12, 2, 0, 0, tzinfo=timezone.utc),
             name="Test Backtest Date Range"
         )
         run = BacktestRun(
@@ -64,36 +70,32 @@ def test_save_without_algorithm(self):
             initial_unallocated=1000,
             created_at=datetime.now(tz=timezone.utc)
         )
-        data_files = [
-            os.path.join("tests", "resources", "market_data_sources_for_testing", "OHLCV_BTC-EUR_BINANCE_2h_2023-08-07-07-59_2023-12-02-00-00.csv"),
-            os.path.join("tests", "resources", "market_data_sources_for_testing", "OHLCV_BTC-EUR_BINANCE_15m_2023-12-14-22-00_2023-12-25-00-00.csv"),
-        ]
 
         backtest = Backtest(
             algorithm_id="alg-025",
             backtest_runs=[run],
         )
-        output_path = os.path.join(self.resource_dir, "backtest_report")
-        backtest.save(output_path)
+        backtest.save(self.output_path)
 
         # Check if the report was saved correctly
-        self.assertTrue(os.path.exists(output_path))
+        self.assertTrue(os.path.exists(self.output_path))
 
         # Check if the runs directory exists
-        runs_dir = os.path.join(output_path, "runs")
+        runs_dir = os.path.join(self.output_path, "runs")
         self.assertTrue(os.path.exists(runs_dir))
 
         # Check if the backtest run directory exists
         backtest_run_dir = os.path.join(
-            runs_dir, "backtest_EUR_20230807_20231201"
+            runs_dir, "backtest_EUR_20230807_20231202"
         )
         self.assertTrue(os.path.exists(backtest_run_dir))
 
         # Check if the results were saved correctly
         self.assertTrue(
             os.path.exists(os.path.join(backtest_run_dir, "run.json"))
         )
-        self.assertTrue(
+        # No backtest_metrics provided, so metrics.json should not exist
+        self.assertFalse(
             os.path.exists(os.path.join(backtest_run_dir, "metrics.json"))
         )
 
@@ -118,8 +120,8 @@ def test_save_with_strategies_directory(self):
             )
         ]
         backtest_date_range = BacktestDateRange(
-            start_date="2023-08-07 07:00:00",
-            end_date="2023-12-02 00:00:00",
+            start_date=datetime(2023, 8, 7, 7, 0, tzinfo=timezone.utc),
+            end_date=datetime(2023, 12, 2, 0, 0, tzinfo=timezone.utc),
             name="Test Backtest Date Range"
         )
         results = BacktestRun(
@@ -160,21 +162,18 @@ def test_save_with_strategies_directory(self):
             backtest_runs=[results],
             risk_free_rate=0.0
         )
-        output_path = os.path.join(self.resource_dir, "backtest_report")
-        backtest.save(output_path)
-
-        print(output_path)
+        backtest.save(self.output_path)
 
         # Check if the report was saved correctly
-        self.assertTrue(os.path.exists(output_path))
+        self.assertTrue(os.path.exists(self.output_path))
 
         # Check if the runs directory exists
-        runs_dir = os.path.join(output_path, "runs")
+        runs_dir = os.path.join(self.output_path, "runs")
         self.assertTrue(os.path.exists(runs_dir))
 
         # Check if the backtest run directory exists
         backtest_run_dir = os.path.join(
-            runs_dir, "backtest_EUR_20230807_20231201"
+            runs_dir, "backtest_EUR_20230807_20231202"
         )
         self.assertTrue(os.path.exists(backtest_run_dir))
 

tests/domain/backtests/test_backtest_save.py

@@ -46,8 +46,8 @@ def _create_snapshots(self):
 
     def _create_date_range(self):
         return BacktestDateRange(
-            start_date="2023-08-07 07:00:00",
-            end_date="2023-12-02 00:00:00",
+            start_date=datetime(2023, 8, 7, 7, 0, tzinfo=timezone.utc),
+            end_date=datetime(2023, 12, 2, 0, 0, tzinfo=timezone.utc),
             name="Test Backtest Date Range"
         )
 
@@ -80,7 +80,7 @@ def test_save_without_metrics(self):
         self.assertTrue(os.path.exists(runs_dir))
 
         backtest_run_dir = os.path.join(
-            runs_dir, "backtest_EUR_20230807_20231201"
+            runs_dir, "backtest_EUR_20230807_20231202"
         )
         self.assertTrue(os.path.exists(backtest_run_dir))
         self.assertTrue(
@@ -144,7 +144,7 @@ def test_save_with_precomputed_metrics(self):
         self.assertTrue(os.path.exists(runs_dir))
 
         backtest_run_dir = os.path.join(
-            runs_dir, "backtest_EUR_20230807_20231201"
+            runs_dir, "backtest_EUR_20230807_20231202"
         )
         self.assertTrue(os.path.exists(backtest_run_dir))
         self.assertTrue(

tests/domain/utils/test_polars.py

@@ -32,8 +32,11 @@ def test_convert_pandas_to_polars(self):
         column_names = polars_df_converted.columns.tolist()
         self.assertEqual(set(column_names), {'Close'})
 
-        # Check if the index is a datetime object
-        self.assertEqual(polars_df_converted.index.dtype, "datetime64[us]")
+        # Check if the index is a datetime object (accept both ns and us
+        # resolution since it depends on pandas version)
+        self.assertTrue(
+            str(polars_df_converted.index.dtype).startswith("datetime64")
+        )
         self.assertEqual(
             polars_df_converted.index[0], Timestamp('2021-01-01 00:00:00')
         )

tests/infrastructure/data_providers/test_csv_ohlcv_data_provider.py

@@ -34,7 +34,7 @@ def test_right_columns(self):
         file_name = "OHLCV_BTC-EUR_BINANCE" \
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         data_provider = CSVOHLCVDataProvider(
-            storage_path=f"{self.resource_dir}/market_data_sources/"
+            storage_path=f"{self.resource_dir}/test_data/ohlcv/"
                           f"{file_name}",
             window_size=10,
             market="binance",
@@ -65,7 +65,7 @@ def test_throw_exception_when_missing_column_names_columns(self):
             CSVOHLCVDataProvider(
                 storage_path=os.path.join(
                     self.resource_dir,
-                    "market_data_sources_for_testing",
+                    "test_data", "ohlcv",
                     file_name
                 ),
                 window_size=10,
@@ -85,7 +85,7 @@ def test_has_data(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             window_size=10,
             market="binance",
@@ -123,7 +123,7 @@ def test_has_data_backtest_mode(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             window_size=10,
             market="binance",
@@ -167,7 +167,7 @@ def test_get_data_start_date(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         csv_ohlcv_market_data_source = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             window_size=200,
             market="binance",
@@ -210,7 +210,7 @@ def test_get_data_end_date(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         csv_ohlcv_market_data_source = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             window_size=200,
             market="binance",
@@ -239,7 +239,7 @@ def test_get_identifier(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             data_provider_identifier="test",
             window_size=10,
@@ -254,7 +254,7 @@ def test_get_market(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             market="test",
             symbol="BTC/EUR",
@@ -268,7 +268,7 @@ def test_get_symbol(self):
                     "_2h_2023-08-07-07-59_2023-12-02-00-00.csv"
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             symbol="BTC/EUR",
             window_size=10,
@@ -289,7 +289,7 @@ def test_prepare_backtest_data(self):
         )
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             data_provider_identifier="test",
             market="binance",
@@ -345,7 +345,7 @@ def test_get_backtest_data(self):
         )
         data_provider = CSVOHLCVDataProvider(
             storage_path=os.path.join(
-                self.resource_dir, "market_data_sources", file_name
+                self.resource_dir, "test_data", "ohlcv", file_name
             ),
             data_provider_identifier="test",
             market="binance",