From 17ac341b36b5b476d67c2d422091b26a3498dfa0 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 16:20:46 +0200 Subject: [PATCH 1/7] fix: resolve SQLite database file locking on Windows for unittests - Add teardown_sqlalchemy() to properly dispose engine and release file locks - Fix clear_db() to dispose its engine in a finally block - Update all test teardowns to use teardown_sqlalchemy() - Add ignore_errors=True to shutil.rmtree calls as safety net - Add windows-latest to CI test matrix - Use poetry run instead of source $VENV for cross-platform CI Closes #291 --- .github/workflows/test.yml | 8 +++--- .../infrastructure/database/__init__.py | 3 ++- .../infrastructure/database/sql_alchemy.py | 27 ++++++++++++++----- tests/app/algorithm/test_run_strategy.py | 3 +++ .../app/algorithm/test_trade_price_update.py | 3 +++ tests/app/backtesting/test_run_backtest.py | 7 +++-- tests/app/test_backtesting.py | 6 +++-- tests/app/test_eventloop.py | 4 +-- tests/app/test_start.py | 3 +++ .../test_algorithm_backtest/algorithm_id.json | 2 +- .../backtest_EUR_20231201_20231202/run.json | 2 +- .../backtest_EUR_20231202_20231203/run.json | 2 +- tests/resources/test_base.py | 10 +++---- .../test_metadata_preservation.py | 6 ++++- .../test_run_vector_backtests.py | 2 +- .../test_use_backtest_storage_directory.py | 14 +++++----- .../test_with_window_filter_function.py | 2 +- 17 files changed, 67 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1cb29e00..386cba70 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: strategy: fail-fast: true matrix: - os: [ "ubuntu-latest", "macos-latest" ] + os: [ "ubuntu-latest", "macos-latest", "windows-latest" ] python-version: [ "3.10", "3.11" ] defaults: run: @@ -101,14 +101,12 @@ jobs: #---------------------------------------------- - name: Run tests run: | - source $VENV - coverage run -m unittest discover -s tests + poetry run coverage run -m unittest discover -s tests - name: Generate coverage report if: always() continue-on-error: true run: | - source $VENV - coverage xml + poetry run coverage xml #---------------------------------------------- # upload coverage stats #---------------------------------------------- diff --git a/investing_algorithm_framework/infrastructure/database/__init__.py b/investing_algorithm_framework/infrastructure/database/__init__.py index 7e6a29b0..dd8552d1 100644 --- a/investing_algorithm_framework/infrastructure/database/__init__.py +++ b/investing_algorithm_framework/infrastructure/database/__init__.py @@ -1,5 +1,5 @@ from .sql_alchemy import Session, setup_sqlalchemy, SQLBaseModel, \ - create_all_tables, clear_db, SqliteDecimal + create_all_tables, clear_db, teardown_sqlalchemy, SqliteDecimal __all__ = [ "Session", @@ -7,5 +7,6 @@ "SQLBaseModel", "create_all_tables", "clear_db", + "teardown_sqlalchemy", "SqliteDecimal" ] diff --git a/investing_algorithm_framework/infrastructure/database/sql_alchemy.py b/investing_algorithm_framework/infrastructure/database/sql_alchemy.py index ade72c2c..f8823e8f 100644 --- a/investing_algorithm_framework/infrastructure/database/sql_alchemy.py +++ b/investing_algorithm_framework/infrastructure/database/sql_alchemy.py @@ -3,7 +3,7 @@ from sqlalchemy import create_engine, StaticPool, String from sqlalchemy import inspect -from sqlalchemy.orm import DeclarativeBase, sessionmaker +from sqlalchemy.orm import DeclarativeBase, sessionmaker, close_all_sessions from sqlalchemy import TypeDecorator from investing_algorithm_framework.domain import SQLALCHEMY_DATABASE_URI, \ @@ -83,6 +83,21 @@ def create_all_tables(): SQLBaseModel.metadata.create_all(bind=Session().bind) +def teardown_sqlalchemy(): + """ + Dispose the engine and close all sessions to release file locks. + This is essential on Windows where file locks are mandatory and + prevent deletion of SQLite database files while connections are open. + """ + close_all_sessions() + bind = Session.kw.get("bind") + + if bind is not None: + bind.dispose() + + Session.configure(bind=None) + + from sqlalchemy import event from sqlalchemy.orm import mapper from datetime import timezone @@ -98,6 +113,7 @@ def clear_db(db_uri): Returns: None """ + engine = None # Drop all tables before deleting file try: engine = create_engine(db_uri) @@ -107,12 +123,9 @@ def clear_db(db_uri): SQLBaseModel.metadata.drop_all(bind=engine) except Exception as e: logger.error(f"Error dropping tables: {e}") - - # # Clear mappers (if using classical mappings) - # try: - # clear_mappers() - # except Exception: - # pass # ignore if not needed + finally: + if engine is not None: + engine.dispose() @event.listens_for(mapper, "load") diff --git a/tests/app/algorithm/test_run_strategy.py b/tests/app/algorithm/test_run_strategy.py index 694d3aa6..37c82951 100644 --- a/tests/app/algorithm/test_run_strategy.py +++ b/tests/app/algorithm/test_run_strategy.py @@ -8,6 +8,8 @@ from investing_algorithm_framework import create_app, TradingStrategy, \ TimeUnit, PortfolioConfiguration, RESOURCE_DIRECTORY, Algorithm, \ MarketCredential +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy from tests.resources import random_string, OrderExecutorTest, \ PortfolioProviderTest @@ -73,6 +75,7 @@ def setUp(self) -> None: def tearDown(self) -> None: super().tearDown() + teardown_sqlalchemy() database_dir = os.path.join(self.resource_dir, "databases") if os.path.exists(database_dir): shutil.rmtree(database_dir, ignore_errors=True) diff --git a/tests/app/algorithm/test_trade_price_update.py b/tests/app/algorithm/test_trade_price_update.py index 5cbebddc..c3f03a8e 100644 --- a/tests/app/algorithm/test_trade_price_update.py +++ b/tests/app/algorithm/test_trade_price_update.py @@ -8,6 +8,8 @@ TimeUnit, PortfolioConfiguration, RESOURCE_DIRECTORY, \ MarketCredential, DataSource, INDEX_DATETIME, DataType, \ CSVOHLCVDataProvider, BacktestDateRange +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy from tests.resources import random_string, \ PortfolioProviderTest, OrderExecutorTest @@ -55,6 +57,7 @@ def setUp(self) -> None: def tearDown(self) -> None: super().tearDown() + teardown_sqlalchemy() database_dir = os.path.join(self.resource_dir, "databases") if os.path.exists(database_dir): shutil.rmtree(database_dir, ignore_errors=True) diff --git a/tests/app/backtesting/test_run_backtest.py b/tests/app/backtesting/test_run_backtest.py index 584c25f9..80741e16 100644 --- a/tests/app/backtesting/test_run_backtest.py +++ b/tests/app/backtesting/test_run_backtest.py @@ -8,6 +8,8 @@ from investing_algorithm_framework import create_app, RESOURCE_DIRECTORY, \ TradingStrategy, PortfolioConfiguration, TimeUnit, Algorithm, \ BacktestDateRange +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy class TestStrategy(TradingStrategy): @@ -47,12 +49,13 @@ def setUp(self) -> None: def tearDown(self) -> None: super().tearDown() + teardown_sqlalchemy() if os.path.exists(self.backtest_databases_directory): - shutil.rmtree(self.backtest_databases_directory) + shutil.rmtree(self.backtest_databases_directory, ignore_errors=True) if os.path.exists(self.backtest_report_directory): - shutil.rmtree(self.backtest_report_directory) + shutil.rmtree(self.backtest_report_directory, ignore_errors=True) def test_report_creation(self): app = create_app( diff --git a/tests/app/test_backtesting.py b/tests/app/test_backtesting.py index 3e7beda0..a9241151 100644 --- a/tests/app/test_backtesting.py +++ b/tests/app/test_backtesting.py @@ -18,6 +18,8 @@ from investing_algorithm_framework import TradingStrategy, Algorithm, \ create_app, RESOURCE_DIRECTORY, PortfolioConfiguration, \ BacktestDateRange, TimeUnit +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy from investing_algorithm_framework.domain import SQLALCHEMY_DATABASE_URI @@ -70,13 +72,14 @@ def setUp(self) -> None: def tearDown(self) -> None: super().tearDown() + teardown_sqlalchemy() for path in ( os.path.join(self.resource_dir, "databases"), self.backtest_databases_dir, self.backtest_reports_dir, ): if os.path.exists(path): - shutil.rmtree(path) + shutil.rmtree(path, ignore_errors=True) # --------------------------------------------------------------------------- @@ -253,4 +256,3 @@ def test_run_backtests(self): risk_free_rate=0.027 ) self.assertEqual(3, len(reports)) - diff --git a/tests/app/test_eventloop.py b/tests/app/test_eventloop.py index eaec81ee..5082d1f3 100644 --- a/tests/app/test_eventloop.py +++ b/tests/app/test_eventloop.py @@ -240,7 +240,7 @@ def tearDown(self) -> None: ) if os.path.exists(databases_directory): - shutil.rmtree(databases_directory) + shutil.rmtree(databases_directory, ignore_errors=True) if os.path.exists(backtest_databases_directory): - shutil.rmtree(backtest_databases_directory) + shutil.rmtree(backtest_databases_directory, ignore_errors=True) diff --git a/tests/app/test_start.py b/tests/app/test_start.py index fd2fb242..37573616 100644 --- a/tests/app/test_start.py +++ b/tests/app/test_start.py @@ -7,6 +7,8 @@ from investing_algorithm_framework import create_app, TradingStrategy, \ TimeUnit, RESOURCE_DIRECTORY, PortfolioConfiguration, Algorithm, \ MarketCredential +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy from tests.resources import OrderExecutorTest, PortfolioProviderTest @@ -106,6 +108,7 @@ def setUp(self) -> None: ) def tearDown(self): + teardown_sqlalchemy() for subdir in ("databases", "backtest_databases"): path = os.path.join(self.resource_dir, subdir) if os.path.exists(path): diff --git a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/algorithm_id.json b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/algorithm_id.json index 9ab210ec..fdb062a0 100644 --- a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/algorithm_id.json +++ b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/algorithm_id.json @@ -1,3 +1,3 @@ { - "algorithm_id": "TestStrategy" + "algorithm_id": "BacktestTestStrategy" } \ No newline at end of file diff --git a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json index 841ff1f7..5978f761 100644 --- a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json +++ b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json @@ -1 +1 @@ -{"backtest_start_date": "2023-12-01 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-02 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-01T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-23 19:17:03", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": []} \ No newline at end of file +{"backtest_start_date": "2023-12-01 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-02 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-01T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-26 14:17:11", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": [], "recorded_values": {}} \ No newline at end of file diff --git a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json index 462b897c..f1364612 100644 --- a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json +++ b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json @@ -1 +1 @@ -{"backtest_start_date": "2023-12-02 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-03 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-03T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-23 23:18:14", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": []} \ No newline at end of file +{"backtest_start_date": "2023-12-02 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-03 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-03T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-26 14:16:44", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": [], "recorded_values": {}} \ No newline at end of file diff --git a/tests/resources/test_base.py b/tests/resources/test_base.py index 912b1d26..d03070d2 100644 --- a/tests/resources/test_base.py +++ b/tests/resources/test_base.py @@ -11,8 +11,8 @@ MarketCredential from investing_algorithm_framework.domain import RESOURCE_DIRECTORY, \ ENVIRONMENT, Environment, BACKTEST_DATA_DIRECTORY_NAME -from investing_algorithm_framework.infrastructure.database import Session -from sqlalchemy.orm import close_all_sessions +from investing_algorithm_framework.infrastructure.database import \ + Session, teardown_sqlalchemy from tests.resources.stubs import OrderExecutorTest, PortfolioProviderTest logger = logging.getLogger(__name__) @@ -121,8 +121,8 @@ def _remove_database_dir(resource_dir): shutil.rmtree(database_dir, ignore_errors=True) def _cleanup_database(self): - """Close SQLAlchemy sessions and remove database files.""" - close_all_sessions() + """Close SQLAlchemy sessions, dispose engine, and remove db files.""" + teardown_sqlalchemy() self._remove_database_dir(self.resource_directory) def remove_database(self): @@ -246,7 +246,7 @@ def create_app(self): return self.iaf_app._flask_app def tearDown(self) -> None: - close_all_sessions() + teardown_sqlalchemy() database_dir = os.path.join(self.resource_directory, "databases") if os.path.exists(database_dir): shutil.rmtree(database_dir, ignore_errors=True) diff --git a/tests/scenarios/vectorized_backtests/test_metadata_preservation.py b/tests/scenarios/vectorized_backtests/test_metadata_preservation.py index 377a1a78..1f2e9576 100644 --- a/tests/scenarios/vectorized_backtests/test_metadata_preservation.py +++ b/tests/scenarios/vectorized_backtests/test_metadata_preservation.py @@ -8,6 +8,9 @@ from typing import Dict, Any from unittest import TestCase +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy + import pandas as pd from pyindicators import ema, rsi, crossover, crossunder @@ -424,5 +427,6 @@ def test_metadata_preserved_multiple_strategies(self): def tearDown(self): # Clean up storage directory after tests + teardown_sqlalchemy() if os.path.exists(self.backtest_storage_dir): - shutil.rmtree(self.backtest_storage_dir) + shutil.rmtree(self.backtest_storage_dir, ignore_errors=True) diff --git a/tests/scenarios/vectorized_backtests/test_run_vector_backtests.py b/tests/scenarios/vectorized_backtests/test_run_vector_backtests.py index dfec0503..6ea7b4c2 100644 --- a/tests/scenarios/vectorized_backtests/test_run_vector_backtests.py +++ b/tests/scenarios/vectorized_backtests/test_run_vector_backtests.py @@ -274,7 +274,7 @@ def test_run(self): # Clean up any existing storage directory if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) strategies = [] for param_set in param_variations: diff --git a/tests/scenarios/vectorized_backtests/test_use_backtest_storage_directory.py b/tests/scenarios/vectorized_backtests/test_use_backtest_storage_directory.py index aa45ed10..18c3724c 100644 --- a/tests/scenarios/vectorized_backtests/test_use_backtest_storage_directory.py +++ b/tests/scenarios/vectorized_backtests/test_use_backtest_storage_directory.py @@ -280,7 +280,7 @@ def test_run_with_backtest_storage_directory(self): # Clean up any existing storage directory if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) param_set = { "rsi_time_frame": "2h", @@ -427,7 +427,7 @@ def test_run_backtests_with_storage_directory(self): # Clean up any existing storage directory if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) start_time = time.time() @@ -475,7 +475,7 @@ def tearDown(self): resource_directory, "backtest_reports_for_testing", "temp_storage" ) if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) def test_preexisting_backtests_not_included_in_new_run(self): """ @@ -519,7 +519,7 @@ def test_preexisting_backtests_not_included_in_new_run(self): # Clean up any existing storage directory if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) # ===== FIRST RUN: Run backtests for first set of strategies ===== app1 = create_app(name="FirstRun", config=config) @@ -693,7 +693,7 @@ def test_preexisting_backtests_not_included_in_new_run(self): # Clean up if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) def test_preexisting_backtests_not_included_with_final_filter(self): """ @@ -735,7 +735,7 @@ def test_preexisting_backtests_not_included_with_final_filter(self): ) if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) # ===== FIRST RUN: Create pre-existing backtests ===== app1 = create_app(name="FirstRun", config=config) @@ -896,4 +896,4 @@ def tracking_final_filter(backtests): # Clean up if os.path.exists(backtest_storage_dir): - shutil.rmtree(backtest_storage_dir) + shutil.rmtree(backtest_storage_dir, ignore_errors=True) diff --git a/tests/scenarios/vectorized_backtests/test_with_window_filter_function.py b/tests/scenarios/vectorized_backtests/test_with_window_filter_function.py index 6e5fb432..22594174 100644 --- a/tests/scenarios/vectorized_backtests/test_with_window_filter_function.py +++ b/tests/scenarios/vectorized_backtests/test_with_window_filter_function.py @@ -627,4 +627,4 @@ def tearDown(self) -> None: ) if os.path.exists(temp_storage_dir): import shutil - shutil.rmtree(temp_storage_dir) + shutil.rmtree(temp_storage_dir, ignore_errors=True) From 7aa00703cdd7748ce7d2ad7981fcfcf3b45ef1c5 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 16:48:42 +0200 Subject: [PATCH 2/7] fix: rename test data files with Windows-incompatible characters Rename OHLCV CSV files that contained colons and spaces in their filenames (e.g. '08:00:00') to use dashes instead (e.g. '08-00'), matching the convention used by all other test data files. Windows does not allow colons in filenames, causing git checkout to fail on Windows CI runners. --- README.md | 8 ++++++-- .../runs/backtest_EUR_20231201_20231202/run.json | 2 +- .../runs/backtest_EUR_20231202_20231203/run.json | 2 +- ...-EUR_BITVAVO_2h_2021-12-15-08-00_2023-12-31-00-00.csv} | 0 ...-EUR_BITVAVO_2h_2023-08-11-16-00_2023-12-02-00-00.csv} | 0 5 files changed, 8 insertions(+), 4 deletions(-) rename tests/resources/test_data/ohlcv/{OHLCV_DOT-EUR_BITVAVO_2h_2021-12-15 08:00:00_2023-12-31 00:00:00.csv => OHLCV_DOT-EUR_BITVAVO_2h_2021-12-15-08-00_2023-12-31-00-00.csv} (100%) rename tests/resources/test_data/ohlcv/{OHLCV_DOT-EUR_BITVAVO_2h_2023-08-11 16:00:00_2023-12-02 00:00:00.csv => OHLCV_DOT-EUR_BITVAVO_2h_2023-08-11-16-00_2023-12-02-00-00.csv} (100%) diff --git a/README.md b/README.md index ce18e485..dac3bfc9 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- Create trading strategies. Compare them side by side. Pick the best one. 🚀 + Create trading strategies. Compare them side by side. Pick the best one and Deploy 🚀

@@ -61,7 +61,7 @@ Most quant frameworks stop at "here's your backtest result." You get a number, maybe a chart, and then you're on your own figuring out which strategy is actually better. -This framework is built around the full loop: **create strategies → backtest them → compare them in a single report → deploy the winner.** It generates a self-contained HTML dashboard that lets you rank, filter, and visually compare every strategy you've tested — all in one view, no notebooks required. +This framework is built around the full loop: **create strategies → vector backtest for signals analysis → compare them in a single report → event backtest the most promising strategies → deploy the winner.** It generates a self-contained HTML dashboard that lets you rank, filter, and visually compare every strategy you've tested — all in one view, no notebooks required.
@@ -69,6 +69,10 @@ This framework is built around the full loop: **create strategies → backtest t
- 📊 **30+ Metrics** — CAGR, Sharpe, Sortino, Calmar, VaR, CVaR, Max DD, Recovery & more +- ⚡ **Vector Backtesting for Signal Analysis** — Quickly test your strategy logic on historical data to see how signals would have behaved before committing to full event-driven backtests +- 🏃 **Event-Driven Backtesting** — Once promising strategies are identified via vector backtests, run full event-driven backtests to simulate realistic execution and portfolio management +- 🔀 **Permutation Testing / Monte Carlo Simulations** — Assess the statistical robustness of your strategies by running them across randomized market scenarios to see how often your results could occur by chance +- 🚀 **Deployment** — Once the best strategy is identified through backtesting and comparison, deploy it to production locally or in the cloud (AWS Lambda / Azure Functions) to start live trading - ⚔️ **Multi-Strategy Comparison** — Rank, filter & compare strategies in a single interactive report - 🪟 **Multi-Window Robustness** — Test across different time periods with window coverage analysis - 📈 **Equity & Drawdown Charts** — Overlay equity curves, rolling Sharpe, drawdown & return distributions diff --git a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json index 5978f761..7686a8c8 100644 --- a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json +++ b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231201_20231202/run.json @@ -1 +1 @@ -{"backtest_start_date": "2023-12-01 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-02 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-01T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-26 14:17:11", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": [], "recorded_values": {}} \ No newline at end of file +{"backtest_start_date": "2023-12-01 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-02 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-01T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-26 14:39:23", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": [], "recorded_values": {}} \ No newline at end of file diff --git a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json index f1364612..d2baec0c 100644 --- a/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json +++ b/tests/resources/backtest_reports_for_testing/test_algorithm_backtest/runs/backtest_EUR_20231202_20231203/run.json @@ -1 +1 @@ -{"backtest_start_date": "2023-12-02 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-03 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-03T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-26 14:16:44", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": [], "recorded_values": {}} \ No newline at end of file +{"backtest_start_date": "2023-12-02 00:00:00", "backtest_date_range_name": null, "backtest_end_date": "2023-12-03 00:00:00", "trading_symbol": "EUR", "initial_unallocated": 1000.0, "number_of_runs": 1441, "portfolio_snapshots": [{"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-02T00:00:00+00:00", "total_value": 1000.0}, {"metadata": "MetaData()", "portfolio_id": "1", "trading_symbol": "EUR", "pending_value": 0.0, "unallocated": 1000.0, "total_net_gain": 0.0, "total_revenue": 0.0, "total_cost": 0.0, "cash_flow": 0.0, "net_size": 1000.0, "created_at": "2023-12-03T00:00:00+00:00", "total_value": 1000.0}], "trades": [], "orders": [], "positions": [{"symbol": "EUR", "amount": 1000.0, "cost": 1000.0, "portfolio_id": 1}], "created_at": "2026-04-26 14:38:58", "symbols": [], "number_of_days": 0, "number_of_trades": 0, "number_of_trades_closed": 0, "number_of_trades_open": 0, "number_of_orders": 0, "number_of_positions": 0, "metadata": {}, "signals": {}, "signal_events": [], "recorded_values": {}} \ No newline at end of file diff --git a/tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2021-12-15 08:00:00_2023-12-31 00:00:00.csv b/tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2021-12-15-08-00_2023-12-31-00-00.csv similarity index 100% rename from tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2021-12-15 08:00:00_2023-12-31 00:00:00.csv rename to tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2021-12-15-08-00_2023-12-31-00-00.csv diff --git a/tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2023-08-11 16:00:00_2023-12-02 00:00:00.csv b/tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2023-08-11-16-00_2023-12-02-00-00.csv similarity index 100% rename from tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2023-08-11 16:00:00_2023-12-02 00:00:00.csv rename to tests/resources/test_data/ohlcv/OHLCV_DOT-EUR_BITVAVO_2h_2023-08-11-16-00_2023-12-02-00-00.csv From 089b09e65c528a33edcf933975582a6954e68684 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 17:26:02 +0200 Subject: [PATCH 3/7] fix: resolve Windows database locking and Unicode encoding issues - Call teardown_sqlalchemy() before os.remove() in initialize_storage() to release SQLite file locks on Windows (PermissionError WinError 32) - Use remove_database_if_exists=True in test setUp to prevent stale DB state between tests - Replace manual os.remove/os.rmdir loops with shutil.rmtree in web controller test teardowns - Open verbose output file with encoding='utf-8' in validate_backtest_checkpoints to fix UnicodeEncodeError on Windows (cp1252 cannot encode checkmark/cross characters) --- investing_algorithm_framework/app/app.py | 12 ++++++++++++ .../cli/validate_backtest_checkpoints.py | 2 +- .../controllers/order_controller/test_list_orders.py | 9 ++------- .../portfolio_controller/test_list_portfolio.py | 9 ++------- .../position_controller/test_list_positions.py | 9 ++------- tests/resources/test_base.py | 4 ++-- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/investing_algorithm_framework/app/app.py b/investing_algorithm_framework/app/app.py index b779acf4..6a3f4b36 100644 --- a/investing_algorithm_framework/app/app.py +++ b/investing_algorithm_framework/app/app.py @@ -368,6 +368,18 @@ def initialize_storage(self, remove_database_if_exists: bool = False): logger.info( f"Removing existing database at {database_path}" ) + + # Dispose the existing engine to release file locks + # (required on Windows where locks are mandatory) + from investing_algorithm_framework.infrastructure.database \ + import Session + from sqlalchemy.orm import close_all_sessions + close_all_sessions() + bind = Session.kw.get("bind") + + if bind is not None: + bind.dispose() + os.remove(database_path) # Create the sqlalchemy database uri diff --git a/investing_algorithm_framework/cli/validate_backtest_checkpoints.py b/investing_algorithm_framework/cli/validate_backtest_checkpoints.py index 3ff75ddc..5e591929 100644 --- a/investing_algorithm_framework/cli/validate_backtest_checkpoints.py +++ b/investing_algorithm_framework/cli/validate_backtest_checkpoints.py @@ -50,7 +50,7 @@ def validate_and_create_checkpoints( verbose_file_handle = None if verbose_output_file is not None: - verbose_file_handle = open(verbose_output_file, 'w') + verbose_file_handle = open(verbose_output_file, 'w', encoding='utf-8') def echo(msg): verbose_file_handle.write(msg + "\n") diff --git a/tests/app/web/controllers/order_controller/test_list_orders.py b/tests/app/web/controllers/order_controller/test_list_orders.py index 41138556..bac0afd2 100644 --- a/tests/app/web/controllers/order_controller/test_list_orders.py +++ b/tests/app/web/controllers/order_controller/test_list_orders.py @@ -1,5 +1,6 @@ import json import os +import shutil from investing_algorithm_framework import PortfolioConfiguration, \ MarketCredential @@ -33,13 +34,7 @@ def tearDown(self) -> None: ) if os.path.exists(database_dir): - for root, dirs, files in os.walk(database_dir, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - - os.rmdir(database_dir) + shutil.rmtree(database_dir, ignore_errors=True) def test_list_portfolios(self): self.iaf_app.add_strategy(StrategyOne) diff --git a/tests/app/web/controllers/portfolio_controller/test_list_portfolio.py b/tests/app/web/controllers/portfolio_controller/test_list_portfolio.py index 003e367b..039f86f0 100644 --- a/tests/app/web/controllers/portfolio_controller/test_list_portfolio.py +++ b/tests/app/web/controllers/portfolio_controller/test_list_portfolio.py @@ -1,5 +1,6 @@ import json import os +import shutil from investing_algorithm_framework import MarketCredential, \ PortfolioConfiguration, DataSource, INDEX_DATETIME @@ -33,13 +34,7 @@ def tearDown(self) -> None: ) if os.path.exists(database_dir): - for root, dirs, files in os.walk(database_dir, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - - os.rmdir(database_dir) + shutil.rmtree(database_dir, ignore_errors=True) def test_list_portfolios(self): strategy = StrategyOne() diff --git a/tests/app/web/controllers/position_controller/test_list_positions.py b/tests/app/web/controllers/position_controller/test_list_positions.py index ea1120c9..00a04a3c 100644 --- a/tests/app/web/controllers/position_controller/test_list_positions.py +++ b/tests/app/web/controllers/position_controller/test_list_positions.py @@ -1,5 +1,6 @@ import json import os +import shutil from investing_algorithm_framework import PortfolioConfiguration, \ MarketCredential @@ -33,13 +34,7 @@ def tearDown(self) -> None: ) if os.path.exists(database_dir): - for root, dirs, files in os.walk(database_dir, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - - os.rmdir(database_dir) + shutil.rmtree(database_dir, ignore_errors=True) def test_list_portfolios(self): self.iaf_app.add_strategy(StrategyOne) diff --git a/tests/resources/test_base.py b/tests/resources/test_base.py index d03070d2..d25afb1b 100644 --- a/tests/resources/test_base.py +++ b/tests/resources/test_base.py @@ -83,7 +83,7 @@ def setUp(self) -> None: if self.initialize: self.app.initialize_config() - self.app.initialize_storage() + self.app.initialize_storage(remove_database_if_exists=True) self.app.initialize_services() self.app.initialize_portfolios() @@ -215,7 +215,7 @@ def create_app(self): if self.initialize: self.iaf_app.initialize_config() - self.iaf_app.initialize_storage() + self.iaf_app.initialize_storage(remove_database_if_exists=True) self.iaf_app.initialize_services() self.iaf_app.initialize_portfolios() From a70fba43d08375a2194fea9b66ef3bdfc7541645 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 18:10:40 +0200 Subject: [PATCH 4/7] fix: force-close StaticPool DBAPI connection to release SQLite locks StaticPool._close_connection() is a no-op, so engine.dispose() alone never closes the underlying sqlite3 connection. On Windows, this leaves mandatory file locks in place, causing PermissionError when tests try to delete the database file. Use Connection.invalidate() which bypasses the pool's _close_connection and calls dialect.do_close() directly to actually close the DBAPI connection. Also add gc.collect() in initialize_storage to ensure any lingering references are cleaned up before file removal. --- investing_algorithm_framework/app/app.py | 11 +++++++++++ .../infrastructure/database/sql_alchemy.py | 13 +++++++++++++ 2 files changed, 24 insertions(+) diff --git a/investing_algorithm_framework/app/app.py b/investing_algorithm_framework/app/app.py index 6a3f4b36..e8b9749f 100644 --- a/investing_algorithm_framework/app/app.py +++ b/investing_algorithm_framework/app/app.py @@ -378,8 +378,19 @@ def initialize_storage(self, remove_database_if_exists: bool = False): bind = Session.kw.get("bind") if bind is not None: + + try: + conn = bind.connect() + conn.invalidate() + conn.close() + except Exception: + pass + bind.dispose() + import gc + gc.collect() + os.remove(database_path) # Create the sqlalchemy database uri diff --git a/investing_algorithm_framework/infrastructure/database/sql_alchemy.py b/investing_algorithm_framework/infrastructure/database/sql_alchemy.py index f8823e8f..e1bfbd8d 100644 --- a/investing_algorithm_framework/infrastructure/database/sql_alchemy.py +++ b/investing_algorithm_framework/infrastructure/database/sql_alchemy.py @@ -93,6 +93,19 @@ def teardown_sqlalchemy(): bind = Session.kw.get("bind") if bind is not None: + + # StaticPool._close_connection() is a no-op, so + # engine.dispose() alone won't close the underlying DBAPI + # connection. Use invalidate() which bypasses the pool's + # _close_connection and calls dialect.do_close() directly, + # ensuring the sqlite3 file lock is released on Windows. + try: + conn = bind.connect() + conn.invalidate() + conn.close() + except Exception: + pass + bind.dispose() Session.configure(bind=None) From 89f983facf79bac6fbf68ff2469d4c6f997ab096 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 18:49:09 +0200 Subject: [PATCH 5/7] fix: add teardown_sqlalchemy to RunTestBase to prevent stale DB on Windows RunTestBase.tearDown() was calling shutil.rmtree with ignore_errors=True but without first releasing the SQLite file lock via teardown_sqlalchemy(). On Windows, this left the database file behind, causing the next test to find a stale BITVAVO portfolio when expecting BINANCE. --- tests/app/test_run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/app/test_run.py b/tests/app/test_run.py index bf248f01..1c3ecff1 100644 --- a/tests/app/test_run.py +++ b/tests/app/test_run.py @@ -15,6 +15,8 @@ from investing_algorithm_framework import create_app, TradingStrategy, \ TimeUnit, PortfolioConfiguration, RESOURCE_DIRECTORY, \ Algorithm, MarketCredential +from investing_algorithm_framework.infrastructure.database import \ + teardown_sqlalchemy from tests.resources import random_string, OrderExecutorTest, \ PortfolioProviderTest @@ -105,6 +107,7 @@ def setUp(self) -> None: def tearDown(self) -> None: super().tearDown() + teardown_sqlalchemy() for subdir in ("databases", "backtest_databases"): path = os.path.join(self.resource_dir, subdir) if os.path.exists(path): From ff4b77ef80201baf6b97831969e123128f730aa0 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 19:23:56 +0200 Subject: [PATCH 6/7] ci: increase test timeout to 40 minutes for Windows coverage runs --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 386cba70..64a4325c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: flake8 ./investing_algorithm_framework test: needs: linting - timeout-minutes: 25 + timeout-minutes: 40 strategy: fail-fast: true matrix: From 526fb1ace62985b3f7a5918ebf05b1b0a6f40a04 Mon Sep 17 00:00:00 2001 From: marcvanduyn Date: Sun, 26 Apr 2026 19:29:38 +0200 Subject: [PATCH 7/7] ci: retrigger pipeline --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64a4325c..c35c9b4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,7 @@ on: paths: - 'investing_algorithm_framework/**' - 'tests/**' + - '.github/workflows/test.yml' jobs: linting: