Refactored sweep code out of load, and added some tests (#48)

Author: mccaffers

CMakeLists.txt

@@ -56,6 +56,7 @@ target_include_directories(BacktestingEngineLib PUBLIC
     ${CMAKE_SOURCE_DIR}/include/trading
     ${CMAKE_SOURCE_DIR}/include/trading_definitions
     ${CMAKE_SOURCE_DIR}/include/strategies
+    ${CMAKE_SOURCE_DIR}/include/sweep
     ${CMAKE_SOURCE_DIR}/external
 )
 

backtesting-engine-cpp.xcodeproj/project.pbxproj

@@ -7,6 +7,13 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		940F6E532FD9479800B0364A /* randomStrategySweep.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E502FD9479800B0364A /* randomStrategySweep.cpp */; };
+		940F6E542FD9479800B0364A /* runConfigurationBuilder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E512FD9479800B0364A /* runConfigurationBuilder.cpp */; };
+		940F6E552FD9479800B0364A /* randomStrategySweep.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E502FD9479800B0364A /* randomStrategySweep.cpp */; };
+		940F6E562FD9479800B0364A /* runConfigurationBuilder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E512FD9479800B0364A /* runConfigurationBuilder.cpp */; };
+		940F6E582FD947A900B0364A /* decimalConvert.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E572FD947A900B0364A /* decimalConvert.cpp */; };
+		940F6E592FD947A900B0364A /* decimalConvert.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E572FD947A900B0364A /* decimalConvert.cpp */; };
+		940F6E5B2FD947C000B0364A /* sweep.mm in Sources */ = {isa = PBXBuildFile; fileRef = 940F6E5A2FD947B800B0364A /* sweep.mm */; };
 		941408AE2D59F93F000ED1F9 /* sqlManager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 941408AD2D59F93F000ED1F9 /* sqlManager.cpp */; };
 		941408AF2D59F93F000ED1F9 /* sqlManager.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 941408AD2D59F93F000ED1F9 /* sqlManager.cpp */; };
 		941B549A2D3BBADE00E3BF64 /* trading_definitions_json.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 941B54992D3BBADD00E3BF64 /* trading_definitions_json.cpp */; };
@@ -69,6 +76,13 @@
 
 /* Begin PBXFileReference section */
 		9409A61B2FAA6411002C30FF /* strategy.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = strategy.hpp; sourceTree = "<group>"; };
+		940F6E4C2FD9477E00B0364A /* randomStrategySweep.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = randomStrategySweep.hpp; sourceTree = "<group>"; };
+		940F6E4D2FD9477E00B0364A /* runConfigurationBuilder.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = runConfigurationBuilder.hpp; sourceTree = "<group>"; };
+		940F6E4F2FD9478800B0364A /* decimalConvert.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = decimalConvert.hpp; sourceTree = "<group>"; };
+		940F6E502FD9479800B0364A /* randomStrategySweep.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = randomStrategySweep.cpp; sourceTree = "<group>"; };
+		940F6E512FD9479800B0364A /* runConfigurationBuilder.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = runConfigurationBuilder.cpp; sourceTree = "<group>"; };
+		940F6E572FD947A900B0364A /* decimalConvert.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = decimalConvert.cpp; sourceTree = "<group>"; };
+		940F6E5A2FD947B800B0364A /* sweep.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = sweep.mm; sourceTree = "<group>"; };
 		941408AD2D59F93F000ED1F9 /* sqlManager.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = sqlManager.cpp; sourceTree = "<group>"; };
 		941408B02D59F954000ED1F9 /* sqlManager.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = sqlManager.hpp; sourceTree = "<group>"; };
 		941B54902D3BBA4900E3BF64 /* ohlc_variables.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = ohlc_variables.hpp; sourceTree = "<group>"; };
@@ -1323,6 +1337,24 @@
 /* End PBXFrameworksBuildPhase section */
 
 /* Begin PBXGroup section */
+		940F6E4E2FD9477E00B0364A /* sweep */ = {
+			isa = PBXGroup;
+			children = (
+				940F6E4C2FD9477E00B0364A /* randomStrategySweep.hpp */,
+				940F6E4D2FD9477E00B0364A /* runConfigurationBuilder.hpp */,
+			);
+			path = sweep;
+			sourceTree = "<group>";
+		};
+		940F6E522FD9479800B0364A /* sweep */ = {
+			isa = PBXGroup;
+			children = (
+				940F6E502FD9479800B0364A /* randomStrategySweep.cpp */,
+				940F6E512FD9479800B0364A /* runConfigurationBuilder.cpp */,
+			);
+			path = sweep;
+			sourceTree = "<group>";
+		};
 		941B548F2D3BBA3B00E3BF64 /* trading_definitions */ = {
 			isa = PBXGroup;
 			children = (
@@ -1348,6 +1380,7 @@
 		94280BA72D2FC29F00F1CF56 /* utilities */ = {
 			isa = PBXGroup;
 			children = (
+				940F6E572FD947A900B0364A /* decimalConvert.cpp */,
 				94829FC42FCC1D1A00710E6E /* env.cpp */,
 				942EC5612FBEF94700CCBB5D /* redisConnection.cpp */,
 				94280BA22D2FC00200F1CF56 /* base64.cpp */,
@@ -1459,7 +1492,7 @@
 		9470B5A22C8C5AD0007D9CC6 /* source */ = {
 			isa = PBXGroup;
 			children = (
-				943770442FD4396F00317424 /* boostRedisImpl.cpp */,
+				940F6E522FD9479800B0364A /* sweep */,
 				9437703E2FD42FC000317424 /* reporting */,
 				941B54982D3BBAD800E3BF64 /* trading_definitions */,
 				94674B862D533B4000973137 /* trading */,
@@ -1468,6 +1501,7 @@
 				94D3A7252FC1B3AD00EBEA32 /* commands */,
 				94280BA72D2FC29F00F1CF56 /* utilities */,
 				942EC5642FBEF95000CCBB5D /* backtestRunner.cpp */,
+				943770442FD4396F00317424 /* boostRedisImpl.cpp */,
 				942EC5652FBEF95000CCBB5D /* redisLoader.cpp */,
 				942FDDDF2FC5C8B20096F318 /* tradingResults.cpp */,
 				942EC5662FBEF95000CCBB5D /* redisRunner.cpp */,
@@ -1482,6 +1516,7 @@
 		9470B5AD2C8C5B99007D9CC6 /* tests */ = {
 			isa = PBXGroup;
 			children = (
+				940F6E5A2FD947B800B0364A /* sweep.mm */,
 				9464E5F02FA7467200D82BAD /* symbolScale.mm */,
 				943398262D57E54000287A2D /* jsonParser.mm */,
 				94674B892D533BDA00973137 /* tradeManager.mm */,
@@ -1493,6 +1528,7 @@
 		94B8C7932D3D770800E17EB6 /* utilities */ = {
 			isa = PBXGroup;
 			children = (
+				940F6E4F2FD9478800B0364A /* decimalConvert.hpp */,
 				94B4F02A2FD618B300B08FB4 /* backtestLog.hpp */,
 				94B4F02B2FD618B300B08FB4 /* threadPool.hpp */,
 				945F475C2FD5607E00D19164 /* queueKeys.hpp */,
@@ -3616,6 +3652,7 @@
 		94DE4F772C8C3E7C00FE48FF /* include */ = {
 			isa = PBXGroup;
 			children = (
+				940F6E4E2FD9477E00B0364A /* sweep */,
 				943770422FD42FDD00317424 /* reporting */,
 				94D3A7232FC1B3A600EBEA32 /* commands */,
 				942966D72D48E84100532862 /* models */,
@@ -3740,9 +3777,12 @@
 				942EC56B2FBEF95000CCBB5D /* redisLoader.cpp in Sources */,
 				94829FC52FCC1D1A00710E6E /* env.cpp in Sources */,
 				942EC56C2FBEF95000CCBB5D /* redisRunner.cpp in Sources */,
+				940F6E592FD947A900B0364A /* decimalConvert.cpp in Sources */,
 				94280BA32D2FC00200F1CF56 /* base64.cpp in Sources */,
 				943770402FD42FC000317424 /* elasticClient.cpp in Sources */,
 				941B549B2D3BBADE00E3BF64 /* trading_definitions_json.cpp in Sources */,
+				940F6E532FD9479800B0364A /* randomStrategySweep.cpp in Sources */,
+				940F6E542FD9479800B0364A /* runConfigurationBuilder.cpp in Sources */,
 				946EFF7E2FB9F44E008D9647 /* reporting.cpp in Sources */,
 				94674B872D533B4000973137 /* tradeManager.cpp in Sources */,
 				94CD8BA02D2E8CE500041BBA /* databaseConnection.cpp in Sources */,
@@ -3779,6 +3819,10 @@
 				942FDDE22FC5C8B20096F318 /* tradingResults.cpp in Sources */,
 				9437703F2FD42FC000317424 /* elasticClient.cpp in Sources */,
 				9470B5B62C8C5BFD007D9CC6 /* main.cpp in Sources */,
+				940F6E552FD9479800B0364A /* randomStrategySweep.cpp in Sources */,
+				940F6E5B2FD947C000B0364A /* sweep.mm in Sources */,
+				940F6E562FD9479800B0364A /* runConfigurationBuilder.cpp in Sources */,
+				940F6E582FD947A900B0364A /* decimalConvert.cpp in Sources */,
 				94364CB62D416D8D00F35B55 /* db.mm in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

include/sweep/randomStrategySweep.hpp

@@ -0,0 +1,18 @@
+// Backtesting Engine in C++
+//
+// (c) 2026 Ryan McCaffery | https://mccaffers.com
+// This code is licensed under MIT license (see LICENSE.txt for details)
+// ---------------------------------------
+
+#pragma once
+
+#include "parameterSweep.hpp"
+
+namespace sweep {
+
+// Declares which parameters to sweep for the RandomStrategy. Keeping the ranges
+// in one place means a new strategy (or extra swept parameter) is a localised
+// edit: register it here, then read it back in makeStrategy().
+ParameterGenerator buildRandomStrategySweep();
+
+}  // namespace sweep

include/sweep/runConfigurationBuilder.hpp

@@ -0,0 +1,20 @@
+// Backtesting Engine in C++
+//
+// (c) 2026 Ryan McCaffery | https://mccaffers.com
+// This code is licensed under MIT license (see LICENSE.txt for details)
+// ---------------------------------------
+
+#pragma once
+
+#include <string>
+
+#include "run_configuration.hpp"
+
+namespace sweep {
+
+// The run-level descriptor: what tick data to pull from QuestDB, plus the risk
+// limits every strategy in the sweep runs under. Shared by every strategy in
+// this sweep and linked to them by RUN_ID.
+trading_definitions::RunConfiguration makeRunConfiguration(const std::string& runId);
+
+}  // namespace sweep

include/utilities/decimalConvert.hpp

@@ -0,0 +1,20 @@
+// Backtesting Engine in C++
+//
+// (c) 2026 Ryan McCaffery | https://mccaffers.com
+// This code is licensed under MIT license (see LICENSE.txt for details)
+// ---------------------------------------
+
+#pragma once
+
+#include <boost/decimal.hpp>
+
+namespace decimal_convert {
+
+// Converts a double into a decimal64_t via its shortest round-trip decimal
+// string. Routing through text (rather than constructing from the binary double)
+// stops clean decimals like 1.5 / 2.0 from snapping to an IEEE-754 neighbour,
+// consistent with how decimal_json.hpp moves values through JSON. Feed it
+// binary-exact values (halves, quarters) or explicit lists to keep this exact.
+boost::decimal::decimal64_t toDecimal(double value);
+
+}  // namespace decimal_convert

scripts/generate_clangd.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Generates the .clangd IntelliSense config at the repo root.
+#
+# .clangd needs absolute, machine-specific paths (clangd resolves relative -I
+# flags against the inferred command's working directory, not the repo root),
+# so the file is gitignored and each user regenerates it locally with this
+# script. Re-run it after adding a new include/ subdirectory — and add the new
+# directory to the list below, mirroring CMakeLists.txt.
+
+# Fail fast: -e exits on any error, -u errors on undefined vars,
+# pipefail propagates failures through pipes.
+set -euo pipefail
+
+# Resolve paths relative to this script, not the caller's working directory,
+# so the script works no matter where it's invoked from.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Homebrew lives at /opt/homebrew on Apple Silicon and /usr/local on Intel;
+# `brew --prefix` resolves the right one. The keg-only formulas (libomp,
+# libpq) are addressed via their own prefixes rather than <prefix>/opt/... so
+# the script also works if they were installed elsewhere.
+BREW_PREFIX="$(brew --prefix)"
+LIBOMP_PREFIX="$(brew --prefix libomp)"
+LIBPQ_PREFIX="$(brew --prefix libpq)"
+
+# Where the XCTest framework lives for the active Xcode installation.
+XCODE_FRAMEWORKS="$(xcode-select -p)/Platforms/MacOSX.platform/Developer/Library/Frameworks"
+
+OUTPUT="$REPO_ROOT/.clangd"
+
+cat > "$OUTPUT" <<EOF
+# clangd IntelliSense config — generated by scripts/generate_clangd.sh; edit
+# that script (not this file) and re-run it.
+# The XCTest tests (tests/*.mm) are built by Xcode, not CMake, so they have no
+# entry in build/compile_commands.json. Without flags, clangd can't find the
+# XCTest framework or project headers. These flags mirror what CMake emits in
+# build/compile_commands.json, scoped to the .mm test files.
+#
+# NOTE: paths must be absolute. clangd resolves a relative -I against the
+# inferred command's working directory (build/external/libpqxx/src), not the
+# repo root, so relative paths break header resolution. This file is therefore
+# machine-specific and gitignored.
+If:
+  PathMatch: .*\.mm
+CompileFlags:
+  Add:
+    - -xobjective-c++
+    - -std=gnu++2b
+    - -F$XCODE_FRAMEWORKS
+    # Project headers (include/ and its subdirs — tests use bare #import "x.hpp")
+    - -I$REPO_ROOT/include
+    - -I$REPO_ROOT/include/commands
+    - -I$REPO_ROOT/include/models
+    - -I$REPO_ROOT/include/reporting
+    - -I$REPO_ROOT/include/strategies
+    - -I$REPO_ROOT/include/sweep
+    - -I$REPO_ROOT/include/trading
+    - -I$REPO_ROOT/include/trading_definitions
+    - -I$REPO_ROOT/include/utilities
+    # Third-party headers
+    - -I$REPO_ROOT/external
+    - -I$REPO_ROOT/external/boost-decimal/include
+    - -I$REPO_ROOT/external/libpqxx/include
+    - -I$REPO_ROOT/build/external/libpqxx/include
+    - -I$LIBOMP_PREFIX/include
+    - -isystem$BREW_PREFIX/include
+    - -isystem$LIBPQ_PREFIX/include
+    - -isystem$LIBPQ_PREFIX/include/postgresql/server
+EOF
+
+echo "Wrote $OUTPUT"

source/commands/loadCommand.cpp

@@ -6,90 +6,31 @@
 
 #include "loadCommand.hpp"
 
-#include <array>
-#include <charconv>
 #include <print>
 #include <ranges>
-#include <stdexcept>
 #include <string>
-#include <system_error>
 #include <vector>
 
 #include <boost/decimal.hpp>
-#include <boost/decimal/charconv.hpp>
 #include <boost/decimal/literals.hpp>
 #include <nlohmann/json.hpp>
 
 #include <boost/uuid/random_generator.hpp>
 #include <boost/uuid/uuid_io.hpp>
 
+#include "decimalConvert.hpp"
 #include "env.hpp"
 #include "parameterSweep.hpp"
 #include "queueKeys.hpp"
+#include "randomStrategySweep.hpp"
 #include "redisLoader.hpp"
-#include "run_configuration.hpp"
+#include "runConfigurationBuilder.hpp"
 #include "trading_definitions/strategy.hpp"
 
 namespace {
 
-using trading_definitions::RunConfiguration;
 using trading_definitions::Strategy;
 
-// Converts a swept double into a decimal64_t via its shortest round-trip decimal
-// string. Routing through text (rather than constructing from the binary double)
-// stops clean decimals like 1.5 / 2.0 from snapping to an IEEE-754 neighbour,
-// consistent with how decimal_json.hpp moves values through JSON. Sweep with
-// binary-exact steps (halves, quarters) or explicit lists to keep this exact.
-boost::decimal::decimal64_t toDecimal(double value) {
-    std::array<char, 64> buffer;
-    const auto [ptr, ec] =
-        std::to_chars(buffer.data(), buffer.data() + buffer.size(), value);
-    if (ec != std::errc{}) {
-        throw std::runtime_error("loadCommand: to_chars failed for swept decimal");
-    }
-    boost::decimal::decimal64_t result;
-    const auto parsed = boost::decimal::from_chars(buffer.data(), ptr, result);
-    if (parsed.ec != std::errc{}) {
-        throw std::runtime_error("loadCommand: from_chars failed for swept decimal");
-    }
-    return result;
-}
-
-// Declares which parameters to sweep for the RandomStrategy. Keeping the ranges
-// in one place means a new strategy (or extra swept parameter) is a localised
-// edit: register it here, then read it back in makeConfiguration().
-sweep::ParameterGenerator buildRandomStrategySweep() {
-    sweep::ParameterGenerator generator;
-    // generator.addRange("OHLC_COUNT", 80, 20, 140);  // 80, 100, 120, 140
-    // generator.addList("OHLC_MINUTES", {1, 3, 5, 8});
-    generator.addList("STOP_DISTANCE_IN_PIPS", {1.0, 1.5, 10.0});
-    generator.addList("LIMIT_DISTANCE_IN_PIPS", {1.0, 1.5, 10.0});
-    return generator;
-}
-
-// The run-level descriptor: what tick data to pull from QuestDB, plus the risk
-// limits every strategy in the sweep runs under. Shared by every strategy in
-// this sweep and linked to them by RUN_ID.
-RunConfiguration makeRunConfiguration(const std::string& runId) {
-    using namespace boost::decimal::literals;
-    return RunConfiguration{
-        .RUN_ID = runId,
-        .SYMBOLS = "EURUSD",
-        .LAST_MONTHS = 6,
-        .STARTING_BALANCE = trading_definitions::DEFAULT_STARTING_BALANCE,
-        // Cut a run off once it has lost 5% of the account (fail fast);
-        // set <= 0 to run without any loss cutoff.
-        .MAX_LOSS_PERCENT = 5_DD,
-        // Cap on simultaneously open positions per run (<= 0 = unlimited).
-        // The per-symbol gate already limits to one trade per symbol, so this
-        // only bites on multi-symbol runs.
-        .MAX_OPEN_TRADES = 10,
-        // Flip to false to silence liquidated runs from Elasticsearch once
-        // sweeps scale up and loss-limit cutoffs are expected noise.
-        .REPORT_FAILURES = true,
-    };
-}
-
 // Maps one point in the parameter grid onto a Strategy. Fields not being swept
 // keep their fixed defaults; swept fields are pulled from `combo`.
 Strategy makeStrategy(const sweep::Combination& combo) {
@@ -102,8 +43,10 @@ Strategy makeStrategy(const sweep::Combination& combo) {
         .UUID = boost::uuids::to_string(boost::uuids::random_generator()()),
         .TRADING_VARIABLES = TradingVariables{
             .STRATEGY = "RandomStrategy",
-            .STOP_DISTANCE_IN_PIPS = toDecimal(combo.get("STOP_DISTANCE_IN_PIPS")),
-            .LIMIT_DISTANCE_IN_PIPS = toDecimal(combo.get("LIMIT_DISTANCE_IN_PIPS")),
+            .STOP_DISTANCE_IN_PIPS =
+                decimal_convert::toDecimal(combo.get("STOP_DISTANCE_IN_PIPS")),
+            .LIMIT_DISTANCE_IN_PIPS =
+                decimal_convert::toDecimal(combo.get("LIMIT_DISTANCE_IN_PIPS")),
             .TRADING_SIZE = 1_DD,
         },
         .OHLC_VARIABLES = {
@@ -129,7 +72,7 @@ int LoadCommand::run() {
     const auto redisHost = env::getOr("REDIS_HOST", "127.0.0.1");
 
     // Build random here
-    const auto generator = buildRandomStrategySweep();
+    const auto generator = sweep::buildRandomStrategySweep();
 
     const auto combinations = generator.generateAllCombinations();
 
@@ -159,7 +102,7 @@ int LoadCommand::run() {
     // Now advertise the run so workers can pick it up. runJson is pinned to
     // nlohmann::json (not auto) because makeRunConfiguration returns a
     // RunConfiguration and relies on the implicit conversion for .dump().
-    const nlohmann::json runJson = makeRunConfiguration(runId);
+    const nlohmann::json runJson = sweep::makeRunConfiguration(runId);
     return RedisLoader::loadPayload(redisHost, 6379, queue_keys::RUN,
                                     runJson.dump());
 }