Skip to content

fix(benchmarker): use approximate problem scale instead of log in the report #2336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
triceo merged 2 commits into from
Jun 6, 2026
Merged

fix(benchmarker): use approximate problem scale instead of log in the report #2336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
86ca247
chore: use approximate problem scale instead of log in benchmark report
Christopher-Chianelli Jun 5, 2026
45f0d53
chore: be backward compatiable when possible
Christopher-Chianelli Jun 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 1 addition & 51 deletions core/src/main/java/ai/timefold/solver/core/api/solver/ProblemSizeStatistics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package ai.timefold.solver.core.api.solver;

import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.Locale;

import ai.timefold.solver.core.impl.util.MathUtils;
Expand All @@ -23,13 +21,6 @@ public record ProblemSizeStatistics(long entityCount,
long approximateValueCount,
double approximateProblemSizeLog) {

private static final Locale FORMATTER_LOCALE = Locale.getDefault();
private static final DecimalFormat BASIC_FORMATTER = new DecimalFormat("#,###");

// Exponent should not use grouping, unlike basic
private static final DecimalFormat EXPONENT_FORMATTER = new DecimalFormat("#");
private static final DecimalFormat SIGNIFICANT_FIGURE_FORMATTER = new DecimalFormat("0.######");

/**
* Return the {@link #approximateProblemSizeLog} as a fixed point integer.
*/
Expand All @@ -38,48 +29,7 @@ public long approximateProblemScaleLogAsFixedPointLong() {
}

public String approximateProblemScaleAsFormattedString() {
return approximateProblemScaleAsFormattedString(Locale.getDefault());
}

String approximateProblemScaleAsFormattedString(Locale locale) {
if (Double.isNaN(approximateProblemSizeLog) || Double.isInfinite(approximateProblemSizeLog)) {
return "0";
}

if (approximateProblemSizeLog < 10) { // log_10(10_000_000_000) = 10
return "%s".formatted(format(Math.pow(10d, approximateProblemSizeLog), BASIC_FORMATTER, locale));
}
// The actual number will often be too large to fit in a double, so cannot use basic
// formatting.
// Separate the exponent into its integral and fractional parts
// Use the integral part as the power of 10, and the fractional part as the significant digits.
double exponentPart = Math.floor(approximateProblemSizeLog);
double remainderPartAsExponent = approximateProblemSizeLog - exponentPart;
double remainderPart = Math.pow(10, remainderPartAsExponent);
return "%s × 10^%s".formatted(
format(remainderPart, SIGNIFICANT_FIGURE_FORMATTER, locale),
format(exponentPart, EXPONENT_FORMATTER, locale));
}

/**
* In order for tests to work currently regardless of the default system locale,
* we need to set the locale to a known value before running the tests.
* And because the {@link DecimalFormat} instances are initialized statically for reasons of performance,
* we cannot expect them to be in the locale that the test expects them to be in.
* This method exists to allow for an override.
*
* @return the given decimalFormat with the given locale
*/
private static String format(double number, DecimalFormat decimalFormat, Locale locale) {
if (locale.equals(FORMATTER_LOCALE)) {
return decimalFormat.format(number);
}
try { // Slow path for corner cases where input locale doesn't match the default locale.
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(locale));
return decimalFormat.format(number);
} finally {
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(FORMATTER_LOCALE));
}
return MathUtils.approximateProblemScaleAsFormattedString(approximateProblemSizeLog, Locale.getDefault());
}

}
49 changes: 49 additions & 0 deletions core/src/main/java/ai/timefold/solver/core/impl/util/MathUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package ai.timefold.solver.core.impl.util;

import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.Locale;

/**
* Includes code taken from Apache's commons-math library, v3.6.1 (Apache 2.0-licensed)
* These methods are clearly marked in comments, their modifications listed.
Expand All @@ -9,6 +13,11 @@ public class MathUtils {
public static final long LOG_PRECISION = 1_000_000L;
private static final int FACTORIAL_MAX_N = 20;
private static final long[] FACTORIAL_CACHE = new long[FACTORIAL_MAX_N - 1]; // 0 and 1 are hard-coded.
public static final Locale FORMATTER_LOCALE = Locale.getDefault();
private static final DecimalFormat BASIC_FORMATTER = new DecimalFormat("#,###");
// Exponent should not use grouping, unlike basic
private static final DecimalFormat EXPONENT_FORMATTER = new DecimalFormat("#");
private static final DecimalFormat SIGNIFICANT_FIGURE_FORMATTER = new DecimalFormat("0.######");

private MathUtils() {
}
Expand Down Expand Up @@ -346,4 +355,44 @@ private static long mulAndCheck(long a, long b) {
}
}

public static String approximateProblemScaleAsFormattedString(double approximateProblemSizeLog, Locale locale) {
if (Double.isNaN(approximateProblemSizeLog) || Double.isInfinite(approximateProblemSizeLog)) {
return "0";
}

if (approximateProblemSizeLog < 10) { // log_10(10_000_000_000) = 10
return "%s".formatted(formatNumber(Math.pow(10d, approximateProblemSizeLog), BASIC_FORMATTER, locale));
}
// The actual number will often be too large to fit in a double, so cannot use basic
// formatting.
// Separate the exponent into its integral and fractional parts
// Use the integral part as the power of 10, and the fractional part as the significant digits.
double exponentPart = Math.floor(approximateProblemSizeLog);
double remainderPartAsExponent = approximateProblemSizeLog - exponentPart;
double remainderPart = Math.pow(10, remainderPartAsExponent);
return "%s × 10^%s".formatted(
formatNumber(remainderPart, SIGNIFICANT_FIGURE_FORMATTER, locale),
formatNumber(exponentPart, EXPONENT_FORMATTER, locale));
}

/**
* In order for tests to work currently regardless of the default system locale,
* we need to set the locale to a known value before running the tests.
* And because the {@link DecimalFormat} instances are initialized statically for reasons of performance,
* we cannot expect them to be in the locale that the test expects them to be in.
* This method exists to allow for an override.
*
* @return the given decimalFormat with the given locale
*/
private static String formatNumber(double number, DecimalFormat decimalFormat, Locale locale) {
if (locale.equals(MathUtils.FORMATTER_LOCALE)) {
return decimalFormat.format(number);
}
try { // Slow path for corner cases where input locale doesn't match the default locale.
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(locale));
return decimalFormat.format(number);
} finally {
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(MathUtils.FORMATTER_LOCALE));
}
}
}
11 changes: 6 additions & 5 deletions ...c/main/java/ai/timefold/solver/benchmark/aggregator/swingui/BenchmarkAggregatorFrame.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,12 +518,13 @@ private MixedCheckBox createSolverBenchmarkCheckBox(SolverBenchmarkResult solver
}

private MixedCheckBox createProblemBenchmarkCheckBox(ProblemBenchmarkResult problemBenchmarkResult) {
String problemBenchmarkDetail = String.format(
"Entity count: %d%n"
+ "Problem scale: %d%n"
+ "Used memory: %s",
var problemBenchmarkDetail = String.format(
"""
Entity count: %d%n\
Problem scale: %s%n\
Used memory: %s""",
problemBenchmarkResult.getEntityCount(),
problemBenchmarkResult.getProblemScale(),
problemBenchmarkResult.getFormattedProblemScale(),
toEmptyStringIfNull(problemBenchmarkResult.getAverageUsedMemoryAfterInputSolution()));
return new MixedCheckBox(problemBenchmarkResult.getName(), problemBenchmarkDetail);
}
Expand Down
12 changes: 6 additions & 6 deletions tools/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/report/BenchmarkReport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,8 +372,8 @@ private List<LineChart<Long, Double>> createBestScoreScalabilitySummaryChart() {
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
double[] levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
var problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
var levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= builderList.size()) {
builderList.add(new LineChart.Builder<>());
Expand Down Expand Up @@ -556,8 +556,8 @@ private LineChart<Long, Long> createScalabilitySummaryChart(ToLongFunction<Singl
.stream()
.filter(SingleBenchmarkResult::hasAllSuccess)
.forEach(singleBenchmarkResult -> {
long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
long timeMillisSpent = valueFunction.applyAsLong(singleBenchmarkResult);
var problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
var timeMillisSpent = valueFunction.applyAsLong(singleBenchmarkResult);
builder.add(solverLabel, problemScale, timeMillisSpent);
});
}
Expand All @@ -570,8 +570,8 @@ private List<LineChart<Long, Double>> createBestScorePerTimeSpentSummaryChart()
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) {
if (singleBenchmarkResult.hasAllSuccess()) {
long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
double[] levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
var timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
var levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
if (i >= builderList.size()) {
builderList.add(new LineChart.Builder<>());
Expand Down
31 changes: 17 additions & 14 deletions ...chmark/src/main/java/ai/timefold/solver/benchmark/impl/result/PlannerBenchmarkResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.Comparator;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
Expand All @@ -23,7 +24,7 @@
import ai.timefold.solver.core.config.solver.EnvironmentMode;
import ai.timefold.solver.core.config.util.ConfigUtils;
import ai.timefold.solver.core.enterprise.TimefoldSolverEnterpriseService;
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
import ai.timefold.solver.core.impl.util.MathUtils;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -67,7 +68,7 @@ public class PlannerBenchmarkResult {
// ************************************************************************

private Integer failureCount = null;
private Long averageProblemScale = null;
private String averageProblemScale = null;
private Score averageScore = null;
private SolverBenchmarkResult favoriteSolverBenchmarkResult = null;

Expand Down Expand Up @@ -184,7 +185,7 @@ public Integer getFailureCount() {
return failureCount;
}

public Long getAverageProblemScale() {
public String getAverageProblemScale() {
return averageProblemScale;
}

Expand Down Expand Up @@ -316,32 +317,34 @@ public void accumulateResults(BenchmarkReport benchmarkReport) {

private <Score_ extends Score<Score_>> void determineTotalsAndAverages() {
failureCount = 0;
long totalProblemScale = 0L;
int problemScaleCount = 0;
for (ProblemBenchmarkResult problemBenchmarkResult : unifiedProblemBenchmarkResultList) {
Long problemScale = problemBenchmarkResult.getProblemScale();
var totalProblemScale = 0L;
var problemScaleCount = 0;
for (var problemBenchmarkResult : unifiedProblemBenchmarkResultList) {
var problemScale = problemBenchmarkResult.getProblemScale();
if (problemScale != null && problemScale >= 0L) {
totalProblemScale += problemScale;
problemScaleCount++;
}
failureCount += problemBenchmarkResult.getFailureCount();
}
averageProblemScale = problemScaleCount == 0 ? null : totalProblemScale / problemScaleCount;
averageProblemScale = problemScaleCount == 0 ? null
: MathUtils.approximateProblemScaleAsFormattedString(
(double) totalProblemScale / problemScaleCount / MathUtils.LOG_PRECISION, Locale.getDefault());
Score_ totalScore = null;
int solverBenchmarkCount = 0;
boolean firstSolverBenchmarkResult = true;
for (SolverBenchmarkResult solverBenchmarkResult : solverBenchmarkResultList) {
EnvironmentMode solverEnvironmentMode = solverBenchmarkResult.getEnvironmentMode();
var solverBenchmarkCount = 0;
var firstSolverBenchmarkResult = true;
for (var solverBenchmarkResult : solverBenchmarkResultList) {
var solverEnvironmentMode = solverBenchmarkResult.getEnvironmentMode();
if (firstSolverBenchmarkResult && solverEnvironmentMode != null) {
environmentMode = solverEnvironmentMode;
firstSolverBenchmarkResult = false;
} else if (!firstSolverBenchmarkResult && solverEnvironmentMode != environmentMode) {
environmentMode = null;
}

Score_ score = (Score_) solverBenchmarkResult.getAverageScore();
var score = (Score_) solverBenchmarkResult.getAverageScore();
if (score != null) {
ScoreDefinition<Score_> scoreDefinition = solverBenchmarkResult.getScoreDefinition();
var scoreDefinition = solverBenchmarkResult.getScoreDefinition();
if (totalScore != null && !scoreDefinition.isCompatibleArithmeticArgument(totalScore)) {
// Mixing different use cases with different score definitions.
totalScore = null;
Expand Down
22 changes: 19 additions & 3 deletions ...chmark/src/main/java/ai/timefold/solver/benchmark/impl/result/ProblemBenchmarkResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ public class ProblemBenchmarkResult<Solution_> {
private Long entityCount = null;
private Long variableCount = null;
private Long maximumValueCount = null;
private String formattedProblemScale = null;
/**
* Note: this is a fixed-point long of the problem scale log
*/
private Long problemScale = null;
private Long inputSolutionLoadingTimeMillisSpent = null;

Expand Down Expand Up @@ -177,6 +181,13 @@ public Long getMaximumValueCount() {
return maximumValueCount;
}

public String getFormattedProblemScale() {
return formattedProblemScale;
}

/**
* Returns the fixed-point long of the problem scale log
*/
public Long getProblemScale() {
return problemScale;
}
Expand Down Expand Up @@ -454,13 +465,15 @@ public void registerProblemSizeStatistics(ProblemSizeStatistics problemSizeStati
// The approximateValueCount is not unknown (null), but known to be ambiguous
maximumValueCount = -1L;
}
if (problemScale == null) {
if (formattedProblemScale == null) {
formattedProblemScale = problemSizeStatistics.approximateProblemScaleAsFormattedString();
problemScale = problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong();
} else if (problemScale.longValue() != problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong()) {
} else if (!Objects.equals(problemScale, problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong())) {
LOGGER.warn("The problemBenchmarkResult ({}) has different problemScale values ([{},{}]).\n"
+ "This is normally impossible for 1 inputSolutionFile.",
getName(), problemScale, problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong());
getName(), formattedProblemScale, problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong());
// The problemScale is not unknown (null), but known to be ambiguous
formattedProblemScale = "-1";
problemScale = -1L;
}
}
Expand Down Expand Up @@ -516,6 +529,7 @@ protected static <Solution_> Map<ProblemBenchmarkResult, ProblemBenchmarkResult>
newResult.entityCount = oldResult.entityCount;
newResult.variableCount = oldResult.variableCount;
newResult.maximumValueCount = oldResult.maximumValueCount;
newResult.formattedProblemScale = oldResult.formattedProblemScale;
newResult.problemScale = oldResult.problemScale;
problemProviderToNewResultMap.put(oldResult.problemProvider, newResult);
newPlannerBenchmarkResult.getUnifiedProblemBenchmarkResultList().add(newResult);
Expand All @@ -533,6 +547,8 @@ protected static <Solution_> Map<ProblemBenchmarkResult, ProblemBenchmarkResult>
newResult.variableCount = ConfigUtils.meldProperty(oldResult.variableCount, newResult.variableCount);
newResult.maximumValueCount = ConfigUtils.meldProperty(oldResult.maximumValueCount,
newResult.maximumValueCount);
newResult.formattedProblemScale =
ConfigUtils.meldProperty(oldResult.formattedProblemScale, newResult.formattedProblemScale);
newResult.problemScale = ConfigUtils.meldProperty(oldResult.problemScale, newResult.problemScale);
}
mergeMap.put(oldResult, newResult);
Expand Down
Loading
Loading