Skip to content

Commit caf43ea

Browse files
Christopher-ChianelliChristopher-Chianelli
authored
fix(benchmarker): use approximate problem scale instead of log in the report (#2336)
1 parent e52f30f commit caf43ea

7 files changed

Lines changed: 102 additions & 83 deletions

File tree

‎core/src/main/java/ai/timefold/solver/core/api/solver/ProblemSizeStatistics.java‎

Lines changed: 1 addition & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package ai.timefold.solver.core.api.solver;
22

3-
import java.text.DecimalFormat;
4-
import java.text.DecimalFormatSymbols;
53
import java.util.Locale;
64

75
import ai.timefold.solver.core.impl.util.MathUtils;
@@ -23,13 +21,6 @@ public record ProblemSizeStatistics(long entityCount,
2321
long approximateValueCount,
2422
double approximateProblemSizeLog) {
2523

26-
private static final Locale FORMATTER_LOCALE = Locale.getDefault();
27-
private static final DecimalFormat BASIC_FORMATTER = new DecimalFormat("#,###");
28-
29-
// Exponent should not use grouping, unlike basic
30-
private static final DecimalFormat EXPONENT_FORMATTER = new DecimalFormat("#");
31-
private static final DecimalFormat SIGNIFICANT_FIGURE_FORMATTER = new DecimalFormat("0.######");
32-
3324
/**
3425
* Return the {@link #approximateProblemSizeLog} as a fixed point integer.
3526
*/
@@ -38,48 +29,7 @@ public long approximateProblemScaleLogAsFixedPointLong() {
3829
}
3930

4031
public String approximateProblemScaleAsFormattedString() {
41-
return approximateProblemScaleAsFormattedString(Locale.getDefault());
42-
}
43-
44-
String approximateProblemScaleAsFormattedString(Locale locale) {
45-
if (Double.isNaN(approximateProblemSizeLog) || Double.isInfinite(approximateProblemSizeLog)) {
46-
return "0";
47-
}
48-
49-
if (approximateProblemSizeLog < 10) { // log_10(10_000_000_000) = 10
50-
return "%s".formatted(format(Math.pow(10d, approximateProblemSizeLog), BASIC_FORMATTER, locale));
51-
}
52-
// The actual number will often be too large to fit in a double, so cannot use basic
53-
// formatting.
54-
// Separate the exponent into its integral and fractional parts
55-
// Use the integral part as the power of 10, and the fractional part as the significant digits.
56-
double exponentPart = Math.floor(approximateProblemSizeLog);
57-
double remainderPartAsExponent = approximateProblemSizeLog - exponentPart;
58-
double remainderPart = Math.pow(10, remainderPartAsExponent);
59-
return "%s × 10^%s".formatted(
60-
format(remainderPart, SIGNIFICANT_FIGURE_FORMATTER, locale),
61-
format(exponentPart, EXPONENT_FORMATTER, locale));
62-
}
63-
64-
/**
65-
* In order for tests to work currently regardless of the default system locale,
66-
* we need to set the locale to a known value before running the tests.
67-
* And because the {@link DecimalFormat} instances are initialized statically for reasons of performance,
68-
* we cannot expect them to be in the locale that the test expects them to be in.
69-
* This method exists to allow for an override.
70-
*
71-
* @return the given decimalFormat with the given locale
72-
*/
73-
private static String format(double number, DecimalFormat decimalFormat, Locale locale) {
74-
if (locale.equals(FORMATTER_LOCALE)) {
75-
return decimalFormat.format(number);
76-
}
77-
try { // Slow path for corner cases where input locale doesn't match the default locale.
78-
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(locale));
79-
return decimalFormat.format(number);
80-
} finally {
81-
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(FORMATTER_LOCALE));
82-
}
32+
return MathUtils.approximateProblemScaleAsFormattedString(approximateProblemSizeLog, Locale.getDefault());
8333
}
8434

8535
}

‎core/src/main/java/ai/timefold/solver/core/impl/util/MathUtils.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
package ai.timefold.solver.core.impl.util;
22

3+
import java.text.DecimalFormat;
4+
import java.text.DecimalFormatSymbols;
5+
import java.util.Locale;
6+
37
/**
48
* Includes code taken from Apache's commons-math library, v3.6.1 (Apache 2.0-licensed)
59
* These methods are clearly marked in comments, their modifications listed.
@@ -9,6 +13,11 @@ public class MathUtils {
913
public static final long LOG_PRECISION = 1_000_000L;
1014
private static final int FACTORIAL_MAX_N = 20;
1115
private static final long[] FACTORIAL_CACHE = new long[FACTORIAL_MAX_N - 1]; // 0 and 1 are hard-coded.
16+
public static final Locale FORMATTER_LOCALE = Locale.getDefault();
17+
private static final DecimalFormat BASIC_FORMATTER = new DecimalFormat("#,###");
18+
// Exponent should not use grouping, unlike basic
19+
private static final DecimalFormat EXPONENT_FORMATTER = new DecimalFormat("#");
20+
private static final DecimalFormat SIGNIFICANT_FIGURE_FORMATTER = new DecimalFormat("0.######");
1221

1322
private MathUtils() {
1423
}
@@ -346,4 +355,44 @@ private static long mulAndCheck(long a, long b) {
346355
}
347356
}
348357

358+
public static String approximateProblemScaleAsFormattedString(double approximateProblemSizeLog, Locale locale) {
359+
if (Double.isNaN(approximateProblemSizeLog) || Double.isInfinite(approximateProblemSizeLog)) {
360+
return "0";
361+
}
362+
363+
if (approximateProblemSizeLog < 10) { // log_10(10_000_000_000) = 10
364+
return "%s".formatted(formatNumber(Math.pow(10d, approximateProblemSizeLog), BASIC_FORMATTER, locale));
365+
}
366+
// The actual number will often be too large to fit in a double, so cannot use basic
367+
// formatting.
368+
// Separate the exponent into its integral and fractional parts
369+
// Use the integral part as the power of 10, and the fractional part as the significant digits.
370+
double exponentPart = Math.floor(approximateProblemSizeLog);
371+
double remainderPartAsExponent = approximateProblemSizeLog - exponentPart;
372+
double remainderPart = Math.pow(10, remainderPartAsExponent);
373+
return "%s × 10^%s".formatted(
374+
formatNumber(remainderPart, SIGNIFICANT_FIGURE_FORMATTER, locale),
375+
formatNumber(exponentPart, EXPONENT_FORMATTER, locale));
376+
}
377+
378+
/**
379+
* In order for tests to work currently regardless of the default system locale,
380+
* we need to set the locale to a known value before running the tests.
381+
* And because the {@link DecimalFormat} instances are initialized statically for reasons of performance,
382+
* we cannot expect them to be in the locale that the test expects them to be in.
383+
* This method exists to allow for an override.
384+
*
385+
* @return the given decimalFormat with the given locale
386+
*/
387+
private static String formatNumber(double number, DecimalFormat decimalFormat, Locale locale) {
388+
if (locale.equals(MathUtils.FORMATTER_LOCALE)) {
389+
return decimalFormat.format(number);
390+
}
391+
try { // Slow path for corner cases where input locale doesn't match the default locale.
392+
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(locale));
393+
return decimalFormat.format(number);
394+
} finally {
395+
decimalFormat.setDecimalFormatSymbols(DecimalFormatSymbols.getInstance(MathUtils.FORMATTER_LOCALE));
396+
}
397+
}
349398
}

‎tools/benchmark-aggregator/src/main/java/ai/timefold/solver/benchmark/aggregator/swingui/BenchmarkAggregatorFrame.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -518,12 +518,13 @@ private MixedCheckBox createSolverBenchmarkCheckBox(SolverBenchmarkResult solver
518518
}
519519

520520
private MixedCheckBox createProblemBenchmarkCheckBox(ProblemBenchmarkResult problemBenchmarkResult) {
521-
String problemBenchmarkDetail = String.format(
522-
"Entity count: %d%n"
523-
+ "Problem scale: %d%n"
524-
+ "Used memory: %s",
521+
var problemBenchmarkDetail = String.format(
522+
"""
523+
Entity count: %d%n\
524+
Problem scale: %s%n\
525+
Used memory: %s""",
525526
problemBenchmarkResult.getEntityCount(),
526-
problemBenchmarkResult.getProblemScale(),
527+
problemBenchmarkResult.getFormattedProblemScale(),
527528
toEmptyStringIfNull(problemBenchmarkResult.getAverageUsedMemoryAfterInputSolution()));
528529
return new MixedCheckBox(problemBenchmarkResult.getName(), problemBenchmarkDetail);
529530
}

‎tools/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/report/BenchmarkReport.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -372,8 +372,8 @@ private List<LineChart<Long, Double>> createBestScoreScalabilitySummaryChart() {
372372
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
373373
for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) {
374374
if (singleBenchmarkResult.hasAllSuccess()) {
375-
long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
376-
double[] levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
375+
var problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
376+
var levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
377377
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
378378
if (i >= builderList.size()) {
379379
builderList.add(new LineChart.Builder<>());
@@ -556,8 +556,8 @@ private LineChart<Long, Long> createScalabilitySummaryChart(ToLongFunction<Singl
556556
.stream()
557557
.filter(SingleBenchmarkResult::hasAllSuccess)
558558
.forEach(singleBenchmarkResult -> {
559-
long problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
560-
long timeMillisSpent = valueFunction.applyAsLong(singleBenchmarkResult);
559+
var problemScale = singleBenchmarkResult.getProblemBenchmarkResult().getProblemScale();
560+
var timeMillisSpent = valueFunction.applyAsLong(singleBenchmarkResult);
561561
builder.add(solverLabel, problemScale, timeMillisSpent);
562562
});
563563
}
@@ -570,8 +570,8 @@ private List<LineChart<Long, Double>> createBestScorePerTimeSpentSummaryChart()
570570
String solverLabel = solverBenchmarkResult.getNameWithFavoriteSuffix();
571571
for (SingleBenchmarkResult singleBenchmarkResult : solverBenchmarkResult.getSingleBenchmarkResultList()) {
572572
if (singleBenchmarkResult.hasAllSuccess()) {
573-
long timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
574-
double[] levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
573+
var timeMillisSpent = singleBenchmarkResult.getTimeMillisSpent();
574+
var levelValues = singleBenchmarkResult.getAverageScore().toLevelDoubles();
575575
for (int i = 0; i < levelValues.length && i < CHARTED_SCORE_LEVEL_SIZE; i++) {
576576
if (i >= builderList.size()) {
577577
builderList.add(new LineChart.Builder<>());

‎tools/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/PlannerBenchmarkResult.java‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import java.util.Comparator;
1010
import java.util.IdentityHashMap;
1111
import java.util.List;
12+
import java.util.Locale;
1213
import java.util.Map;
1314
import java.util.SortedMap;
1415
import java.util.TreeMap;
@@ -23,7 +24,7 @@
2324
import ai.timefold.solver.core.config.solver.EnvironmentMode;
2425
import ai.timefold.solver.core.config.util.ConfigUtils;
2526
import ai.timefold.solver.core.enterprise.TimefoldSolverEnterpriseService;
26-
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
27+
import ai.timefold.solver.core.impl.util.MathUtils;
2728

2829
import org.slf4j.Logger;
2930
import org.slf4j.LoggerFactory;
@@ -67,7 +68,7 @@ public class PlannerBenchmarkResult {
6768
// ************************************************************************
6869

6970
private Integer failureCount = null;
70-
private Long averageProblemScale = null;
71+
private String averageProblemScale = null;
7172
private Score averageScore = null;
7273
private SolverBenchmarkResult favoriteSolverBenchmarkResult = null;
7374

@@ -184,7 +185,7 @@ public Integer getFailureCount() {
184185
return failureCount;
185186
}
186187

187-
public Long getAverageProblemScale() {
188+
public String getAverageProblemScale() {
188189
return averageProblemScale;
189190
}
190191

@@ -316,32 +317,34 @@ public void accumulateResults(BenchmarkReport benchmarkReport) {
316317

317318
private <Score_ extends Score<Score_>> void determineTotalsAndAverages() {
318319
failureCount = 0;
319-
long totalProblemScale = 0L;
320-
int problemScaleCount = 0;
321-
for (ProblemBenchmarkResult problemBenchmarkResult : unifiedProblemBenchmarkResultList) {
322-
Long problemScale = problemBenchmarkResult.getProblemScale();
320+
var totalProblemScale = 0L;
321+
var problemScaleCount = 0;
322+
for (var problemBenchmarkResult : unifiedProblemBenchmarkResultList) {
323+
var problemScale = problemBenchmarkResult.getProblemScale();
323324
if (problemScale != null && problemScale >= 0L) {
324325
totalProblemScale += problemScale;
325326
problemScaleCount++;
326327
}
327328
failureCount += problemBenchmarkResult.getFailureCount();
328329
}
329-
averageProblemScale = problemScaleCount == 0 ? null : totalProblemScale / problemScaleCount;
330+
averageProblemScale = problemScaleCount == 0 ? null
331+
: MathUtils.approximateProblemScaleAsFormattedString(
332+
(double) totalProblemScale / problemScaleCount / MathUtils.LOG_PRECISION, Locale.getDefault());
330333
Score_ totalScore = null;
331-
int solverBenchmarkCount = 0;
332-
boolean firstSolverBenchmarkResult = true;
333-
for (SolverBenchmarkResult solverBenchmarkResult : solverBenchmarkResultList) {
334-
EnvironmentMode solverEnvironmentMode = solverBenchmarkResult.getEnvironmentMode();
334+
var solverBenchmarkCount = 0;
335+
var firstSolverBenchmarkResult = true;
336+
for (var solverBenchmarkResult : solverBenchmarkResultList) {
337+
var solverEnvironmentMode = solverBenchmarkResult.getEnvironmentMode();
335338
if (firstSolverBenchmarkResult && solverEnvironmentMode != null) {
336339
environmentMode = solverEnvironmentMode;
337340
firstSolverBenchmarkResult = false;
338341
} else if (!firstSolverBenchmarkResult && solverEnvironmentMode != environmentMode) {
339342
environmentMode = null;
340343
}
341344

342-
Score_ score = (Score_) solverBenchmarkResult.getAverageScore();
345+
var score = (Score_) solverBenchmarkResult.getAverageScore();
343346
if (score != null) {
344-
ScoreDefinition<Score_> scoreDefinition = solverBenchmarkResult.getScoreDefinition();
347+
var scoreDefinition = solverBenchmarkResult.getScoreDefinition();
345348
if (totalScore != null && !scoreDefinition.isCompatibleArithmeticArgument(totalScore)) {
346349
// Mixing different use cases with different score definitions.
347350
totalScore = null;

‎tools/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/ProblemBenchmarkResult.java‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ public class ProblemBenchmarkResult<Solution_> {
8484
private Long entityCount = null;
8585
private Long variableCount = null;
8686
private Long maximumValueCount = null;
87+
private String formattedProblemScale = null;
88+
/**
89+
* Note: this is a fixed-point long of the problem scale log
90+
*/
8791
private Long problemScale = null;
8892
private Long inputSolutionLoadingTimeMillisSpent = null;
8993

@@ -177,6 +181,13 @@ public Long getMaximumValueCount() {
177181
return maximumValueCount;
178182
}
179183

184+
public String getFormattedProblemScale() {
185+
return formattedProblemScale;
186+
}
187+
188+
/**
189+
* Returns the fixed-point long of the problem scale log
190+
*/
180191
public Long getProblemScale() {
181192
return problemScale;
182193
}
@@ -454,13 +465,15 @@ public void registerProblemSizeStatistics(ProblemSizeStatistics problemSizeStati
454465
// The approximateValueCount is not unknown (null), but known to be ambiguous
455466
maximumValueCount = -1L;
456467
}
457-
if (problemScale == null) {
468+
if (formattedProblemScale == null) {
469+
formattedProblemScale = problemSizeStatistics.approximateProblemScaleAsFormattedString();
458470
problemScale = problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong();
459-
} else if (problemScale.longValue() != problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong()) {
471+
} else if (!Objects.equals(problemScale, problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong())) {
460472
LOGGER.warn("The problemBenchmarkResult ({}) has different problemScale values ([{},{}]).\n"
461473
+ "This is normally impossible for 1 inputSolutionFile.",
462-
getName(), problemScale, problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong());
474+
getName(), formattedProblemScale, problemSizeStatistics.approximateProblemScaleLogAsFixedPointLong());
463475
// The problemScale is not unknown (null), but known to be ambiguous
476+
formattedProblemScale = "-1";
464477
problemScale = -1L;
465478
}
466479
}
@@ -516,6 +529,7 @@ protected static <Solution_> Map<ProblemBenchmarkResult, ProblemBenchmarkResult>
516529
newResult.entityCount = oldResult.entityCount;
517530
newResult.variableCount = oldResult.variableCount;
518531
newResult.maximumValueCount = oldResult.maximumValueCount;
532+
newResult.formattedProblemScale = oldResult.formattedProblemScale;
519533
newResult.problemScale = oldResult.problemScale;
520534
problemProviderToNewResultMap.put(oldResult.problemProvider, newResult);
521535
newPlannerBenchmarkResult.getUnifiedProblemBenchmarkResultList().add(newResult);
@@ -533,6 +547,8 @@ protected static <Solution_> Map<ProblemBenchmarkResult, ProblemBenchmarkResult>
533547
newResult.variableCount = ConfigUtils.meldProperty(oldResult.variableCount, newResult.variableCount);
534548
newResult.maximumValueCount = ConfigUtils.meldProperty(oldResult.maximumValueCount,
535549
newResult.maximumValueCount);
550+
newResult.formattedProblemScale =
551+
ConfigUtils.meldProperty(oldResult.formattedProblemScale, newResult.formattedProblemScale);
536552
newResult.problemScale = ConfigUtils.meldProperty(oldResult.problemScale, newResult.problemScale);
537553
}
538554
mergeMap.put(oldResult, newResult);

0 commit comments

Comments
 (0)