Skip to content

Commit a3f7862

Browse files
triceotriceo
authored
fix: Benchmarker best score graph no longer empty (#1636)
During the init score removal (cf87aa7), we did not notice that this impacts metrics as well. Benchmarker reads its data from metrics, and therefore lost the information on what score is or is not initialized. As a result, the default value was always false, resulting in empty score graph. This commit fixes that issue by registering a metric for the unassigned count, which will be reused by benchmarker to provide the correct initialization state.
1 parent 64a1115 commit a3f7862

36 files changed

Lines changed: 387 additions & 155 deletions

File tree

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,7 @@ protected static SubSingleBenchmarkResult createMerge(
304304
// Skip oldResult.usedMemoryAfterInputSolution
305305
newResult.succeeded = oldResult.succeeded;
306306
newResult.score = oldResult.score;
307+
newResult.initialized = oldResult.initialized;
307308
newResult.timeMillisSpent = oldResult.timeMillisSpent;
308309
newResult.scoreCalculationCount = oldResult.scoreCalculationCount;
309310
newResult.moveEvaluationCount = oldResult.moveEvaluationCount;

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java‎

Lines changed: 18 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@
1212
import java.util.function.ObjLongConsumer;
1313
import java.util.stream.Collectors;
1414

15-
import ai.timefold.solver.core.api.score.Score;
1615
import ai.timefold.solver.core.api.score.constraint.ConstraintRef;
1716
import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
1817
import ai.timefold.solver.core.impl.phase.event.PhaseLifecycleListener;
1918
import ai.timefold.solver.core.impl.phase.scope.AbstractPhaseScope;
2019
import ai.timefold.solver.core.impl.phase.scope.AbstractStepScope;
2120
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
21+
import ai.timefold.solver.core.impl.score.director.InnerScore;
22+
import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
2223
import ai.timefold.solver.core.impl.solver.scope.SolverScope;
2324

2425
import io.micrometer.core.instrument.Meter;
@@ -88,21 +89,18 @@ public Set<Meter.Id> getMeterIds(SolverMetric metric, Tags runId) {
8889
.collect(Collectors.toSet());
8990
}
9091

91-
public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer<Score<?>> scoreConsumer) {
92-
var labelNames = scoreDefinition.getLevelLabels();
93-
for (var i = 0; i < labelNames.length; i++) {
94-
labelNames[i] = labelNames[i].replace(' ', '.');
95-
}
96-
var levelNumbers = new Number[labelNames.length];
97-
for (var i = 0; i < labelNames.length; i++) {
98-
var scoreLevelGauge = this.find(metric.getMeterId() + "." + labelNames[i]).tags(runId).gauge();
92+
public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer<InnerScore<?>> scoreConsumer) {
93+
var score = SolverMetricUtil.extractScore(metric, scoreDefinition, id -> {
94+
var scoreLevelGauge = this.find(id).tags(runId).gauge();
9995
if (scoreLevelGauge != null && Double.isFinite(scoreLevelGauge.value())) {
100-
levelNumbers[i] = scoreLevelNumberConverter.apply(scoreLevelGauge.value());
96+
return scoreLevelNumberConverter.apply(scoreLevelGauge.value());
10197
} else {
102-
return;
98+
return null;
10399
}
100+
});
101+
if (score != null) {
102+
scoreConsumer.accept(score);
104103
}
105-
scoreConsumer.accept(scoreDefinition.fromLevelNumbers(levelNumbers));
106104
}
107105

108106
@SuppressWarnings({ "rawtypes", "unchecked" })
@@ -119,23 +117,17 @@ public void extractConstraintSummariesFromMeters(SolverMetric metric, Tags runId
119117
// Get the score from the corresponding constraint package and constraint name meters
120118
extractScoreFromMeters(metric, constraintMatchTotalRunId,
121119
// Get the count gauge (add constraint package and constraint name to the run tags)
122-
score -> getGaugeValue(metric.getMeterId() + ".count", constraintMatchTotalRunId,
123-
count -> constraintMatchTotalConsumer
124-
.accept(new ConstraintSummary(constraintRef, score, count.intValue()))));
120+
score -> {
121+
var count = SolverMetricUtil.getGaugeValue(this, SolverMetricUtil.getGaugeName(metric, "count"),
122+
constraintMatchTotalRunId);
123+
if (count != null) {
124+
constraintMatchTotalConsumer
125+
.accept(new ConstraintSummary(constraintRef, score.raw(), count.intValue()));
126+
}
127+
});
125128
});
126129
}
127130

128-
public void getGaugeValue(SolverMetric metric, Tags runId, Consumer<Number> gaugeConsumer) {
129-
getGaugeValue(metric.getMeterId(), runId, gaugeConsumer);
130-
}
131-
132-
public void getGaugeValue(String meterId, Tags runId, Consumer<Number> gaugeConsumer) {
133-
var gauge = this.find(meterId).tags(runId).gauge();
134-
if (gauge != null && Double.isFinite(gauge.value())) {
135-
gaugeConsumer.accept(gauge.value());
136-
}
137-
}
138-
139131
public void extractMoveCountPerType(SolverScope<Solution_> solverScope, ObjLongConsumer<String> gaugeConsumer) {
140132
solverScope.getMoveCountTypes().forEach(type -> {
141133
var gauge = this.find(SolverMetric.MOVE_COUNT_PER_TYPE.getMeterId() + "." + type)

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
3232
registry.addListener(SolverMetric.BEST_SCORE,
3333
timestamp -> registry.extractScoreFromMeters(SolverMetric.BEST_SCORE, runTag,
3434
score -> pointList
35-
.add(new BestScoreStatisticPoint(timestamp, score, subSingleBenchmarkResult.isInitialized()))));
35+
.add(new BestScoreStatisticPoint(timestamp, score.raw(), score.isFullyAssigned()))));
3636
}
3737

3838
// ************************************************************************

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestsolutionmutation/BestSolutionMutationSubSingleStatistic.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
1010
import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
1111
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
12+
import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
1213

1314
import io.micrometer.core.instrument.Tags;
1415

@@ -30,9 +31,12 @@ public BestSolutionMutationSubSingleStatistic(SubSingleBenchmarkResult subSingle
3031
@Override
3132
public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
3233
registry.addListener(SolverMetric.BEST_SOLUTION_MUTATION,
33-
timestamp -> registry.getGaugeValue(SolverMetric.BEST_SOLUTION_MUTATION, runTag,
34-
mutationCount -> pointList
35-
.add(new BestSolutionMutationStatisticPoint(timestamp, mutationCount.intValue()))));
34+
timestamp -> {
35+
var mutationCount = SolverMetricUtil.getGaugeValue(registry, SolverMetric.BEST_SOLUTION_MUTATION, runTag);
36+
if (mutationCount != null) {
37+
pointList.add(new BestSolutionMutationStatisticPoint(timestamp, mutationCount.intValue()));
38+
}
39+
});
3640
}
3741

3842
// ************************************************************************

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/common/AbstractCalculationSpeedSubSingleStatistic.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
1111
import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
1212
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
13+
import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
1314

1415
import io.micrometer.core.instrument.Tags;
1516

@@ -44,7 +45,8 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
4445
@Override
4546
public void accept(Long timeMillisSpent) {
4647
if (timeMillisSpent >= nextTimeMillisThreshold) {
47-
registry.getGaugeValue(solverMetric, runTag, countNumber -> {
48+
var countNumber = SolverMetricUtil.getGaugeValue(registry, solverMetric, runTag);
49+
if (countNumber != null) {
4850
var moveEvaluationCount = countNumber.longValue();
4951
var countInterval = moveEvaluationCount - lastCalculationCount.get();
5052
var timeMillisSpentInterval = timeMillisSpent - lastTimeMillisSpent;
@@ -55,7 +57,7 @@ public void accept(Long timeMillisSpent) {
5557
var speed = countInterval * 1000L / timeMillisSpentInterval;
5658
pointList.add(new LongStatisticPoint(timeMillisSpent, speed));
5759
lastCalculationCount.set(moveEvaluationCount);
58-
});
60+
}
5961
lastTimeMillisSpent = timeMillisSpent;
6062
nextTimeMillisThreshold += timeMillisThresholdInterval;
6163
if (nextTimeMillisThreshold < timeMillisSpent) {

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/memoryuse/MemoryUseSubSingleStatistic.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
1111
import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
1212
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
13+
import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
1314

1415
import io.micrometer.core.instrument.Tags;
1516

@@ -58,10 +59,11 @@ public MemoryUseSubSingleStatisticListener(StatisticRegistry<?> registry, Tags t
5859
@Override
5960
public void accept(Long timeMillisSpent) {
6061
if (timeMillisSpent >= nextTimeMillisThreshold) {
61-
registry.getGaugeValue(SolverMetric.MEMORY_USE, tags,
62-
memoryUse -> pointList.add(
63-
new MemoryUseStatisticPoint(timeMillisSpent, memoryUse.longValue(),
64-
(long) registry.find("jvm.memory.max").tags(tags).gauge().value())));
62+
var memoryUse = SolverMetricUtil.getGaugeValue(registry, SolverMetric.MEMORY_USE, tags);
63+
if (memoryUse != null) {
64+
var max = SolverMetricUtil.getGaugeValue(registry, "jvm.memory.max", tags);
65+
pointList.add(new MemoryUseStatisticPoint(timeMillisSpent, memoryUse.longValue(), max.longValue()));
66+
}
6567

6668
nextTimeMillisThreshold += timeMillisThresholdInterval;
6769
if (nextTimeMillisThreshold < timeMillisSpent) {

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/movecountperstep/MoveCountPerStepSubSingleStatistic.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
1010
import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
1111
import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
12+
import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
1213

1314
import io.micrometer.core.instrument.Tags;
1415

@@ -30,10 +31,18 @@ public MoveCountPerStepSubSingleStatistic(SubSingleBenchmarkResult subSingleBenc
3031
@Override
3132
public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
3233
registry.addListener(SolverMetric.MOVE_COUNT_PER_STEP,
33-
timeMillisSpent -> registry.getGaugeValue(SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".accepted", runTag,
34-
accepted -> registry.getGaugeValue(SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".selected", runTag,
35-
selected -> pointList.add(new MoveCountPerStepStatisticPoint(timeMillisSpent,
36-
accepted.longValue(), selected.longValue())))));
34+
timeMillisSpent -> {
35+
var accepted = SolverMetricUtil.getGaugeValue(registry,
36+
SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".accepted", runTag);
37+
if (accepted != null) {
38+
var selected = SolverMetricUtil.getGaugeValue(registry,
39+
SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".selected", runTag);
40+
if (selected != null) {
41+
pointList.add(new MoveCountPerStepStatisticPoint(timeMillisSpent, accepted.longValue(),
42+
selected.longValue()));
43+
}
44+
}
45+
});
3746
}
3847

3948
// ************************************************************************

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ public StepScoreSubSingleStatistic(SubSingleBenchmarkResult subSingleBenchmarkRe
3131
public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
3232
registry.addListener(SolverMetric.STEP_SCORE,
3333
timeMillisSpent -> registry.extractScoreFromMeters(SolverMetric.STEP_SCORE, runTag,
34-
score -> pointList.add(new StepScoreStatisticPoint(timeMillisSpent, score,
35-
subSingleBenchmarkResult.isInitialized()))));
34+
score -> pointList
35+
.add(new StepScoreStatisticPoint(timeMillisSpent, score.raw(), score.isFullyAssigned()))));
3636
}
3737

3838
// ************************************************************************

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypebestscore/PickedMoveTypeBestScoreDiffSubSingleStatistic.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
3535
registry.extractScoreFromMeters(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF,
3636
runTag.and(Tag.of("move.type", moveType)),
3737
score -> pointList.add(new PickedMoveTypeBestScoreDiffStatisticPoint(
38-
timeMillisSpent, moveType, score)));
38+
timeMillisSpent, moveType, score.raw())));
3939
}
4040
});
4141
}

‎benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypestepscore/PickedMoveTypeStepScoreDiffSubSingleStatistic.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
3535
registry.extractScoreFromMeters(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF,
3636
runTag.and(Tag.of("move.type", moveType)),
3737
score -> pointList.add(new PickedMoveTypeStepScoreDiffStatisticPoint(
38-
timeMillisSpent, moveType, score)));
38+
timeMillisSpent, moveType, score.raw())));
3939
}
4040
});
4141
}

0 commit comments

Comments
 (0)