fix: Benchmarker best score graph no longer empty

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java

@@ -304,6 +304,7 @@ protected static SubSingleBenchmarkResult createMerge(
         // Skip oldResult.usedMemoryAfterInputSolution
         newResult.succeeded = oldResult.succeeded;
         newResult.score = oldResult.score;
+        newResult.initialized = oldResult.initialized;
         newResult.timeMillisSpent = oldResult.timeMillisSpent;
         newResult.scoreCalculationCount = oldResult.scoreCalculationCount;
         newResult.moveEvaluationCount = oldResult.moveEvaluationCount;

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java

@@ -12,13 +12,14 @@
 import java.util.function.ObjLongConsumer;
 import java.util.stream.Collectors;
 
-import ai.timefold.solver.core.api.score.Score;
 import ai.timefold.solver.core.api.score.constraint.ConstraintRef;
 import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
 import ai.timefold.solver.core.impl.phase.event.PhaseLifecycleListener;
 import ai.timefold.solver.core.impl.phase.scope.AbstractPhaseScope;
 import ai.timefold.solver.core.impl.phase.scope.AbstractStepScope;
 import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
+import ai.timefold.solver.core.impl.score.director.InnerScore;
+import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
 import ai.timefold.solver.core.impl.solver.scope.SolverScope;
 
 import io.micrometer.core.instrument.Meter;
@@ -88,21 +89,18 @@ public Set<Meter.Id> getMeterIds(SolverMetric metric, Tags runId) {
                 .collect(Collectors.toSet());
     }
 
-    public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer<Score<?>> scoreConsumer) {
-        var labelNames = scoreDefinition.getLevelLabels();
-        for (var i = 0; i < labelNames.length; i++) {
-            labelNames[i] = labelNames[i].replace(' ', '.');
-        }
-        var levelNumbers = new Number[labelNames.length];
-        for (var i = 0; i < labelNames.length; i++) {
-            var scoreLevelGauge = this.find(metric.getMeterId() + "." + labelNames[i]).tags(runId).gauge();
+    public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer<InnerScore<?>> scoreConsumer) {
+        var score = SolverMetricUtil.extractScore(metric, scoreDefinition, id -> {
+            var scoreLevelGauge = this.find(id).tags(runId).gauge();
             if (scoreLevelGauge != null && Double.isFinite(scoreLevelGauge.value())) {
-                levelNumbers[i] = scoreLevelNumberConverter.apply(scoreLevelGauge.value());
+                return scoreLevelNumberConverter.apply(scoreLevelGauge.value());
             } else {
-                return;
+                return null;
             }
+        });
+        if (score != null) {
+            scoreConsumer.accept(score);
         }
-        scoreConsumer.accept(scoreDefinition.fromLevelNumbers(levelNumbers));
     }
 
     @SuppressWarnings({ "rawtypes", "unchecked" })
@@ -119,23 +117,17 @@ public void extractConstraintSummariesFromMeters(SolverMetric metric, Tags runId
                     // Get the score from the corresponding constraint package and constraint name meters
                     extractScoreFromMeters(metric, constraintMatchTotalRunId,
                             // Get the count gauge (add constraint package and constraint name to the run tags)
-                            score -> getGaugeValue(metric.getMeterId() + ".count", constraintMatchTotalRunId,
-                                    count -> constraintMatchTotalConsumer
-                                            .accept(new ConstraintSummary(constraintRef, score, count.intValue()))));
+                            score -> {
+                                var count = SolverMetricUtil.getGaugeValue(this, SolverMetricUtil.getGaugeName(metric, "count"),
+                                        constraintMatchTotalRunId);
+                                if (count != null) {
+                                    constraintMatchTotalConsumer
+                                            .accept(new ConstraintSummary(constraintRef, score.raw(), count.intValue()));
+                                }
+                            });
                 });
     }
 
-    public void getGaugeValue(SolverMetric metric, Tags runId, Consumer<Number> gaugeConsumer) {
-        getGaugeValue(metric.getMeterId(), runId, gaugeConsumer);
-    }
-
-    public void getGaugeValue(String meterId, Tags runId, Consumer<Number> gaugeConsumer) {
-        var gauge = this.find(meterId).tags(runId).gauge();
-        if (gauge != null && Double.isFinite(gauge.value())) {
-            gaugeConsumer.accept(gauge.value());
-        }
-    }
-
     public void extractMoveCountPerType(SolverScope<Solution_> solverScope, ObjLongConsumer<String> gaugeConsumer) {
         solverScope.getMoveCountTypes().forEach(type -> {
             var gauge = this.find(SolverMetric.MOVE_COUNT_PER_TYPE.getMeterId() + "." + type)

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java

@@ -32,7 +32,7 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
         registry.addListener(SolverMetric.BEST_SCORE,
                 timestamp -> registry.extractScoreFromMeters(SolverMetric.BEST_SCORE, runTag,
                         score -> pointList
-                                .add(new BestScoreStatisticPoint(timestamp, score, subSingleBenchmarkResult.isInitialized()))));
+                                .add(new BestScoreStatisticPoint(timestamp, score.raw(), score.isFullyAssigned()))));
     }
 
     // ************************************************************************

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestsolutionmutation/BestSolutionMutationSubSingleStatistic.java

@@ -9,6 +9,7 @@
 import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
 import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
 import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
+import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
 
 import io.micrometer.core.instrument.Tags;
 
@@ -30,9 +31,12 @@ public BestSolutionMutationSubSingleStatistic(SubSingleBenchmarkResult subSingle
     @Override
     public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
         registry.addListener(SolverMetric.BEST_SOLUTION_MUTATION,
-                timestamp -> registry.getGaugeValue(SolverMetric.BEST_SOLUTION_MUTATION, runTag,
-                        mutationCount -> pointList
-                                .add(new BestSolutionMutationStatisticPoint(timestamp, mutationCount.intValue()))));
+                timestamp -> {
+                    var mutationCount = SolverMetricUtil.getGaugeValue(registry, SolverMetric.BEST_SOLUTION_MUTATION, runTag);
+                    if (mutationCount != null) {
+                        pointList.add(new BestSolutionMutationStatisticPoint(timestamp, mutationCount.intValue()));
+                    }
+                });
     }
 
     // ************************************************************************

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/common/AbstractCalculationSpeedSubSingleStatistic.java

@@ -10,6 +10,7 @@
 import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
 import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
 import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
+import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
 
 import io.micrometer.core.instrument.Tags;
 
@@ -44,7 +45,8 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
             @Override
             public void accept(Long timeMillisSpent) {
                 if (timeMillisSpent >= nextTimeMillisThreshold) {
-                    registry.getGaugeValue(solverMetric, runTag, countNumber -> {
+                    var countNumber = SolverMetricUtil.getGaugeValue(registry, solverMetric, runTag);
+                    if (countNumber != null) {
                         var moveEvaluationCount = countNumber.longValue();
                         var countInterval = moveEvaluationCount - lastCalculationCount.get();
                         var timeMillisSpentInterval = timeMillisSpent - lastTimeMillisSpent;
@@ -55,7 +57,7 @@ public void accept(Long timeMillisSpent) {
                         var speed = countInterval * 1000L / timeMillisSpentInterval;
                         pointList.add(new LongStatisticPoint(timeMillisSpent, speed));
                         lastCalculationCount.set(moveEvaluationCount);
-                    });
+                    }
                     lastTimeMillisSpent = timeMillisSpent;
                     nextTimeMillisThreshold += timeMillisThresholdInterval;
                     if (nextTimeMillisThreshold < timeMillisSpent) {

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/memoryuse/MemoryUseSubSingleStatistic.java

@@ -10,6 +10,7 @@
 import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
 import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
 import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
+import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
 
 import io.micrometer.core.instrument.Tags;
 
@@ -58,10 +59,11 @@ public MemoryUseSubSingleStatisticListener(StatisticRegistry<?> registry, Tags t
         @Override
         public void accept(Long timeMillisSpent) {
             if (timeMillisSpent >= nextTimeMillisThreshold) {
-                registry.getGaugeValue(SolverMetric.MEMORY_USE, tags,
-                        memoryUse -> pointList.add(
-                                new MemoryUseStatisticPoint(timeMillisSpent, memoryUse.longValue(),
-                                        (long) registry.find("jvm.memory.max").tags(tags).gauge().value())));
+                var memoryUse = SolverMetricUtil.getGaugeValue(registry, SolverMetric.MEMORY_USE, tags);
+                if (memoryUse != null) {
+                    var max = SolverMetricUtil.getGaugeValue(registry, "jvm.memory.max", tags);
+                    pointList.add(new MemoryUseStatisticPoint(timeMillisSpent, memoryUse.longValue(), max.longValue()));
+                }
 
                 nextTimeMillisThreshold += timeMillisThresholdInterval;
                 if (nextTimeMillisThreshold < timeMillisSpent) {

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/movecountperstep/MoveCountPerStepSubSingleStatistic.java

@@ -9,6 +9,7 @@
 import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry;
 import ai.timefold.solver.core.config.solver.monitoring.SolverMetric;
 import ai.timefold.solver.core.impl.score.definition.ScoreDefinition;
+import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil;
 
 import io.micrometer.core.instrument.Tags;
 
@@ -30,10 +31,18 @@ public MoveCountPerStepSubSingleStatistic(SubSingleBenchmarkResult subSingleBenc
     @Override
     public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
         registry.addListener(SolverMetric.MOVE_COUNT_PER_STEP,
-                timeMillisSpent -> registry.getGaugeValue(SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".accepted", runTag,
-                        accepted -> registry.getGaugeValue(SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".selected", runTag,
-                                selected -> pointList.add(new MoveCountPerStepStatisticPoint(timeMillisSpent,
-                                        accepted.longValue(), selected.longValue())))));
+                timeMillisSpent -> {
+                    var accepted = SolverMetricUtil.getGaugeValue(registry,
+                            SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".accepted", runTag);
+                    if (accepted != null) {
+                        var selected = SolverMetricUtil.getGaugeValue(registry,
+                                SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".selected", runTag);
+                        if (selected != null) {
+                            pointList.add(new MoveCountPerStepStatisticPoint(timeMillisSpent, accepted.longValue(),
+                                    selected.longValue()));
+                        }
+                    }
+                });
     }
 
     // ************************************************************************

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java

@@ -31,8 +31,8 @@ public StepScoreSubSingleStatistic(SubSingleBenchmarkResult subSingleBenchmarkRe
     public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
         registry.addListener(SolverMetric.STEP_SCORE,
                 timeMillisSpent -> registry.extractScoreFromMeters(SolverMetric.STEP_SCORE, runTag,
-                        score -> pointList.add(new StepScoreStatisticPoint(timeMillisSpent, score,
-                                subSingleBenchmarkResult.isInitialized()))));
+                        score -> pointList
+                                .add(new StepScoreStatisticPoint(timeMillisSpent, score.raw(), score.isFullyAssigned()))));
     }
 
     // ************************************************************************

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypebestscore/PickedMoveTypeBestScoreDiffSubSingleStatistic.java

@@ -35,7 +35,7 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
                 registry.extractScoreFromMeters(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF,
                         runTag.and(Tag.of("move.type", moveType)),
                         score -> pointList.add(new PickedMoveTypeBestScoreDiffStatisticPoint(
-                                timeMillisSpent, moveType, score)));
+                                timeMillisSpent, moveType, score.raw())));
             }
         });
     }

benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypestepscore/PickedMoveTypeStepScoreDiffSubSingleStatistic.java

@@ -35,7 +35,7 @@ public void open(StatisticRegistry<Solution_> registry, Tags runTag) {
                 registry.extractScoreFromMeters(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF,
                         runTag.and(Tag.of("move.type", moveType)),
                         score -> pointList.add(new PickedMoveTypeStepScoreDiffStatisticPoint(
-                                timeMillisSpent, moveType, score)));
+                                timeMillisSpent, moveType, score.raw())));
             }
         });
     }

benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java

@@ -0,0 +1,81 @@
+package ai.timefold.solver.benchmark.api;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+import static org.assertj.core.api.SoftAssertions.assertSoftly;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+
+import ai.timefold.solver.benchmark.config.PlannerBenchmarkConfig;
+import ai.timefold.solver.benchmark.config.SolverBenchmarkConfig;
+import ai.timefold.solver.benchmark.impl.DefaultPlannerBenchmark;
+import ai.timefold.solver.core.config.solver.SolverConfig;
+import ai.timefold.solver.core.config.solver.termination.TerminationConfig;
+import ai.timefold.solver.core.testdomain.TestdataConstraintProvider;
+import ai.timefold.solver.core.testdomain.TestdataEntity;
+import ai.timefold.solver.core.testdomain.TestdataSolution;
+import ai.timefold.solver.core.testdomain.TestdataValue;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class PlannerBenchmarkTest {
+
+    @Test
+    void runPlannerBenchmark(@TempDir Path benchmarkTestDir) {
+        var benchmarkConfig = new PlannerBenchmarkConfig();
+        benchmarkConfig.setBenchmarkDirectory(benchmarkTestDir.toFile());
+        benchmarkConfig.setWarmUpMillisecondsSpentLimit(1L); // Minimize warmup.
+        var inheritedSolverConfig = new SolverBenchmarkConfig();
+        inheritedSolverConfig.setSolverConfig(new SolverConfig()
+                .withSolutionClass(TestdataSolution.class)
+                .withEntityClasses(TestdataEntity.class)
+                .withConstraintProviderClass(TestdataConstraintProvider.class)
+                // Only run for a short amount of time.
+                .withTerminationConfig(new TerminationConfig().withUnimprovedMillisecondsSpentLimit(100L)));
+        benchmarkConfig.setInheritedSolverBenchmarkConfig(inheritedSolverConfig);
+        benchmarkConfig.setSolverBenchmarkConfigList(List.of(new SolverBenchmarkConfig()));
+        var benchmarkFactory = PlannerBenchmarkFactory.create(benchmarkConfig);
+
+        var solution1 = new TestdataSolution("s1");
+        solution1.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3")));
+        solution1.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2")));
+
+        var plannerBenchmark = (DefaultPlannerBenchmark) benchmarkFactory.buildPlannerBenchmark(solution1);
+        plannerBenchmark.benchmark(); // Run the benchmark.
+        var folder = plannerBenchmark.getBenchmarkReport().getHtmlOverviewFile()
+                .toPath()
+                .getParent();
+        var csv = folder.resolve(Path.of("Problem_0", "Config_0", "sub0", "BEST_SCORE.csv"));
+        assertThat(csv).exists();
+
+        try (var lines = Files.lines(csv)) {
+            var lineList = lines.toList();
+            assertThat(lineList).hasSizeGreaterThan(1);
+            assertSoftly(softly -> {
+                // Proper header.
+                softly.assertThat(lineList)
+                        .first()
+                        .isEqualTo("""
+                                "timeMillisSpent","score","initialized"
+                                """.trim());
+                // Checks that best score was recorded at least once.
+                // Requires LS to have started, as CH does not store best score.
+                // We only check score+initialized, as "timeMillisSpent" can be anything.
+                softly.assertThat(lineList)
+                        .last()
+                        .asString()
+                        .endsWith("""
+                                "-3","true"
+                                """.trim());
+            });
+        } catch (IOException e) {
+            fail(e);
+        }
+    }
+
+}

core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java

@@ -15,6 +15,7 @@
  *
  * @param <Solution_> the solution type, the class with the {@link PlanningSolution} annotation
  */
+// TODO In Solver 2.0, maybe convert this to an interface.
 public class BestSolutionChangedEvent<Solution_> extends EventObject {
 
     private final Solver<Solution_> solver;
@@ -25,7 +26,7 @@ public class BestSolutionChangedEvent<Solution_> extends EventObject {
 
     /**
      * @param timeMillisSpent {@code >= 0L}
-     * @deprecated Use {@link #BestSolutionChangedEvent(Solver, long, Object, Score, boolean)} instead.
+     * @deprecated Users should not manually construct instances of this event.
      */
     @Deprecated(forRemoval = true, since = "1.22.0")
     public BestSolutionChangedEvent(@NonNull Solver<Solution_> solver, long timeMillisSpent,
@@ -35,7 +36,9 @@ public BestSolutionChangedEvent(@NonNull Solver<Solution_> solver, long timeMill
 
     /**
      * @param timeMillisSpent {@code >= 0L}
+     * @deprecated Users should not manually construct instances of this event.
      */
+    @Deprecated(forRemoval = true, since = "1.23.0")
     public BestSolutionChangedEvent(@NonNull Solver<Solution_> solver, long timeMillisSpent,
             @NonNull Solution_ newBestSolution, @NonNull Score newBestScore,
             boolean isNewBestSolutionInitialized) {