From aabd7bff07544b00b4402da425ac50f8edaaaa59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 07:45:11 +0200 Subject: [PATCH 1/9] best solution event gets unassigned count --- .../event/BestSolutionChangedEvent.java | 4 +++- .../DefaultBestSolutionChangedEvent.java | 22 +++++++++++++++++++ .../impl/solver/event/SolverEventSupport.java | 4 +--- 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java diff --git a/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java b/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java index fc5589e1097..083a231bb8c 100644 --- a/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java +++ b/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java @@ -25,7 +25,7 @@ public class BestSolutionChangedEvent extends EventObject { /** * @param timeMillisSpent {@code >= 0L} - * @deprecated Use {@link #BestSolutionChangedEvent(Solver, long, Object, Score, boolean)} instead. + * @deprecated Users should not manually construct instances of this event. */ @Deprecated(forRemoval = true, since = "1.22.0") public BestSolutionChangedEvent(@NonNull Solver solver, long timeMillisSpent, @@ -35,7 +35,9 @@ public BestSolutionChangedEvent(@NonNull Solver solver, long timeMill /** * @param timeMillisSpent {@code >= 0L} + * @deprecated Users should not manually construct instances of this event. */ + @Deprecated(forRemoval = true, since = "1.23.0") public BestSolutionChangedEvent(@NonNull Solver solver, long timeMillisSpent, @NonNull Solution_ newBestSolution, @NonNull Score newBestScore, boolean isNewBestSolutionInitialized) { diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java new file mode 100644 index 00000000000..3e2096341be --- /dev/null +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java @@ -0,0 +1,22 @@ +package ai.timefold.solver.core.impl.solver.event; + +import ai.timefold.solver.core.api.solver.Solver; +import ai.timefold.solver.core.api.solver.event.BestSolutionChangedEvent; +import ai.timefold.solver.core.impl.score.director.InnerScore; + +import org.jspecify.annotations.NonNull; + +public final class DefaultBestSolutionChangedEvent extends BestSolutionChangedEvent { + + private final int unassignedCount; + + public DefaultBestSolutionChangedEvent(@NonNull Solver solver, long timeMillisSpent, @NonNull Solution_ newBestSolution, @NonNull InnerScore newBestScore) { + super(solver, timeMillisSpent, newBestSolution, newBestScore.raw(), newBestScore.fullyAssigned()); + this.unassignedCount = newBestScore.unassignedCount(); + } + + public int getUnassignedCount() { + return unassignedCount; + } + +} diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/SolverEventSupport.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/SolverEventSupport.java index a1357b1504b..591933a1b51 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/SolverEventSupport.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/SolverEventSupport.java @@ -2,7 +2,6 @@ import ai.timefold.solver.core.api.domain.solution.PlanningSolution; import ai.timefold.solver.core.api.solver.Solver; -import ai.timefold.solver.core.api.solver.event.BestSolutionChangedEvent; import ai.timefold.solver.core.api.solver.event.SolverEventListener; import ai.timefold.solver.core.impl.solver.scope.SolverScope; @@ -27,8 +26,7 @@ public void fireBestSolutionChanged(SolverScope solverScope, Solution // We need to clone the new best solution, or we may share the same instance with user consumers. // Reusing the instance can lead to inconsistent states if intermediary consumers change the solution. var newBestSolutionCloned = solverScope.getScoreDirector().cloneSolution(newBestSolution); - var event = new BestSolutionChangedEvent<>(solver, timeMillisSpent, newBestSolutionCloned, bestScore.raw(), - bestScore.fullyAssigned()); + var event = new DefaultBestSolutionChangedEvent<>(solver, timeMillisSpent, newBestSolutionCloned, bestScore); do { it.next().bestSolutionChanged(event); } while (it.hasNext()); From d8b1c6b62e4920e4a4cc3ea9aebb4773edc8bebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 09:06:46 +0200 Subject: [PATCH 2/9] track unassigned count as a gauge --- .../solver/monitoring/SolverMetric.java | 58 +++++-------------- .../localsearch/DefaultLocalSearchPhase.java | 20 ++++--- .../solver/core/impl/phase/AbstractPhase.java | 17 ++++-- .../DefaultBestSolutionChangedEvent.java | 3 +- .../impl/solver/monitoring/ScoreLevels.java | 33 +++++++++++ .../solver/monitoring/SolverMetricUtil.java | 49 ++++++++++++++++ .../statistic/BestScoreStatistic.java | 17 ++++-- .../BestSolutionMutationCountStatistic.java | 2 +- .../statistic/MemoryUseStatistic.java | 2 +- .../statistic/MoveCountPerTypeStatistic.java | 2 +- .../PickedMoveBestScoreDiffStatistic.java | 19 +++--- .../PickedMoveStepScoreDiffStatistic.java | 19 +++--- .../statistic/SolverScopeStatistic.java | 2 +- .../statistic/SolverStatistic.java | 2 +- .../statistic/StatelessSolverStatistic.java | 2 +- .../core/impl/solver/scope/SolverScope.java | 6 +- 16 files changed, 160 insertions(+), 93 deletions(-) create mode 100644 core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java create mode 100644 core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/BestScoreStatistic.java (65%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/BestSolutionMutationCountStatistic.java (97%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/MemoryUseStatistic.java (91%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/MoveCountPerTypeStatistic.java (98%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/PickedMoveBestScoreDiffStatistic.java (83%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/PickedMoveStepScoreDiffStatistic.java (83%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/SolverScopeStatistic.java (95%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/SolverStatistic.java (74%) rename core/src/main/java/ai/timefold/solver/core/impl/{ => solver/monitoring}/statistic/StatelessSolverStatistic.java (86%) diff --git a/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java b/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java index 54e7c46e2b4..6c1c2950b2f 100644 --- a/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java +++ b/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java @@ -1,35 +1,26 @@ package ai.timefold.solver.core.config.solver.monitoring; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.ToDoubleFunction; import jakarta.xml.bind.annotation.XmlEnum; -import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.api.solver.Solver; -import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.BestScoreStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.BestSolutionMutationCountStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.MemoryUseStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.MoveCountPerTypeStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.PickedMoveBestScoreDiffStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.PickedMoveStepScoreDiffStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.SolverScopeStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.SolverStatistic; +import ai.timefold.solver.core.impl.solver.monitoring.statistic.StatelessSolverStatistic; import ai.timefold.solver.core.impl.solver.scope.SolverScope; -import ai.timefold.solver.core.impl.statistic.BestScoreStatistic; -import ai.timefold.solver.core.impl.statistic.BestSolutionMutationCountStatistic; -import ai.timefold.solver.core.impl.statistic.MemoryUseStatistic; -import ai.timefold.solver.core.impl.statistic.MoveCountPerTypeStatistic; -import ai.timefold.solver.core.impl.statistic.PickedMoveBestScoreDiffStatistic; -import ai.timefold.solver.core.impl.statistic.PickedMoveStepScoreDiffStatistic; -import ai.timefold.solver.core.impl.statistic.SolverScopeStatistic; -import ai.timefold.solver.core.impl.statistic.SolverStatistic; -import ai.timefold.solver.core.impl.statistic.StatelessSolverStatistic; import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.NullMarked; - -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.Tags; @XmlEnum public enum SolverMetric { + SOLVE_DURATION("timefold.solver.solve.duration", false), ERROR_COUNT("timefold.solver.errors", false), SCORE_CALCULATION_COUNT("timefold.solver.score.calculation.count", @@ -63,6 +54,9 @@ public enum SolverMetric { PICKED_MOVE_TYPE_STEP_SCORE_DIFF("timefold.solver.move.type.step.score.diff", new PickedMoveStepScoreDiffStatistic<>(), false); + // Necessary for benchmarker, but otherwise undocumented and not considered public. + public static final String UNASSIGNED_COUNT = "unassigned.count"; + private final String meterId; @SuppressWarnings("rawtypes") private final SolverStatistic registerFunction; @@ -97,30 +91,6 @@ public enum SolverMetric { return meterId; } - @NullMarked - public static void registerScoreMetrics(SolverMetric metric, Tags tags, ScoreDefinition scoreDefinition, - Map>> tagToScoreLevels, Score innerScore) { - Number[] levelValues = innerScore.toLevelNumbers(); - if (tagToScoreLevels.containsKey(tags)) { - List> scoreLevels = tagToScoreLevels.get(tags); - for (int i = 0; i < levelValues.length; i++) { - scoreLevels.get(i).set(levelValues[i]); - } - } else { - String[] levelLabels = scoreDefinition.getLevelLabels(); - for (int i = 0; i < levelLabels.length; i++) { - levelLabels[i] = levelLabels[i].replace(' ', '.'); - } - List> scoreLevels = new ArrayList<>(levelValues.length); - for (int i = 0; i < levelValues.length; i++) { - scoreLevels.add(Metrics.gauge(metric.getMeterId() + "." + levelLabels[i], - tags, new AtomicReference<>(levelValues[i]), - ar -> ar.get().doubleValue())); - } - tagToScoreLevels.put(tags, scoreLevels); - } - } - public boolean isMetricBestSolutionBased() { return isBestSolutionBased; } @@ -130,7 +100,6 @@ public boolean isMetricConstraintMatchBased() { } @SuppressWarnings("unchecked") - // TODO: clarify @NonNull ok here public void register(@NonNull Solver solver) { registerFunction.register(solver); } @@ -139,4 +108,5 @@ public void register(@NonNull Solver solver) { public void unregister(@NonNull Solver solver) { registerFunction.unregister(solver); } + } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java b/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java index b5c97a015d2..930cad42b4e 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java @@ -1,12 +1,11 @@ package ai.timefold.solver.core.impl.localsearch; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import ai.timefold.solver.core.api.domain.solution.PlanningSolution; +import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.api.score.constraint.ConstraintMatchTotal; import ai.timefold.solver.core.config.solver.EnvironmentMode; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; @@ -16,6 +15,9 @@ import ai.timefold.solver.core.impl.localsearch.scope.LocalSearchStepScope; import ai.timefold.solver.core.impl.phase.AbstractPhase; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.score.director.InnerScore; +import ai.timefold.solver.core.impl.solver.monitoring.ScoreLevels; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import ai.timefold.solver.core.impl.solver.scope.SolverScope; import ai.timefold.solver.core.impl.solver.termination.PhaseTermination; @@ -35,8 +37,8 @@ public class DefaultLocalSearchPhase extends AbstractPhase protected final AtomicLong selectedMoveCountPerStep = new AtomicLong(0); protected final Map constraintMatchTotalTagsToStepCount = new ConcurrentHashMap<>(); protected final Map constraintMatchTotalTagsToBestCount = new ConcurrentHashMap<>(); - protected final Map>> constraintMatchTotalStepScoreMap = new ConcurrentHashMap<>(); - protected final Map>> constraintMatchTotalBestScoreMap = new ConcurrentHashMap<>(); + protected final Map constraintMatchTotalStepScoreMap = new ConcurrentHashMap<>(); + protected final Map constraintMatchTotalBestScoreMap = new ConcurrentHashMap<>(); private DefaultLocalSearchPhase(Builder builder) { super(builder); @@ -181,9 +183,10 @@ private void collectMetrics(LocalSearchStepScope stepScope) { } } - private void collectConstraintMatchTotalMetrics(SolverMetric metric, Tags tags, Map countMap, - Map>> scoreMap, ConstraintMatchTotal constraintMatchTotal, - ScoreDefinition scoreDefinition, SolverScope solverScope) { + private > void collectConstraintMatchTotalMetrics(SolverMetric metric, Tags tags, + Map countMap, Map scoreMap, + ConstraintMatchTotal constraintMatchTotal, ScoreDefinition scoreDefinition, + SolverScope solverScope) { if (solverScope.isMetricEnabled(metric)) { if (countMap.containsKey(tags)) { countMap.get(tags).set(constraintMatchTotal.getConstraintMatchCount()); @@ -193,7 +196,8 @@ private void collectConstraintMatchTotalMetrics(SolverMetric metric, Tags tags, Metrics.gauge(metric.getMeterId() + ".count", tags, count); } - SolverMetric.registerScoreMetrics(metric, tags, scoreDefinition, scoreMap, constraintMatchTotal.getScore()); + SolverMetricUtil.registerScoreMetrics(metric, tags, scoreDefinition, scoreMap, + InnerScore.fullyAssigned(constraintMatchTotal.getScore())); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java b/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java index a2f4915717c..beaa2e16427 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java @@ -1,5 +1,7 @@ package ai.timefold.solver.core.impl.phase; +import java.util.Map; + import ai.timefold.solver.core.api.domain.solution.PlanningSolution; import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.config.solver.EnvironmentMode; @@ -9,8 +11,11 @@ import ai.timefold.solver.core.impl.phase.event.PhaseLifecycleSupport; import ai.timefold.solver.core.impl.phase.scope.AbstractPhaseScope; import ai.timefold.solver.core.impl.phase.scope.AbstractStepScope; +import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; import ai.timefold.solver.core.impl.solver.exception.ScoreCorruptionException; import ai.timefold.solver.core.impl.solver.exception.VariableCorruptionException; +import ai.timefold.solver.core.impl.solver.monitoring.ScoreLevels; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import ai.timefold.solver.core.impl.solver.scope.SolverScope; import ai.timefold.solver.core.impl.solver.termination.PhaseTermination; import ai.timefold.solver.core.impl.solver.termination.Termination; @@ -18,6 +23,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.micrometer.core.instrument.Tags; + /** * @param the solution type, the class with the {@link PlanningSolution} annotation * @see DefaultLocalSearchPhase @@ -193,11 +200,11 @@ public void stepEnded(AbstractStepScope stepScope) { private static void collectMetrics(AbstractStepScope stepScope) { var solverScope = stepScope.getPhaseScope().getSolverScope(); if (solverScope.isMetricEnabled(SolverMetric.STEP_SCORE) && stepScope.getScore().fullyAssigned()) { - SolverMetric.registerScoreMetrics(SolverMetric.STEP_SCORE, - solverScope.getMonitoringTags(), - solverScope.getScoreDefinition(), - solverScope.getStepScoreMap(), - stepScope.getScore().raw()); + Tags tags = solverScope.getMonitoringTags(); + ScoreDefinition scoreDefinition = solverScope.getScoreDefinition(); + Map tagToScoreLevels = solverScope.getStepScoreMap(); + SolverMetricUtil.registerScoreMetrics(SolverMetric.STEP_SCORE, tags, scoreDefinition, tagToScoreLevels, + stepScope.getScore()); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java index 3e2096341be..5c21219676c 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java @@ -10,7 +10,8 @@ public final class DefaultBestSolutionChangedEvent extends BestSoluti private final int unassignedCount; - public DefaultBestSolutionChangedEvent(@NonNull Solver solver, long timeMillisSpent, @NonNull Solution_ newBestSolution, @NonNull InnerScore newBestScore) { + public DefaultBestSolutionChangedEvent(@NonNull Solver solver, long timeMillisSpent, + @NonNull Solution_ newBestSolution, @NonNull InnerScore newBestScore) { super(solver, timeMillisSpent, newBestSolution, newBestScore.raw(), newBestScore.fullyAssigned()); this.unassignedCount = newBestScore.unassignedCount(); } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java new file mode 100644 index 00000000000..7113e3b4e0d --- /dev/null +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java @@ -0,0 +1,33 @@ +package ai.timefold.solver.core.impl.solver.monitoring; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public final class ScoreLevels { + + final AtomicReference[] levelValues; + final AtomicInteger unnassignedCount; + + @SuppressWarnings("unchecked") + ScoreLevels(int unnassignedCount, Number[] levelValues) { + // We store the values inside a constant reference, + // so that the metric can always load the latest value. + // If we stored the value directly and just overwrote it, + // the metric would always hold a reference to the old value, + // effectively ignoring the update. + this.unnassignedCount = new AtomicInteger(unnassignedCount); + this.levelValues = Arrays.stream(levelValues) + .map(AtomicReference::new) + .toArray(AtomicReference[]::new); + } + + void setLevelValue(int level, Number value) { + levelValues[level].set(value); + } + + void setUnnassignedCount(int unnassignedCount) { + this.unnassignedCount.set(unnassignedCount); + } + +} diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java new file mode 100644 index 00000000000..77a2c4efdfa --- /dev/null +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java @@ -0,0 +1,49 @@ +package ai.timefold.solver.core.impl.solver.monitoring; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; +import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.score.director.InnerScore; + +import org.jspecify.annotations.NullMarked; + +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tags; + +@NullMarked +public final class SolverMetricUtil { + + public static void registerScoreMetrics(SolverMetric metric, Tags tags, ScoreDefinition scoreDefinition, + Map tagToScoreLevels, InnerScore innerScore) { + var levelValues = innerScore.raw().toLevelNumbers(); + if (tagToScoreLevels.containsKey(tags)) { + var scoreLevels = tagToScoreLevels.get(tags); + scoreLevels.setUnnassignedCount(innerScore.unassignedCount()); + for (var i = 0; i < levelValues.length; i++) { + scoreLevels.setLevelValue(i, levelValues[i]); + } + } else { + var levelLabels = scoreDefinition.getLevelLabels(); + for (var i = 0; i < levelLabels.length; i++) { + levelLabels[i] = levelLabels[i].replace(' ', '.'); + } + var scoreLevels = new Number[levelLabels.length]; + System.arraycopy(levelValues, 0, scoreLevels, 0, levelValues.length); + var result = new ScoreLevels(innerScore.unassignedCount(), scoreLevels); + tagToScoreLevels.put(tags, result); + Metrics.gauge(metric.getMeterId() + "." + SolverMetric.UNASSIGNED_COUNT, tags, result.unnassignedCount, + AtomicInteger::doubleValue); + for (var i = 0; i < levelValues.length; i++) { + Metrics.gauge(metric.getMeterId() + "." + levelLabels[i], tags, result.levelValues[i], + ref -> ref.get().doubleValue()); + } + } + } + + private SolverMetricUtil() { + // No external instances. + } + +} diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/BestScoreStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java similarity index 65% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/BestScoreStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java index 3c126981242..32eb9fa83f5 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/BestScoreStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java @@ -1,21 +1,23 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; -import java.util.List; import java.util.Map; import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import ai.timefold.solver.core.api.solver.Solver; import ai.timefold.solver.core.api.solver.event.SolverEventListener; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; +import ai.timefold.solver.core.impl.score.director.InnerScore; import ai.timefold.solver.core.impl.solver.DefaultSolver; +import ai.timefold.solver.core.impl.solver.event.DefaultBestSolutionChangedEvent; +import ai.timefold.solver.core.impl.solver.monitoring.ScoreLevels; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; public class BestScoreStatistic implements SolverStatistic { - private final Map>> tagsToBestScoreMap = new ConcurrentHashMap<>(); + private final Map tagsToBestScoreMap = new ConcurrentHashMap<>(); private final Map, SolverEventListener> solverToEventListenerMap = new WeakHashMap<>(); @Override @@ -38,8 +40,11 @@ public void register(Solver solver) { var scoreDefinition = defaultSolver.getSolverScope().getScoreDefinition(); var tags = extractTags(solver); SolverEventListener listener = - event -> SolverMetric.registerScoreMetrics(SolverMetric.BEST_SCORE, tags, scoreDefinition, tagsToBestScoreMap, - event.getNewBestScore()); + event -> { + var castEvent = (DefaultBestSolutionChangedEvent) event; + SolverMetricUtil.registerScoreMetrics(SolverMetric.BEST_SCORE, tags, scoreDefinition, tagsToBestScoreMap, + InnerScore.withUnassignedCount(event.getNewBestScore(), castEvent.getUnassignedCount())); + }; solverToEventListenerMap.put(defaultSolver, listener); defaultSolver.addEventListener(listener); } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/BestSolutionMutationCountStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestSolutionMutationCountStatistic.java similarity index 97% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/BestSolutionMutationCountStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestSolutionMutationCountStatistic.java index 454390ca47d..7cd43c5bdaa 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/BestSolutionMutationCountStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestSolutionMutationCountStatistic.java @@ -1,4 +1,4 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; import java.util.Map; import java.util.WeakHashMap; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/MemoryUseStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/MemoryUseStatistic.java similarity index 91% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/MemoryUseStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/MemoryUseStatistic.java index 3adaf334996..5fbfdc9c21d 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/MemoryUseStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/MemoryUseStatistic.java @@ -1,4 +1,4 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; import ai.timefold.solver.core.api.solver.Solver; import ai.timefold.solver.core.impl.solver.DefaultSolver; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/MoveCountPerTypeStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/MoveCountPerTypeStatistic.java similarity index 98% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/MoveCountPerTypeStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/MoveCountPerTypeStatistic.java index 5e1db3ed45a..b51f171fb04 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/MoveCountPerTypeStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/MoveCountPerTypeStatistic.java @@ -1,4 +1,4 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; import java.util.HashMap; import java.util.Map; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/PickedMoveBestScoreDiffStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java similarity index 83% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/PickedMoveBestScoreDiffStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java index 9022a0454da..b5527520312 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/PickedMoveBestScoreDiffStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java @@ -1,10 +1,8 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; -import java.util.List; import java.util.Map; import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.api.solver.Solver; @@ -15,7 +13,10 @@ import ai.timefold.solver.core.impl.phase.scope.AbstractPhaseScope; import ai.timefold.solver.core.impl.phase.scope.AbstractStepScope; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.score.director.InnerScore; import ai.timefold.solver.core.impl.solver.DefaultSolver; +import ai.timefold.solver.core.impl.solver.monitoring.ScoreLevels; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; @@ -47,7 +48,7 @@ private static class PickedMoveBestScoreDiffStatisticListener scoreDefinition; - private final Map>> tagsToMoveScoreMap = new ConcurrentHashMap<>(); + private final Map tagsToMoveScoreMap = new ConcurrentHashMap<>(); public PickedMoveBestScoreDiffStatisticListener(ScoreDefinition scoreDefinition) { this.scoreDefinition = scoreDefinition; @@ -80,12 +81,10 @@ private void localSearchStepEnded(LocalSearchStepScope stepScope) { var newBestScore = stepScope. getScore().raw(); var bestScoreDiff = newBestScore.subtract(oldBestScore); oldBestScore = newBestScore; - SolverMetric.registerScoreMetrics(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF, - stepScope.getPhaseScope().getSolverScope().getMonitoringTags() - .and("move.type", moveType), - scoreDefinition, - tagsToMoveScoreMap, - bestScoreDiff); + Tags tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() + .and("move.type", moveType); + SolverMetricUtil.registerScoreMetrics(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF, tags, scoreDefinition, + tagsToMoveScoreMap, InnerScore.fullyAssigned(bestScoreDiff)); } } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/PickedMoveStepScoreDiffStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java similarity index 83% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/PickedMoveStepScoreDiffStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java index 9282820dd8c..9c3e321a2f7 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/PickedMoveStepScoreDiffStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java @@ -1,10 +1,8 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; -import java.util.List; import java.util.Map; import java.util.WeakHashMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.api.solver.Solver; @@ -15,7 +13,10 @@ import ai.timefold.solver.core.impl.phase.scope.AbstractPhaseScope; import ai.timefold.solver.core.impl.phase.scope.AbstractStepScope; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.score.director.InnerScore; import ai.timefold.solver.core.impl.solver.DefaultSolver; +import ai.timefold.solver.core.impl.solver.monitoring.ScoreLevels; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; @@ -48,7 +49,7 @@ private static class PickedMoveStepScoreDiffStatisticListener scoreDefinition; - private final Map>> tagsToMoveScoreMap = new ConcurrentHashMap<>(); + private final Map tagsToMoveScoreMap = new ConcurrentHashMap<>(); public PickedMoveStepScoreDiffStatisticListener(ScoreDefinition scoreDefinition) { this.scoreDefinition = scoreDefinition; @@ -82,12 +83,10 @@ private void localSearchStepEnded(LocalSearchStepScope stepScope) { var stepScoreDiff = newStepScore.subtract(oldStepScore); oldStepScore = newStepScore; - SolverMetric.registerScoreMetrics(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF, - stepScope.getPhaseScope().getSolverScope().getMonitoringTags() - .and("move.type", moveType), - scoreDefinition, - tagsToMoveScoreMap, - stepScoreDiff); + Tags tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() + .and("move.type", moveType); + SolverMetricUtil.registerScoreMetrics(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF, tags, scoreDefinition, + tagsToMoveScoreMap, InnerScore.fullyAssigned(stepScoreDiff)); } } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/SolverScopeStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/SolverScopeStatistic.java similarity index 95% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/SolverScopeStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/SolverScopeStatistic.java index 38a686b0fae..fdacd0e8294 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/SolverScopeStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/SolverScopeStatistic.java @@ -1,4 +1,4 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; import java.util.function.ToDoubleFunction; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/SolverStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/SolverStatistic.java similarity index 74% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/SolverStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/SolverStatistic.java index b39423beef8..49c72aa383e 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/SolverStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/SolverStatistic.java @@ -1,4 +1,4 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; import ai.timefold.solver.core.api.solver.Solver; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/statistic/StatelessSolverStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/StatelessSolverStatistic.java similarity index 86% rename from core/src/main/java/ai/timefold/solver/core/impl/statistic/StatelessSolverStatistic.java rename to core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/StatelessSolverStatistic.java index 3d06b44e208..c6643f2302a 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/statistic/StatelessSolverStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/StatelessSolverStatistic.java @@ -1,4 +1,4 @@ -package ai.timefold.solver.core.impl.statistic; +package ai.timefold.solver.core.impl.solver.monitoring.statistic; import ai.timefold.solver.core.api.solver.Solver; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java index 20f9f83cf7b..4c0de5ae169 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java @@ -5,7 +5,6 @@ import java.time.Clock; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Random; @@ -27,6 +26,7 @@ import ai.timefold.solver.core.impl.score.director.InnerScoreDirector; import ai.timefold.solver.core.impl.solver.AbstractSolver; import ai.timefold.solver.core.impl.solver.change.DefaultProblemChangeDirector; +import ai.timefold.solver.core.impl.solver.monitoring.ScoreLevels; import ai.timefold.solver.core.impl.solver.termination.PhaseTermination; import ai.timefold.solver.core.impl.solver.thread.ChildThreadType; @@ -69,7 +69,7 @@ public class SolverScope { /** * Used for tracking step score */ - private final Map>> stepScoreMap = new ConcurrentHashMap<>(); + private final Map stepScoreMap = new ConcurrentHashMap<>(); /** * Used for tracking move count per move type @@ -122,7 +122,7 @@ public void setMonitoringTags(Tags monitoringTags) { this.monitoringTags = monitoringTags; } - public Map>> getStepScoreMap() { + public Map getStepScoreMap() { return stepScoreMap; } From 19d91a49dfdaed50e0cf33542552ffdbb1b4954b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 10:12:08 +0200 Subject: [PATCH 3/9] finally fix the benchmarker issue --- .../impl/statistic/StatisticRegistry.java | 25 ++- .../BestScoreSubSingleStatistic.java | 2 +- .../StepScoreSubSingleStatistic.java | 4 +- ...veTypeBestScoreDiffSubSingleStatistic.java | 2 +- ...veTypeStepScoreDiffSubSingleStatistic.java | 2 +- .../api/PlannerBenchmarkFactoryTest.java | 162 ++++++++---------- .../benchmark/api/PlannerBenchmarkTest.java | 81 +++++++++ .../solver/monitoring/SolverMetric.java | 3 - .../localsearch/DefaultLocalSearchPhase.java | 2 +- .../solver/core/impl/phase/AbstractPhase.java | 2 +- .../solver/monitoring/SolverMetricUtil.java | 26 ++- .../statistic/BestScoreStatistic.java | 2 +- .../PickedMoveBestScoreDiffStatistic.java | 2 +- .../PickedMoveStepScoreDiffStatistic.java | 2 +- 14 files changed, 199 insertions(+), 118 deletions(-) create mode 100644 benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java index 119aa76e771..c5dbfa7c7ef 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java @@ -12,13 +12,14 @@ import java.util.function.ObjLongConsumer; import java.util.stream.Collectors; -import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.api.score.constraint.ConstraintRef; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; import ai.timefold.solver.core.impl.phase.event.PhaseLifecycleListener; import ai.timefold.solver.core.impl.phase.scope.AbstractPhaseScope; import ai.timefold.solver.core.impl.phase.scope.AbstractStepScope; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.score.director.InnerScore; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import ai.timefold.solver.core.impl.solver.scope.SolverScope; import io.micrometer.core.instrument.Meter; @@ -88,21 +89,17 @@ public Set getMeterIds(SolverMetric metric, Tags runId) { .collect(Collectors.toSet()); } - public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer> scoreConsumer) { - var labelNames = scoreDefinition.getLevelLabels(); - for (var i = 0; i < labelNames.length; i++) { - labelNames[i] = labelNames[i].replace(' ', '.'); - } - var levelNumbers = new Number[labelNames.length]; - for (var i = 0; i < labelNames.length; i++) { - var scoreLevelGauge = this.find(metric.getMeterId() + "." + labelNames[i]).tags(runId).gauge(); + public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer> scoreConsumer) { + var score = SolverMetricUtil.extractScore(metric, scoreDefinition, id -> { + var gaugeId = metric.getMeterId() + "." + id; + var scoreLevelGauge = this.find(gaugeId).tags(runId).gauge(); if (scoreLevelGauge != null && Double.isFinite(scoreLevelGauge.value())) { - levelNumbers[i] = scoreLevelNumberConverter.apply(scoreLevelGauge.value()); + return scoreLevelNumberConverter.apply(scoreLevelGauge.value()); } else { - return; + return scoreLevelNumberConverter.apply(0.0); } - } - scoreConsumer.accept(scoreDefinition.fromLevelNumbers(levelNumbers)); + }); + scoreConsumer.accept(score); } @SuppressWarnings({ "rawtypes", "unchecked" }) @@ -121,7 +118,7 @@ public void extractConstraintSummariesFromMeters(SolverMetric metric, Tags runId // Get the count gauge (add constraint package and constraint name to the run tags) score -> getGaugeValue(metric.getMeterId() + ".count", constraintMatchTotalRunId, count -> constraintMatchTotalConsumer - .accept(new ConstraintSummary(constraintRef, score, count.intValue())))); + .accept(new ConstraintSummary(constraintRef, score.raw(), count.intValue())))); }); } diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java index 5115b967007..210b5a88570 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java @@ -32,7 +32,7 @@ public void open(StatisticRegistry registry, Tags runTag) { registry.addListener(SolverMetric.BEST_SCORE, timestamp -> registry.extractScoreFromMeters(SolverMetric.BEST_SCORE, runTag, score -> pointList - .add(new BestScoreStatisticPoint(timestamp, score, subSingleBenchmarkResult.isInitialized())))); + .add(new BestScoreStatisticPoint(timestamp, score.raw(), score.fullyAssigned())))); } // ************************************************************************ diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java index 6a2a86f4801..118b612ffd2 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java @@ -31,8 +31,8 @@ public StepScoreSubSingleStatistic(SubSingleBenchmarkResult subSingleBenchmarkRe public void open(StatisticRegistry registry, Tags runTag) { registry.addListener(SolverMetric.STEP_SCORE, timeMillisSpent -> registry.extractScoreFromMeters(SolverMetric.STEP_SCORE, runTag, - score -> pointList.add(new StepScoreStatisticPoint(timeMillisSpent, score, - subSingleBenchmarkResult.isInitialized())))); + score -> pointList + .add(new StepScoreStatisticPoint(timeMillisSpent, score.raw(), score.fullyAssigned())))); } // ************************************************************************ diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypebestscore/PickedMoveTypeBestScoreDiffSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypebestscore/PickedMoveTypeBestScoreDiffSubSingleStatistic.java index 700a94bcc08..738e0592c64 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypebestscore/PickedMoveTypeBestScoreDiffSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypebestscore/PickedMoveTypeBestScoreDiffSubSingleStatistic.java @@ -35,7 +35,7 @@ public void open(StatisticRegistry registry, Tags runTag) { registry.extractScoreFromMeters(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF, runTag.and(Tag.of("move.type", moveType)), score -> pointList.add(new PickedMoveTypeBestScoreDiffStatisticPoint( - timeMillisSpent, moveType, score))); + timeMillisSpent, moveType, score.raw()))); } }); } diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypestepscore/PickedMoveTypeStepScoreDiffSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypestepscore/PickedMoveTypeStepScoreDiffSubSingleStatistic.java index 9657a08147b..4fb1a0d8e97 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypestepscore/PickedMoveTypeStepScoreDiffSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/subsingle/pickedmovetypestepscore/PickedMoveTypeStepScoreDiffSubSingleStatistic.java @@ -35,7 +35,7 @@ public void open(StatisticRegistry registry, Tags runTag) { registry.extractScoreFromMeters(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF, runTag.and(Tag.of("move.type", moveType)), score -> pointList.add(new PickedMoveTypeStepScoreDiffStatisticPoint( - timeMillisSpent, moveType, score))); + timeMillisSpent, moveType, score.raw()))); } }); } diff --git a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java index b07232be03f..d7958ce29f4 100644 --- a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java +++ b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java @@ -3,10 +3,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; -import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.Collections; @@ -27,63 +26,51 @@ import ai.timefold.solver.core.testutil.PlannerTestUtils; import org.jspecify.annotations.NonNull; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class PlannerBenchmarkFactoryTest { - private static File benchmarkTestDir; - private static File benchmarkOutputTestDir; - - @BeforeAll - static void setup() throws IOException { - benchmarkTestDir = new File("target/test/benchmarkTest/"); - benchmarkTestDir.mkdirs(); - new File(benchmarkTestDir, "input.xml").createNewFile(); - benchmarkOutputTestDir = new File(benchmarkTestDir, "output/"); - benchmarkOutputTestDir.mkdir(); - } - // ************************************************************************ // Static creation methods: SolverConfig XML // ************************************************************************ @Test - void createFromSolverConfigXmlResource() { - PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( + void createFromSolverConfigXmlResource(@TempDir Path benchmarkTestDir) { + var benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( "ai/timefold/solver/core/config/solver/testdataSolverConfig.xml"); - TestdataSolution solution = new TestdataSolution("s1"); + var solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( - "ai/timefold/solver/core/config/solver/testdataSolverConfig.xml", benchmarkOutputTestDir); + "ai/timefold/solver/core/config/solver/testdataSolverConfig.xml", benchmarkTestDir.toFile()); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); } @Test - void createFromSolverConfigXmlResource_classLoader() { + void createFromSolverConfigXmlResource_classLoader(@TempDir Path benchmarkTestDir) { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( + var benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( "divertThroughClassLoader/ai/timefold/solver/core/api/solver/classloaderTestdataSolverConfig.xml", classLoader); - TestdataSolution solution = new TestdataSolution("s1"); + var solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( "divertThroughClassLoader/ai/timefold/solver/core/api/solver/classloaderTestdataSolverConfig.xml", - benchmarkOutputTestDir, classLoader); + benchmarkTestDir.toFile(), classLoader); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); } @Test void problemIsNotASolutionInstance() { - SolverConfig solverConfig = PlannerTestUtils.buildSolverConfig( + var solverConfig = PlannerTestUtils.buildSolverConfig( TestdataSolution.class, TestdataEntity.class); - PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.create( + var benchmarkFactory = PlannerBenchmarkFactory.create( PlannerBenchmarkConfig.createFromSolverConfig(solverConfig)); assertThatIllegalArgumentException().isThrownBy( () -> benchmarkFactory.buildPlannerBenchmark("This is not a solution instance.")); @@ -91,11 +78,11 @@ void problemIsNotASolutionInstance() { @Test void problemIsNull() { - SolverConfig solverConfig = PlannerTestUtils.buildSolverConfig( + var solverConfig = PlannerTestUtils.buildSolverConfig( TestdataSolution.class, TestdataEntity.class); - PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.create( + var benchmarkFactory = PlannerBenchmarkFactory.create( PlannerBenchmarkConfig.createFromSolverConfig(solverConfig)); - TestdataSolution solution = new TestdataSolution("s1"); + var solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); assertThatIllegalArgumentException().isThrownBy(() -> benchmarkFactory.buildPlannerBenchmark(solution, null)); @@ -107,9 +94,9 @@ void problemIsNull() { @Test void createFromXmlResource() { - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml"); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -118,17 +105,17 @@ void createFromXmlResource() { void createFromXmlResource_classLoader() { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( "divertThroughClassLoader/ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfig.xml", classLoader); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test void createFromXmlResource_nonExisting() { - final String nonExistingBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/nonExistingBenchmarkConfig.xml"; + final var nonExistingBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/nonExistingBenchmarkConfig.xml"; assertThatIllegalArgumentException() .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlResource(nonExistingBenchmarkConfigResource)) .withMessageContaining(nonExistingBenchmarkConfigResource); @@ -136,7 +123,7 @@ void createFromXmlResource_nonExisting() { @Test void createFromInvalidXmlResource_failsShowingBothResourceAndReason() { - final String invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; + final var invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; assertThatIllegalArgumentException() .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlResource(invalidXmlBenchmarkConfigResource)) .withMessageContaining(invalidXmlBenchmarkConfigResource) @@ -144,66 +131,66 @@ void createFromInvalidXmlResource_failsShowingBothResourceAndReason() { } @Test - void createFromInvalidXmlFile_failsShowingBothPathAndReason() throws IOException { - final String invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; - File file = new File(benchmarkTestDir, "invalidBenchmarkConfig.xml"); - try (InputStream in = getClass().getClassLoader().getResourceAsStream(invalidXmlBenchmarkConfigResource)) { - Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + void createFromInvalidXmlFile_failsShowingBothPathAndReason(@TempDir Path benchmarkTestDir) throws IOException { + var invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; + var path = benchmarkTestDir.resolve("invalidBenchmarkConfig.xml"); + try (var in = getClass().getClassLoader().getResourceAsStream(invalidXmlBenchmarkConfigResource)) { + Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); } assertThatIllegalArgumentException() - .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlFile(file)) - .withMessageContaining(file.toString()) + .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlFile(path.toFile())) + .withMessageContaining(path.toString()) .withStackTraceContaining("invalidElementThatShouldNotBeHere"); } @Test void createFromXmlResource_uninitializedBestSolution() { - PlannerBenchmarkConfig benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( + var benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml"); - SolverBenchmarkConfig solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); - CustomPhaseConfig phaseConfig = new CustomPhaseConfig(); + var solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); + var phaseConfig = new CustomPhaseConfig(); phaseConfig.setCustomPhaseCommandClassList(Collections.singletonList(NoChangeCustomPhaseCommand.class)); solverBenchmarkConfig.getSolverConfig().setPhaseConfigList(Collections.singletonList(phaseConfig)); - PlannerBenchmark plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); + var plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test void createFromXmlResource_subSingleCount() { - PlannerBenchmarkConfig benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( + var benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml"); - SolverBenchmarkConfig solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); + var solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); solverBenchmarkConfig.setSubSingleCount(3); - PlannerBenchmark plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); + var plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test - void createFromXmlFile() throws IOException { - File file = new File(benchmarkTestDir, "testdataBenchmarkConfig.xml"); - try (InputStream in = getClass().getClassLoader().getResourceAsStream( + void createFromXmlFile(@TempDir Path benchmarkTestDir) throws IOException { + var path = benchmarkTestDir.resolve("testdataBenchmarkConfig.xml"); + try (var in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml")) { - Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); } - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(file); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(path.toFile()); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test - void createFromXmlFile_classLoader() throws IOException { + void createFromXmlFile_classLoader(@TempDir Path benchmarkTestDir) throws IOException { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - File file = new File(benchmarkTestDir, "classloaderTestdataBenchmarkConfig.xml"); - try (InputStream in = getClass().getClassLoader().getResourceAsStream( + var path = benchmarkTestDir.resolve("classloaderTestdataBenchmarkConfig.xml"); + try (var in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfig.xml")) { - Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); } - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(file, classLoader); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(path.toFile(), classLoader); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -214,9 +201,9 @@ void createFromXmlFile_classLoader() throws IOException { @Test void createFromFreemarkerXmlResource() { - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfigTemplate.xml.ftl"); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -225,10 +212,10 @@ void createFromFreemarkerXmlResource() { void createFromFreemarkerXmlResource_classLoader() { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( "divertThroughClassLoader/ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfigTemplate.xml.ftl", classLoader); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -240,30 +227,29 @@ void createFromFreemarkerXmlResource_nonExisting() { } @Test - void createFromFreemarkerXmlFile() throws IOException { - File file = new File(benchmarkTestDir, "testdataBenchmarkConfigTemplate.xml.ftl"); - try (InputStream in = getClass().getClassLoader().getResourceAsStream( + void createFromFreemarkerXmlFile(@TempDir Path benchmarkTestDir) throws IOException { + var path = benchmarkTestDir.resolve("testdataBenchmarkConfigTemplate.xml.ftl"); + try (var in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfigTemplate.xml.ftl")) { - Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); } - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(file); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(path.toFile()); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test - void createFromFreemarkerXmlFile_classLoader() throws IOException { + void createFromFreemarkerXmlFile_classLoader(@TempDir Path benchmarkTestDir) throws IOException { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - File file = new File(benchmarkTestDir, "classloaderTestdataBenchmarkConfigTemplate.xml.ftl"); - try (InputStream in = getClass().getClassLoader().getResourceAsStream( + var path = benchmarkTestDir.resolve("classloaderTestdataBenchmarkConfigTemplate.xml.ftl"); + try (var in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfigTemplate.xml.ftl")) { - Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); } - PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(file, - classLoader); - PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(path.toFile(), classLoader); + var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -273,17 +259,17 @@ void createFromFreemarkerXmlFile_classLoader() throws IOException { // ************************************************************************ @Test - void createFromSolverConfig() { - SolverConfig solverConfig = PlannerTestUtils.buildSolverConfig(TestdataSolution.class, TestdataEntity.class); + void createFromSolverConfig(@TempDir Path benchmarkTestDir) { + var solverConfig = PlannerTestUtils.buildSolverConfig(TestdataSolution.class, TestdataEntity.class); - TestdataSolution solution = new TestdataSolution("s1"); + var solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); - PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig); + var benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); - benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig, benchmarkOutputTestDir); + benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig, benchmarkTestDir.toFile()); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); } @@ -293,8 +279,8 @@ void createFromSolverConfig() { @Test void buildPlannerBenchmark() { - PlannerBenchmarkConfig benchmarkConfig = new PlannerBenchmarkConfig(); - SolverBenchmarkConfig inheritedSolverConfig = new SolverBenchmarkConfig(); + var benchmarkConfig = new PlannerBenchmarkConfig(); + var inheritedSolverConfig = new SolverBenchmarkConfig(); inheritedSolverConfig.setSolverConfig(new SolverConfig() .withSolutionClass(TestdataSolution.class) .withEntityClasses(TestdataEntity.class) @@ -304,16 +290,16 @@ void buildPlannerBenchmark() { benchmarkConfig.setSolverBenchmarkConfigList(Arrays.asList( new SolverBenchmarkConfig(), new SolverBenchmarkConfig(), new SolverBenchmarkConfig())); - PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.create(benchmarkConfig); + var benchmarkFactory = PlannerBenchmarkFactory.create(benchmarkConfig); - TestdataSolution solution1 = new TestdataSolution("s1"); + var solution1 = new TestdataSolution("s1"); solution1.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution1.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); - TestdataSolution solution2 = new TestdataSolution("s2"); + var solution2 = new TestdataSolution("s2"); solution2.setEntityList(Arrays.asList(new TestdataEntity("e11"), new TestdataEntity("e12"), new TestdataEntity("e13"))); solution2.setValueList(Arrays.asList(new TestdataValue("v11"), new TestdataValue("v12"))); - DefaultPlannerBenchmark plannerBenchmark = + var plannerBenchmark = (DefaultPlannerBenchmark) benchmarkFactory.buildPlannerBenchmark(solution1, solution2); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.getPlannerBenchmarkResult().getSolverBenchmarkResultList()).hasSize(3); diff --git a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java new file mode 100644 index 00000000000..97c892e1a91 --- /dev/null +++ b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java @@ -0,0 +1,81 @@ +package ai.timefold.solver.benchmark.api; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.assertj.core.api.SoftAssertions.assertSoftly; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; + +import ai.timefold.solver.benchmark.config.PlannerBenchmarkConfig; +import ai.timefold.solver.benchmark.config.SolverBenchmarkConfig; +import ai.timefold.solver.benchmark.impl.DefaultPlannerBenchmark; +import ai.timefold.solver.core.config.solver.SolverConfig; +import ai.timefold.solver.core.config.solver.termination.TerminationConfig; +import ai.timefold.solver.core.testdomain.TestdataConstraintProvider; +import ai.timefold.solver.core.testdomain.TestdataEntity; +import ai.timefold.solver.core.testdomain.TestdataSolution; +import ai.timefold.solver.core.testdomain.TestdataValue; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class PlannerBenchmarkTest { + + @Test + void runPlannerBenchmark(@TempDir Path benchmarkTestDir) { + var benchmarkConfig = new PlannerBenchmarkConfig(); + benchmarkConfig.setBenchmarkDirectory(benchmarkTestDir.toFile()); + benchmarkConfig.setWarmUpMillisecondsSpentLimit(1L); // Minimize warmup. + var inheritedSolverConfig = new SolverBenchmarkConfig(); + inheritedSolverConfig.setSolverConfig(new SolverConfig() + .withSolutionClass(TestdataSolution.class) + .withEntityClasses(TestdataEntity.class) + .withConstraintProviderClass(TestdataConstraintProvider.class) + // Only run for a short amount of time. + .withTerminationConfig(new TerminationConfig().withUnimprovedMillisecondsSpentLimit(100L))); + benchmarkConfig.setInheritedSolverBenchmarkConfig(inheritedSolverConfig); + benchmarkConfig.setSolverBenchmarkConfigList(List.of(new SolverBenchmarkConfig())); + var benchmarkFactory = PlannerBenchmarkFactory.create(benchmarkConfig); + + var solution1 = new TestdataSolution("s1"); + solution1.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); + solution1.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); + + var plannerBenchmark = (DefaultPlannerBenchmark) benchmarkFactory.buildPlannerBenchmark(solution1); + plannerBenchmark.benchmark(); // Run the benchmark. + var folder = plannerBenchmark.getBenchmarkReport().getHtmlOverviewFile() + .toPath() + .getParent(); + var csv = folder.resolve(Path.of("Problem_0", "Config_0", "sub0", "BEST_SCORE.csv")); + assertThat(csv).exists(); + + try (var lines = Files.lines(csv)) { + var lineList = lines.toList(); + assertThat(lineList).isNotEmpty(); + assertSoftly(softly -> { + // Proper header. + softly.assertThat(lineList) + .first() + .isEqualTo(""" + "timeMillisSpent","score","initialized" + """.trim()); + // Checks that best score was recorded at least once. + // Requires LS to have started, as CH does not store best score. + // We only check score+initialized, as "timeMillisSpent" can be anything. + softly.assertThat(lineList) + .last() + .asString() + .endsWith(""" + "0","true" + """.trim()); + }); + } catch (IOException e) { + fail(e); + } + } + +} diff --git a/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java b/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java index 6c1c2950b2f..8b60f38ebb4 100644 --- a/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java +++ b/core/src/main/java/ai/timefold/solver/core/config/solver/monitoring/SolverMetric.java @@ -54,9 +54,6 @@ public enum SolverMetric { PICKED_MOVE_TYPE_STEP_SCORE_DIFF("timefold.solver.move.type.step.score.diff", new PickedMoveStepScoreDiffStatistic<>(), false); - // Necessary for benchmarker, but otherwise undocumented and not considered public. - public static final String UNASSIGNED_COUNT = "unassigned.count"; - private final String meterId; @SuppressWarnings("rawtypes") private final SolverStatistic registerFunction; diff --git a/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java b/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java index 930cad42b4e..bb3c691c183 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/localsearch/DefaultLocalSearchPhase.java @@ -196,7 +196,7 @@ private > void collectConstraintMatchTotalMetrics(S Metrics.gauge(metric.getMeterId() + ".count", tags, count); } - SolverMetricUtil.registerScoreMetrics(metric, tags, scoreDefinition, scoreMap, + SolverMetricUtil.registerScore(metric, tags, scoreDefinition, scoreMap, InnerScore.fullyAssigned(constraintMatchTotal.getScore())); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java b/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java index beaa2e16427..2198300d592 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java @@ -203,7 +203,7 @@ private static void collectMetrics(AbstractStepScope step Tags tags = solverScope.getMonitoringTags(); ScoreDefinition scoreDefinition = solverScope.getScoreDefinition(); Map tagToScoreLevels = solverScope.getStepScoreMap(); - SolverMetricUtil.registerScoreMetrics(SolverMetric.STEP_SCORE, tags, scoreDefinition, tagToScoreLevels, + SolverMetricUtil.registerScore(SolverMetric.STEP_SCORE, tags, scoreDefinition, tagToScoreLevels, stepScope.getScore()); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java index 77a2c4efdfa..fa24c1ef48b 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java @@ -2,7 +2,9 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import ai.timefold.solver.core.api.score.Score; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; import ai.timefold.solver.core.impl.score.director.InnerScore; @@ -15,8 +17,11 @@ @NullMarked public final class SolverMetricUtil { - public static void registerScoreMetrics(SolverMetric metric, Tags tags, ScoreDefinition scoreDefinition, - Map tagToScoreLevels, InnerScore innerScore) { + // Necessary for benchmarker, but otherwise undocumented and not considered public. + private static final String UNASSIGNED_COUNT = "unassigned.count"; + + public static > void registerScore(SolverMetric metric, Tags tags, + ScoreDefinition scoreDefinition, Map tagToScoreLevels, InnerScore innerScore) { var levelValues = innerScore.raw().toLevelNumbers(); if (tagToScoreLevels.containsKey(tags)) { var scoreLevels = tagToScoreLevels.get(tags); @@ -33,7 +38,7 @@ public static void registerScoreMetrics(SolverMetric metric, Tags tags, ScoreDef System.arraycopy(levelValues, 0, scoreLevels, 0, levelValues.length); var result = new ScoreLevels(innerScore.unassignedCount(), scoreLevels); tagToScoreLevels.put(tags, result); - Metrics.gauge(metric.getMeterId() + "." + SolverMetric.UNASSIGNED_COUNT, tags, result.unnassignedCount, + Metrics.gauge(metric.getMeterId() + "." + UNASSIGNED_COUNT, tags, result.unnassignedCount, AtomicInteger::doubleValue); for (var i = 0; i < levelValues.length; i++) { Metrics.gauge(metric.getMeterId() + "." + levelLabels[i], tags, result.levelValues[i], @@ -42,6 +47,21 @@ public static void registerScoreMetrics(SolverMetric metric, Tags tags, ScoreDef } } + public static > InnerScore extractScore(SolverMetric metric, + ScoreDefinition scoreDefinition, Function scoreLevelFunction) { + var labelNames = scoreDefinition.getLevelLabels(); + for (var i = 0; i < labelNames.length; i++) { + labelNames[i] = labelNames[i].replace(' ', '.'); + } + var levelNumbers = new Number[labelNames.length]; + for (var i = 0; i < labelNames.length; i++) { + levelNumbers[i] = scoreLevelFunction.apply(metric.getMeterId() + "." + labelNames[i]); + } + var score = scoreDefinition.fromLevelNumbers(levelNumbers); + var unassignedCount = scoreLevelFunction.apply(metric.getMeterId() + "." + UNASSIGNED_COUNT); + return InnerScore.withUnassignedCount(score, unassignedCount.intValue()); + } + private SolverMetricUtil() { // No external instances. } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java index 32eb9fa83f5..6ecfe5322b7 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/BestScoreStatistic.java @@ -42,7 +42,7 @@ public void register(Solver solver) { SolverEventListener listener = event -> { var castEvent = (DefaultBestSolutionChangedEvent) event; - SolverMetricUtil.registerScoreMetrics(SolverMetric.BEST_SCORE, tags, scoreDefinition, tagsToBestScoreMap, + SolverMetricUtil.registerScore(SolverMetric.BEST_SCORE, tags, scoreDefinition, tagsToBestScoreMap, InnerScore.withUnassignedCount(event.getNewBestScore(), castEvent.getUnassignedCount())); }; solverToEventListenerMap.put(defaultSolver, listener); diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java index b5527520312..61aa00fa4c6 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java @@ -83,7 +83,7 @@ private void localSearchStepEnded(LocalSearchStepScope stepScope) { oldBestScore = newBestScore; Tags tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() .and("move.type", moveType); - SolverMetricUtil.registerScoreMetrics(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF, tags, scoreDefinition, + SolverMetricUtil.registerScore(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF, tags, scoreDefinition, tagsToMoveScoreMap, InnerScore.fullyAssigned(bestScoreDiff)); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java index 9c3e321a2f7..8c3e47d7406 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java @@ -85,7 +85,7 @@ private void localSearchStepEnded(LocalSearchStepScope stepScope) { Tags tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() .and("move.type", moveType); - SolverMetricUtil.registerScoreMetrics(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF, tags, scoreDefinition, + SolverMetricUtil.registerScore(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF, tags, scoreDefinition, tagsToMoveScoreMap, InnerScore.fullyAssigned(stepScoreDiff)); } } From 3441fac04744e51f812dd4ee0b5356d79124cd4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 11:10:56 +0200 Subject: [PATCH 4/9] Reuse some code --- .../benchmark/api/PlannerBenchmarkTest.java | 8 ++--- .../solver/monitoring/SolverMetricUtil.java | 35 ++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java index 97c892e1a91..470fb323855 100644 --- a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java +++ b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java @@ -61,8 +61,8 @@ void runPlannerBenchmark(@TempDir Path benchmarkTestDir) { softly.assertThat(lineList) .first() .isEqualTo(""" - "timeMillisSpent","score","initialized" - """.trim()); + "timeMillisSpent","score","initialized" + """.trim()); // Checks that best score was recorded at least once. // Requires LS to have started, as CH does not store best score. // We only check score+initialized, as "timeMillisSpent" can be anything. @@ -70,8 +70,8 @@ void runPlannerBenchmark(@TempDir Path benchmarkTestDir) { .last() .asString() .endsWith(""" - "0","true" - """.trim()); + "0","true" + """.trim()); }); } catch (IOException e) { fail(e); diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java index fa24c1ef48b..fafaaaa64d5 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java @@ -18,47 +18,56 @@ public final class SolverMetricUtil { // Necessary for benchmarker, but otherwise undocumented and not considered public. - private static final String UNASSIGNED_COUNT = "unassigned.count"; + private static final String UNASSIGNED_COUNT_LABEL = "unassigned.count"; public static > void registerScore(SolverMetric metric, Tags tags, ScoreDefinition scoreDefinition, Map tagToScoreLevels, InnerScore innerScore) { var levelValues = innerScore.raw().toLevelNumbers(); if (tagToScoreLevels.containsKey(tags)) { + // Set new score levels for the previously registered gauges to read. var scoreLevels = tagToScoreLevels.get(tags); scoreLevels.setUnnassignedCount(innerScore.unassignedCount()); for (var i = 0; i < levelValues.length; i++) { scoreLevels.setLevelValue(i, levelValues[i]); } } else { - var levelLabels = scoreDefinition.getLevelLabels(); - for (var i = 0; i < levelLabels.length; i++) { - levelLabels[i] = levelLabels[i].replace(' ', '.'); - } + var levelLabels = getLevelLabels(scoreDefinition); var scoreLevels = new Number[levelLabels.length]; System.arraycopy(levelValues, 0, scoreLevels, 0, levelValues.length); var result = new ScoreLevels(innerScore.unassignedCount(), scoreLevels); tagToScoreLevels.put(tags, result); - Metrics.gauge(metric.getMeterId() + "." + UNASSIGNED_COUNT, tags, result.unnassignedCount, + + // Register the gauges to read the score levels. + Metrics.gauge(getGaugeName(metric, UNASSIGNED_COUNT_LABEL), tags, result.unnassignedCount, AtomicInteger::doubleValue); for (var i = 0; i < levelValues.length; i++) { - Metrics.gauge(metric.getMeterId() + "." + levelLabels[i], tags, result.levelValues[i], + Metrics.gauge(getGaugeName(metric, levelLabels[i]), tags, result.levelValues[i], ref -> ref.get().doubleValue()); } } } - public static > InnerScore extractScore(SolverMetric metric, - ScoreDefinition scoreDefinition, Function scoreLevelFunction) { + private static String[] getLevelLabels(ScoreDefinition scoreDefinition) { var labelNames = scoreDefinition.getLevelLabels(); for (var i = 0; i < labelNames.length; i++) { labelNames[i] = labelNames[i].replace(' ', '.'); } - var levelNumbers = new Number[labelNames.length]; - for (var i = 0; i < labelNames.length; i++) { - levelNumbers[i] = scoreLevelFunction.apply(metric.getMeterId() + "." + labelNames[i]); + return labelNames; + } + + private static String getGaugeName(SolverMetric metric, String label) { + return metric.getMeterId() + "." + label; + } + + public static > InnerScore extractScore(SolverMetric metric, + ScoreDefinition scoreDefinition, Function scoreLevelFunction) { + var levelLabels = getLevelLabels(scoreDefinition); + var levelNumbers = new Number[levelLabels.length]; + for (var i = 0; i < levelLabels.length; i++) { + levelNumbers[i] = scoreLevelFunction.apply(getGaugeName(metric, levelLabels[i])); } var score = scoreDefinition.fromLevelNumbers(levelNumbers); - var unassignedCount = scoreLevelFunction.apply(metric.getMeterId() + "." + UNASSIGNED_COUNT); + var unassignedCount = scoreLevelFunction.apply(getGaugeName(metric, UNASSIGNED_COUNT_LABEL)); return InnerScore.withUnassignedCount(score, unassignedCount.intValue()); } From 4798bdb25d2e31067f1bcd44d501a57773f72aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 11:17:32 +0200 Subject: [PATCH 5/9] Fix test --- .../api/PlannerBenchmarkFactoryTest.java | 162 ++++++++++-------- 1 file changed, 88 insertions(+), 74 deletions(-) diff --git a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java index d7958ce29f4..b07232be03f 100644 --- a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java +++ b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkFactoryTest.java @@ -3,9 +3,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.Collections; @@ -26,51 +27,63 @@ import ai.timefold.solver.core.testutil.PlannerTestUtils; import org.jspecify.annotations.NonNull; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; class PlannerBenchmarkFactoryTest { + private static File benchmarkTestDir; + private static File benchmarkOutputTestDir; + + @BeforeAll + static void setup() throws IOException { + benchmarkTestDir = new File("target/test/benchmarkTest/"); + benchmarkTestDir.mkdirs(); + new File(benchmarkTestDir, "input.xml").createNewFile(); + benchmarkOutputTestDir = new File(benchmarkTestDir, "output/"); + benchmarkOutputTestDir.mkdir(); + } + // ************************************************************************ // Static creation methods: SolverConfig XML // ************************************************************************ @Test - void createFromSolverConfigXmlResource(@TempDir Path benchmarkTestDir) { - var benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( + void createFromSolverConfigXmlResource() { + PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( "ai/timefold/solver/core/config/solver/testdataSolverConfig.xml"); - var solution = new TestdataSolution("s1"); + TestdataSolution solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( - "ai/timefold/solver/core/config/solver/testdataSolverConfig.xml", benchmarkTestDir.toFile()); + "ai/timefold/solver/core/config/solver/testdataSolverConfig.xml", benchmarkOutputTestDir); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); } @Test - void createFromSolverConfigXmlResource_classLoader(@TempDir Path benchmarkTestDir) { + void createFromSolverConfigXmlResource_classLoader() { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - var benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( + PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( "divertThroughClassLoader/ai/timefold/solver/core/api/solver/classloaderTestdataSolverConfig.xml", classLoader); - var solution = new TestdataSolution("s1"); + TestdataSolution solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfigXmlResource( "divertThroughClassLoader/ai/timefold/solver/core/api/solver/classloaderTestdataSolverConfig.xml", - benchmarkTestDir.toFile(), classLoader); + benchmarkOutputTestDir, classLoader); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); } @Test void problemIsNotASolutionInstance() { - var solverConfig = PlannerTestUtils.buildSolverConfig( + SolverConfig solverConfig = PlannerTestUtils.buildSolverConfig( TestdataSolution.class, TestdataEntity.class); - var benchmarkFactory = PlannerBenchmarkFactory.create( + PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.create( PlannerBenchmarkConfig.createFromSolverConfig(solverConfig)); assertThatIllegalArgumentException().isThrownBy( () -> benchmarkFactory.buildPlannerBenchmark("This is not a solution instance.")); @@ -78,11 +91,11 @@ void problemIsNotASolutionInstance() { @Test void problemIsNull() { - var solverConfig = PlannerTestUtils.buildSolverConfig( + SolverConfig solverConfig = PlannerTestUtils.buildSolverConfig( TestdataSolution.class, TestdataEntity.class); - var benchmarkFactory = PlannerBenchmarkFactory.create( + PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.create( PlannerBenchmarkConfig.createFromSolverConfig(solverConfig)); - var solution = new TestdataSolution("s1"); + TestdataSolution solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); assertThatIllegalArgumentException().isThrownBy(() -> benchmarkFactory.buildPlannerBenchmark(solution, null)); @@ -94,9 +107,9 @@ void problemIsNull() { @Test void createFromXmlResource() { - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml"); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -105,17 +118,17 @@ void createFromXmlResource() { void createFromXmlResource_classLoader() { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlResource( "divertThroughClassLoader/ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfig.xml", classLoader); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test void createFromXmlResource_nonExisting() { - final var nonExistingBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/nonExistingBenchmarkConfig.xml"; + final String nonExistingBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/nonExistingBenchmarkConfig.xml"; assertThatIllegalArgumentException() .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlResource(nonExistingBenchmarkConfigResource)) .withMessageContaining(nonExistingBenchmarkConfigResource); @@ -123,7 +136,7 @@ void createFromXmlResource_nonExisting() { @Test void createFromInvalidXmlResource_failsShowingBothResourceAndReason() { - final var invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; + final String invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; assertThatIllegalArgumentException() .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlResource(invalidXmlBenchmarkConfigResource)) .withMessageContaining(invalidXmlBenchmarkConfigResource) @@ -131,66 +144,66 @@ void createFromInvalidXmlResource_failsShowingBothResourceAndReason() { } @Test - void createFromInvalidXmlFile_failsShowingBothPathAndReason(@TempDir Path benchmarkTestDir) throws IOException { - var invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; - var path = benchmarkTestDir.resolve("invalidBenchmarkConfig.xml"); - try (var in = getClass().getClassLoader().getResourceAsStream(invalidXmlBenchmarkConfigResource)) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + void createFromInvalidXmlFile_failsShowingBothPathAndReason() throws IOException { + final String invalidXmlBenchmarkConfigResource = "ai/timefold/solver/benchmark/api/invalidBenchmarkConfig.xml"; + File file = new File(benchmarkTestDir, "invalidBenchmarkConfig.xml"); + try (InputStream in = getClass().getClassLoader().getResourceAsStream(invalidXmlBenchmarkConfigResource)) { + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); } assertThatIllegalArgumentException() - .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlFile(path.toFile())) - .withMessageContaining(path.toString()) + .isThrownBy(() -> PlannerBenchmarkFactory.createFromXmlFile(file)) + .withMessageContaining(file.toString()) .withStackTraceContaining("invalidElementThatShouldNotBeHere"); } @Test void createFromXmlResource_uninitializedBestSolution() { - var benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( + PlannerBenchmarkConfig benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml"); - var solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); - var phaseConfig = new CustomPhaseConfig(); + SolverBenchmarkConfig solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); + CustomPhaseConfig phaseConfig = new CustomPhaseConfig(); phaseConfig.setCustomPhaseCommandClassList(Collections.singletonList(NoChangeCustomPhaseCommand.class)); solverBenchmarkConfig.getSolverConfig().setPhaseConfigList(Collections.singletonList(phaseConfig)); - var plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); + PlannerBenchmark plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test void createFromXmlResource_subSingleCount() { - var benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( + PlannerBenchmarkConfig benchmarkConfig = PlannerBenchmarkConfig.createFromXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml"); - var solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); + SolverBenchmarkConfig solverBenchmarkConfig = benchmarkConfig.getSolverBenchmarkConfigList().get(0); solverBenchmarkConfig.setSubSingleCount(3); - var plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); + PlannerBenchmark plannerBenchmark = PlannerBenchmarkFactory.create(benchmarkConfig).buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test - void createFromXmlFile(@TempDir Path benchmarkTestDir) throws IOException { - var path = benchmarkTestDir.resolve("testdataBenchmarkConfig.xml"); - try (var in = getClass().getClassLoader().getResourceAsStream( + void createFromXmlFile() throws IOException { + File file = new File(benchmarkTestDir, "testdataBenchmarkConfig.xml"); + try (InputStream in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfig.xml")) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); } - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(path.toFile()); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(file); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test - void createFromXmlFile_classLoader(@TempDir Path benchmarkTestDir) throws IOException { + void createFromXmlFile_classLoader() throws IOException { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - var path = benchmarkTestDir.resolve("classloaderTestdataBenchmarkConfig.xml"); - try (var in = getClass().getClassLoader().getResourceAsStream( + File file = new File(benchmarkTestDir, "classloaderTestdataBenchmarkConfig.xml"); + try (InputStream in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfig.xml")) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); } - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(path.toFile(), classLoader); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromXmlFile(file, classLoader); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -201,9 +214,9 @@ void createFromXmlFile_classLoader(@TempDir Path benchmarkTestDir) throws IOExce @Test void createFromFreemarkerXmlResource() { - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfigTemplate.xml.ftl"); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -212,10 +225,10 @@ void createFromFreemarkerXmlResource() { void createFromFreemarkerXmlResource_classLoader() { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlResource( "divertThroughClassLoader/ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfigTemplate.xml.ftl", classLoader); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -227,29 +240,30 @@ void createFromFreemarkerXmlResource_nonExisting() { } @Test - void createFromFreemarkerXmlFile(@TempDir Path benchmarkTestDir) throws IOException { - var path = benchmarkTestDir.resolve("testdataBenchmarkConfigTemplate.xml.ftl"); - try (var in = getClass().getClassLoader().getResourceAsStream( + void createFromFreemarkerXmlFile() throws IOException { + File file = new File(benchmarkTestDir, "testdataBenchmarkConfigTemplate.xml.ftl"); + try (InputStream in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/testdataBenchmarkConfigTemplate.xml.ftl")) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); } - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(path.toFile()); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(file); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @Test - void createFromFreemarkerXmlFile_classLoader(@TempDir Path benchmarkTestDir) throws IOException { + void createFromFreemarkerXmlFile_classLoader() throws IOException { // Mocking loadClass doesn't work well enough, because the className still differs from class.getName() ClassLoader classLoader = new DivertingClassLoader(getClass().getClassLoader()); - var path = benchmarkTestDir.resolve("classloaderTestdataBenchmarkConfigTemplate.xml.ftl"); - try (var in = getClass().getClassLoader().getResourceAsStream( + File file = new File(benchmarkTestDir, "classloaderTestdataBenchmarkConfigTemplate.xml.ftl"); + try (InputStream in = getClass().getClassLoader().getResourceAsStream( "ai/timefold/solver/benchmark/api/classloaderTestdataBenchmarkConfigTemplate.xml.ftl")) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); } - var plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(path.toFile(), classLoader); - var plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); + PlannerBenchmarkFactory plannerBenchmarkFactory = PlannerBenchmarkFactory.createFromFreemarkerXmlFile(file, + classLoader); + PlannerBenchmark plannerBenchmark = plannerBenchmarkFactory.buildPlannerBenchmark(); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.benchmark()).exists(); } @@ -259,17 +273,17 @@ void createFromFreemarkerXmlFile_classLoader(@TempDir Path benchmarkTestDir) thr // ************************************************************************ @Test - void createFromSolverConfig(@TempDir Path benchmarkTestDir) { - var solverConfig = PlannerTestUtils.buildSolverConfig(TestdataSolution.class, TestdataEntity.class); + void createFromSolverConfig() { + SolverConfig solverConfig = PlannerTestUtils.buildSolverConfig(TestdataSolution.class, TestdataEntity.class); - var solution = new TestdataSolution("s1"); + TestdataSolution solution = new TestdataSolution("s1"); solution.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); - var benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig); + PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); - benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig, benchmarkTestDir.toFile()); + benchmarkFactory = PlannerBenchmarkFactory.createFromSolverConfig(solverConfig, benchmarkOutputTestDir); assertThat(benchmarkFactory.buildPlannerBenchmark(solution)).isNotNull(); } @@ -279,8 +293,8 @@ void createFromSolverConfig(@TempDir Path benchmarkTestDir) { @Test void buildPlannerBenchmark() { - var benchmarkConfig = new PlannerBenchmarkConfig(); - var inheritedSolverConfig = new SolverBenchmarkConfig(); + PlannerBenchmarkConfig benchmarkConfig = new PlannerBenchmarkConfig(); + SolverBenchmarkConfig inheritedSolverConfig = new SolverBenchmarkConfig(); inheritedSolverConfig.setSolverConfig(new SolverConfig() .withSolutionClass(TestdataSolution.class) .withEntityClasses(TestdataEntity.class) @@ -290,16 +304,16 @@ void buildPlannerBenchmark() { benchmarkConfig.setSolverBenchmarkConfigList(Arrays.asList( new SolverBenchmarkConfig(), new SolverBenchmarkConfig(), new SolverBenchmarkConfig())); - var benchmarkFactory = PlannerBenchmarkFactory.create(benchmarkConfig); + PlannerBenchmarkFactory benchmarkFactory = PlannerBenchmarkFactory.create(benchmarkConfig); - var solution1 = new TestdataSolution("s1"); + TestdataSolution solution1 = new TestdataSolution("s1"); solution1.setEntityList(Arrays.asList(new TestdataEntity("e1"), new TestdataEntity("e2"), new TestdataEntity("e3"))); solution1.setValueList(Arrays.asList(new TestdataValue("v1"), new TestdataValue("v2"))); - var solution2 = new TestdataSolution("s2"); + TestdataSolution solution2 = new TestdataSolution("s2"); solution2.setEntityList(Arrays.asList(new TestdataEntity("e11"), new TestdataEntity("e12"), new TestdataEntity("e13"))); solution2.setValueList(Arrays.asList(new TestdataValue("v11"), new TestdataValue("v12"))); - var plannerBenchmark = + DefaultPlannerBenchmark plannerBenchmark = (DefaultPlannerBenchmark) benchmarkFactory.buildPlannerBenchmark(solution1, solution2); assertThat(plannerBenchmark).isNotNull(); assertThat(plannerBenchmark.getPlannerBenchmarkResult().getSolverBenchmarkResultList()).hasSize(3); From da42fca668a218db0ace659a937f66a940196ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 11:20:04 +0200 Subject: [PATCH 6/9] Fix copying --- .../solver/benchmark/impl/result/SubSingleBenchmarkResult.java | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java index dd3f5251491..b724c2e6390 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/result/SubSingleBenchmarkResult.java @@ -304,6 +304,7 @@ protected static SubSingleBenchmarkResult createMerge( // Skip oldResult.usedMemoryAfterInputSolution newResult.succeeded = oldResult.succeeded; newResult.score = oldResult.score; + newResult.initialized = oldResult.initialized; newResult.timeMillisSpent = oldResult.timeMillisSpent; newResult.scoreCalculationCount = oldResult.scoreCalculationCount; newResult.moveEvaluationCount = oldResult.moveEvaluationCount; From 67e71fc3b3207b109de31782fa8532be4dceae7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 11:23:08 +0200 Subject: [PATCH 7/9] Add TODO --- .../solver/core/api/solver/event/BestSolutionChangedEvent.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java b/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java index 083a231bb8c..4cc2f6a0056 100644 --- a/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java +++ b/core/src/main/java/ai/timefold/solver/core/api/solver/event/BestSolutionChangedEvent.java @@ -15,6 +15,7 @@ * * @param the solution type, the class with the {@link PlanningSolution} annotation */ +// TODO In Solver 2.0, maybe convert this to an interface. public class BestSolutionChangedEvent extends EventObject { private final Solver solver; From 79f61a7e85102c712101bb3745e2e541e0090526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 15:35:32 +0200 Subject: [PATCH 8/9] Fix --- .../impl/statistic/StatisticRegistry.java | 11 ++++++----- .../benchmark/api/PlannerBenchmarkTest.java | 4 ++-- .../impl/solver/monitoring/SolverMetricUtil.java | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java index c5dbfa7c7ef..cc880c94a1a 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java @@ -91,15 +91,16 @@ public Set getMeterIds(SolverMetric metric, Tags runId) { public void extractScoreFromMeters(SolverMetric metric, Tags runId, Consumer> scoreConsumer) { var score = SolverMetricUtil.extractScore(metric, scoreDefinition, id -> { - var gaugeId = metric.getMeterId() + "." + id; - var scoreLevelGauge = this.find(gaugeId).tags(runId).gauge(); + var scoreLevelGauge = this.find(id).tags(runId).gauge(); if (scoreLevelGauge != null && Double.isFinite(scoreLevelGauge.value())) { return scoreLevelNumberConverter.apply(scoreLevelGauge.value()); } else { - return scoreLevelNumberConverter.apply(0.0); + return null; } }); - scoreConsumer.accept(score); + if (score != null) { + scoreConsumer.accept(score); + } } @SuppressWarnings({ "rawtypes", "unchecked" }) @@ -116,7 +117,7 @@ public void extractConstraintSummariesFromMeters(SolverMetric metric, Tags runId // Get the score from the corresponding constraint package and constraint name meters extractScoreFromMeters(metric, constraintMatchTotalRunId, // Get the count gauge (add constraint package and constraint name to the run tags) - score -> getGaugeValue(metric.getMeterId() + ".count", constraintMatchTotalRunId, + score -> getGaugeValue(SolverMetricUtil.getGaugeName(metric, "count"), constraintMatchTotalRunId, count -> constraintMatchTotalConsumer .accept(new ConstraintSummary(constraintRef, score.raw(), count.intValue())))); }); diff --git a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java index 470fb323855..f4c906c6c25 100644 --- a/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java +++ b/benchmark/src/test/java/ai/timefold/solver/benchmark/api/PlannerBenchmarkTest.java @@ -55,7 +55,7 @@ void runPlannerBenchmark(@TempDir Path benchmarkTestDir) { try (var lines = Files.lines(csv)) { var lineList = lines.toList(); - assertThat(lineList).isNotEmpty(); + assertThat(lineList).hasSizeGreaterThan(1); assertSoftly(softly -> { // Proper header. softly.assertThat(lineList) @@ -70,7 +70,7 @@ void runPlannerBenchmark(@TempDir Path benchmarkTestDir) { .last() .asString() .endsWith(""" - "0","true" + "-3","true" """.trim()); }); } catch (IOException e) { diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java index fafaaaa64d5..33505b540e4 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java @@ -10,6 +10,7 @@ import ai.timefold.solver.core.impl.score.director.InnerScore; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Tags; @@ -55,19 +56,26 @@ private static String[] getLevelLabels(ScoreDefinition scoreDefinition) { return labelNames; } - private static String getGaugeName(SolverMetric metric, String label) { + public static String getGaugeName(SolverMetric metric, String label) { return metric.getMeterId() + "." + label; } - public static > InnerScore extractScore(SolverMetric metric, - ScoreDefinition scoreDefinition, Function scoreLevelFunction) { + public static > @Nullable InnerScore extractScore(SolverMetric metric, + ScoreDefinition scoreDefinition, Function scoreLevelFunction) { var levelLabels = getLevelLabels(scoreDefinition); var levelNumbers = new Number[levelLabels.length]; for (var i = 0; i < levelLabels.length; i++) { - levelNumbers[i] = scoreLevelFunction.apply(getGaugeName(metric, levelLabels[i])); + var levelNumber = scoreLevelFunction.apply(getGaugeName(metric, levelLabels[i])); + if (levelNumber == null) { + return null; + } + levelNumbers[i] = levelNumber; } var score = scoreDefinition.fromLevelNumbers(levelNumbers); var unassignedCount = scoreLevelFunction.apply(getGaugeName(metric, UNASSIGNED_COUNT_LABEL)); + if (unassignedCount == null) { + return null; + } return InnerScore.withUnassignedCount(score, unassignedCount.intValue()); } From 44be703aeadf4f8537614330d478884551704786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Petrovick=C3=BD?= Date: Mon, 9 Jun 2025 17:03:27 +0200 Subject: [PATCH 9/9] Review --- .../impl/statistic/StatisticRegistry.java | 22 +++++++------------ .../BestScoreSubSingleStatistic.java | 2 +- ...estSolutionMutationSubSingleStatistic.java | 10 ++++++--- ...actCalculationSpeedSubSingleStatistic.java | 6 +++-- .../MemoryUseSubSingleStatistic.java | 10 +++++---- .../MoveCountPerStepSubSingleStatistic.java | 17 ++++++++++---- .../StepScoreSubSingleStatistic.java | 2 +- .../solver/core/impl/phase/AbstractPhase.java | 4 ++-- .../impl/score/DefaultScoreExplanation.java | 4 ++-- .../core/impl/score/director/InnerScore.java | 7 ++++-- .../score/director/InnerScoreDirector.java | 2 +- .../DefaultBestSolutionChangedEvent.java | 2 +- .../impl/solver/monitoring/ScoreLevels.java | 10 ++++----- .../solver/monitoring/SolverMetricUtil.java | 18 +++++++++++++-- .../PickedMoveBestScoreDiffStatistic.java | 2 +- .../PickedMoveStepScoreDiffStatistic.java | 2 +- .../solver/recaller/BestSolutionRecaller.java | 4 ++-- .../core/impl/solver/scope/SolverScope.java | 2 +- .../BestScoreFeasibleTermination.java | 4 ++-- .../config/solver/EnvironmentModeTest.java | 2 +- .../AbstractScoreDirectorSemanticsTest.java | 4 ++-- 21 files changed, 82 insertions(+), 54 deletions(-) diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java index cc880c94a1a..1057665a2e9 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/StatisticRegistry.java @@ -117,23 +117,17 @@ public void extractConstraintSummariesFromMeters(SolverMetric metric, Tags runId // Get the score from the corresponding constraint package and constraint name meters extractScoreFromMeters(metric, constraintMatchTotalRunId, // Get the count gauge (add constraint package and constraint name to the run tags) - score -> getGaugeValue(SolverMetricUtil.getGaugeName(metric, "count"), constraintMatchTotalRunId, - count -> constraintMatchTotalConsumer - .accept(new ConstraintSummary(constraintRef, score.raw(), count.intValue())))); + score -> { + var count = SolverMetricUtil.getGaugeValue(this, SolverMetricUtil.getGaugeName(metric, "count"), + constraintMatchTotalRunId); + if (count != null) { + constraintMatchTotalConsumer + .accept(new ConstraintSummary(constraintRef, score.raw(), count.intValue())); + } + }); }); } - public void getGaugeValue(SolverMetric metric, Tags runId, Consumer gaugeConsumer) { - getGaugeValue(metric.getMeterId(), runId, gaugeConsumer); - } - - public void getGaugeValue(String meterId, Tags runId, Consumer gaugeConsumer) { - var gauge = this.find(meterId).tags(runId).gauge(); - if (gauge != null && Double.isFinite(gauge.value())) { - gaugeConsumer.accept(gauge.value()); - } - } - public void extractMoveCountPerType(SolverScope solverScope, ObjLongConsumer gaugeConsumer) { solverScope.getMoveCountTypes().forEach(type -> { var gauge = this.find(SolverMetric.MOVE_COUNT_PER_TYPE.getMeterId() + "." + type) diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java index 210b5a88570..1edee6ffe3c 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestscore/BestScoreSubSingleStatistic.java @@ -32,7 +32,7 @@ public void open(StatisticRegistry registry, Tags runTag) { registry.addListener(SolverMetric.BEST_SCORE, timestamp -> registry.extractScoreFromMeters(SolverMetric.BEST_SCORE, runTag, score -> pointList - .add(new BestScoreStatisticPoint(timestamp, score.raw(), score.fullyAssigned())))); + .add(new BestScoreStatisticPoint(timestamp, score.raw(), score.isFullyAssigned())))); } // ************************************************************************ diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestsolutionmutation/BestSolutionMutationSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestsolutionmutation/BestSolutionMutationSubSingleStatistic.java index 3bdbe9072c0..7b2cb651dc9 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestsolutionmutation/BestSolutionMutationSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/bestsolutionmutation/BestSolutionMutationSubSingleStatistic.java @@ -9,6 +9,7 @@ import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; @@ -30,9 +31,12 @@ public BestSolutionMutationSubSingleStatistic(SubSingleBenchmarkResult subSingle @Override public void open(StatisticRegistry registry, Tags runTag) { registry.addListener(SolverMetric.BEST_SOLUTION_MUTATION, - timestamp -> registry.getGaugeValue(SolverMetric.BEST_SOLUTION_MUTATION, runTag, - mutationCount -> pointList - .add(new BestSolutionMutationStatisticPoint(timestamp, mutationCount.intValue())))); + timestamp -> { + var mutationCount = SolverMetricUtil.getGaugeValue(registry, SolverMetric.BEST_SOLUTION_MUTATION, runTag); + if (mutationCount != null) { + pointList.add(new BestSolutionMutationStatisticPoint(timestamp, mutationCount.intValue())); + } + }); } // ************************************************************************ diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/common/AbstractCalculationSpeedSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/common/AbstractCalculationSpeedSubSingleStatistic.java index feffe454cd4..f400fc51df7 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/common/AbstractCalculationSpeedSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/common/AbstractCalculationSpeedSubSingleStatistic.java @@ -10,6 +10,7 @@ import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; @@ -44,7 +45,8 @@ public void open(StatisticRegistry registry, Tags runTag) { @Override public void accept(Long timeMillisSpent) { if (timeMillisSpent >= nextTimeMillisThreshold) { - registry.getGaugeValue(solverMetric, runTag, countNumber -> { + var countNumber = SolverMetricUtil.getGaugeValue(registry, solverMetric, runTag); + if (countNumber != null) { var moveEvaluationCount = countNumber.longValue(); var countInterval = moveEvaluationCount - lastCalculationCount.get(); var timeMillisSpentInterval = timeMillisSpent - lastTimeMillisSpent; @@ -55,7 +57,7 @@ public void accept(Long timeMillisSpent) { var speed = countInterval * 1000L / timeMillisSpentInterval; pointList.add(new LongStatisticPoint(timeMillisSpent, speed)); lastCalculationCount.set(moveEvaluationCount); - }); + } lastTimeMillisSpent = timeMillisSpent; nextTimeMillisThreshold += timeMillisThresholdInterval; if (nextTimeMillisThreshold < timeMillisSpent) { diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/memoryuse/MemoryUseSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/memoryuse/MemoryUseSubSingleStatistic.java index ef600b9b737..0ef46a2170b 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/memoryuse/MemoryUseSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/memoryuse/MemoryUseSubSingleStatistic.java @@ -10,6 +10,7 @@ import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; @@ -58,10 +59,11 @@ public MemoryUseSubSingleStatisticListener(StatisticRegistry registry, Tags t @Override public void accept(Long timeMillisSpent) { if (timeMillisSpent >= nextTimeMillisThreshold) { - registry.getGaugeValue(SolverMetric.MEMORY_USE, tags, - memoryUse -> pointList.add( - new MemoryUseStatisticPoint(timeMillisSpent, memoryUse.longValue(), - (long) registry.find("jvm.memory.max").tags(tags).gauge().value()))); + var memoryUse = SolverMetricUtil.getGaugeValue(registry, SolverMetric.MEMORY_USE, tags); + if (memoryUse != null) { + var max = SolverMetricUtil.getGaugeValue(registry, "jvm.memory.max", tags); + pointList.add(new MemoryUseStatisticPoint(timeMillisSpent, memoryUse.longValue(), max.longValue())); + } nextTimeMillisThreshold += timeMillisThresholdInterval; if (nextTimeMillisThreshold < timeMillisSpent) { diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/movecountperstep/MoveCountPerStepSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/movecountperstep/MoveCountPerStepSubSingleStatistic.java index 061977e84be..533d5288fa7 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/movecountperstep/MoveCountPerStepSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/movecountperstep/MoveCountPerStepSubSingleStatistic.java @@ -9,6 +9,7 @@ import ai.timefold.solver.benchmark.impl.statistic.StatisticRegistry; import ai.timefold.solver.core.config.solver.monitoring.SolverMetric; import ai.timefold.solver.core.impl.score.definition.ScoreDefinition; +import ai.timefold.solver.core.impl.solver.monitoring.SolverMetricUtil; import io.micrometer.core.instrument.Tags; @@ -30,10 +31,18 @@ public MoveCountPerStepSubSingleStatistic(SubSingleBenchmarkResult subSingleBenc @Override public void open(StatisticRegistry registry, Tags runTag) { registry.addListener(SolverMetric.MOVE_COUNT_PER_STEP, - timeMillisSpent -> registry.getGaugeValue(SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".accepted", runTag, - accepted -> registry.getGaugeValue(SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".selected", runTag, - selected -> pointList.add(new MoveCountPerStepStatisticPoint(timeMillisSpent, - accepted.longValue(), selected.longValue()))))); + timeMillisSpent -> { + var accepted = SolverMetricUtil.getGaugeValue(registry, + SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".accepted", runTag); + if (accepted != null) { + var selected = SolverMetricUtil.getGaugeValue(registry, + SolverMetric.MOVE_COUNT_PER_STEP.getMeterId() + ".selected", runTag); + if (selected != null) { + pointList.add(new MoveCountPerStepStatisticPoint(timeMillisSpent, accepted.longValue(), + selected.longValue())); + } + } + }); } // ************************************************************************ diff --git a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java index 118b612ffd2..efa63c407ad 100644 --- a/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java +++ b/benchmark/src/main/java/ai/timefold/solver/benchmark/impl/statistic/stepscore/StepScoreSubSingleStatistic.java @@ -32,7 +32,7 @@ public void open(StatisticRegistry registry, Tags runTag) { registry.addListener(SolverMetric.STEP_SCORE, timeMillisSpent -> registry.extractScoreFromMeters(SolverMetric.STEP_SCORE, runTag, score -> pointList - .add(new StepScoreStatisticPoint(timeMillisSpent, score.raw(), score.fullyAssigned())))); + .add(new StepScoreStatisticPoint(timeMillisSpent, score.raw(), score.isFullyAssigned())))); } // ************************************************************************ diff --git a/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java b/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java index 2198300d592..fac2ae9e9b5 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/phase/AbstractPhase.java @@ -199,7 +199,7 @@ public void stepEnded(AbstractStepScope stepScope) { private static void collectMetrics(AbstractStepScope stepScope) { var solverScope = stepScope.getPhaseScope().getSolverScope(); - if (solverScope.isMetricEnabled(SolverMetric.STEP_SCORE) && stepScope.getScore().fullyAssigned()) { + if (solverScope.isMetricEnabled(SolverMetric.STEP_SCORE) && stepScope.getScore().isFullyAssigned()) { Tags tags = solverScope.getMonitoringTags(); ScoreDefinition scoreDefinition = solverScope.getScoreDefinition(); Map tagToScoreLevels = solverScope.getStepScoreMap(); @@ -223,7 +223,7 @@ public void removePhaseLifecycleListener(PhaseLifecycleListener phase // ************************************************************************ protected void assertWorkingSolutionInitialized(AbstractPhaseScope phaseScope) { - if (!phaseScope.getStartingScore().fullyAssigned()) { + if (!phaseScope.getStartingScore().isFullyAssigned()) { var scoreDirector = phaseScope.getScoreDirector(); var solutionDescriptor = scoreDirector.getSolutionDescriptor(); var workingSolution = scoreDirector.getWorkingSolution(); diff --git a/core/src/main/java/ai/timefold/solver/core/impl/score/DefaultScoreExplanation.java b/core/src/main/java/ai/timefold/solver/core/impl/score/DefaultScoreExplanation.java index c16d7d32102..3b5a09e2f63 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/score/DefaultScoreExplanation.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/score/DefaultScoreExplanation.java @@ -49,7 +49,7 @@ public static > String explainScore(InnerScore> constraintMatchTotalComparator = comparing(ConstraintMatchTotal::getScore); Comparator> constraintMatchComparator = comparing(ConstraintMatch::getScore); @@ -163,7 +163,7 @@ public DefaultScoreExplanation(Solution_ solution, InnerScore innerScore @Override public boolean isInitialized() { - return innerScore.fullyAssigned(); + return innerScore.isFullyAssigned(); } @Override diff --git a/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScore.java b/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScore.java index 0a9c84cd3e4..3bf634e3902 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScore.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScore.java @@ -7,6 +7,8 @@ import ai.timefold.solver.core.api.domain.variable.PlanningVariable; import ai.timefold.solver.core.api.score.Score; +import org.jspecify.annotations.NullMarked; + /** * Carries information on if the {@link PlanningSolution} of this score was fully initialized when it was calculated. * This only works for solutions where: @@ -19,6 +21,7 @@ * * For solutions which do allow unassigning values, {@link #unassignedCount} is always zero. */ +@NullMarked public record InnerScore>(Score_ raw, int unassignedCount) implements Comparable> { @@ -39,7 +42,7 @@ public static > InnerScore withUnassignedCo } } - public boolean fullyAssigned() { + public boolean isFullyAssigned() { return unassignedCount == 0; } @@ -55,6 +58,6 @@ public int compareTo(InnerScore other) { @Override public String toString() { - return fullyAssigned() ? raw.toString() : "-%dinit/%s".formatted(unassignedCount, raw); + return isFullyAssigned() ? raw.toString() : "-%dinit/%s".formatted(unassignedCount, raw); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScoreDirector.java b/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScoreDirector.java index 4ca62d2dbb7..a7c3d7b033c 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScoreDirector.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScoreDirector.java @@ -367,7 +367,7 @@ default ScoreAnalysis buildScoreAnalysis(ScoreAnalysisFetchPolicy scoreA var constraintAnalysis = getConstraintAnalysis(constraintMatchTotal, scoreAnalysisFetchPolicy); constraintAnalysisMap.put(constraintMatchTotal.getConstraintRef(), constraintAnalysis); } - return new ScoreAnalysis<>(state.raw(), constraintAnalysisMap, state.fullyAssigned()); + return new ScoreAnalysis<>(state.raw(), constraintAnalysisMap, state.isFullyAssigned()); } /* diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java index 5c21219676c..ecfa39ba6b7 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/event/DefaultBestSolutionChangedEvent.java @@ -12,7 +12,7 @@ public final class DefaultBestSolutionChangedEvent extends BestSoluti public DefaultBestSolutionChangedEvent(@NonNull Solver solver, long timeMillisSpent, @NonNull Solution_ newBestSolution, @NonNull InnerScore newBestScore) { - super(solver, timeMillisSpent, newBestSolution, newBestScore.raw(), newBestScore.fullyAssigned()); + super(solver, timeMillisSpent, newBestSolution, newBestScore.raw(), newBestScore.isFullyAssigned()); this.unassignedCount = newBestScore.unassignedCount(); } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java index 7113e3b4e0d..0cee9dd5066 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/ScoreLevels.java @@ -7,16 +7,16 @@ public final class ScoreLevels { final AtomicReference[] levelValues; - final AtomicInteger unnassignedCount; + final AtomicInteger unassignedCount; @SuppressWarnings("unchecked") - ScoreLevels(int unnassignedCount, Number[] levelValues) { + ScoreLevels(int unassignedCount, Number[] levelValues) { // We store the values inside a constant reference, // so that the metric can always load the latest value. // If we stored the value directly and just overwrote it, // the metric would always hold a reference to the old value, // effectively ignoring the update. - this.unnassignedCount = new AtomicInteger(unnassignedCount); + this.unassignedCount = new AtomicInteger(unassignedCount); this.levelValues = Arrays.stream(levelValues) .map(AtomicReference::new) .toArray(AtomicReference[]::new); @@ -26,8 +26,8 @@ void setLevelValue(int level, Number value) { levelValues[level].set(value); } - void setUnnassignedCount(int unnassignedCount) { - this.unnassignedCount.set(unnassignedCount); + void setUnassignedCount(int unassignedCount) { + this.unassignedCount.set(unassignedCount); } } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java index 33505b540e4..145a93589aa 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/SolverMetricUtil.java @@ -12,6 +12,7 @@ import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; +import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Tags; @@ -27,7 +28,7 @@ public static > void registerScore(SolverMetric met if (tagToScoreLevels.containsKey(tags)) { // Set new score levels for the previously registered gauges to read. var scoreLevels = tagToScoreLevels.get(tags); - scoreLevels.setUnnassignedCount(innerScore.unassignedCount()); + scoreLevels.setUnassignedCount(innerScore.unassignedCount()); for (var i = 0; i < levelValues.length; i++) { scoreLevels.setLevelValue(i, levelValues[i]); } @@ -39,7 +40,7 @@ public static > void registerScore(SolverMetric met tagToScoreLevels.put(tags, result); // Register the gauges to read the score levels. - Metrics.gauge(getGaugeName(metric, UNASSIGNED_COUNT_LABEL), tags, result.unnassignedCount, + Metrics.gauge(getGaugeName(metric, UNASSIGNED_COUNT_LABEL), tags, result.unassignedCount, AtomicInteger::doubleValue); for (var i = 0; i < levelValues.length; i++) { Metrics.gauge(getGaugeName(metric, levelLabels[i]), tags, result.levelValues[i], @@ -60,6 +61,19 @@ public static String getGaugeName(SolverMetric metric, String label) { return metric.getMeterId() + "." + label; } + public static @Nullable Double getGaugeValue(MeterRegistry registry, SolverMetric metric, Tags runId) { + return getGaugeValue(registry, metric.getMeterId(), runId); + } + + public static @Nullable Double getGaugeValue(MeterRegistry registry, String meterId, Tags runId) { + var gauge = registry.find(meterId).tags(runId).gauge(); + if (gauge != null && Double.isFinite(gauge.value())) { + return gauge.value(); + } else { + return null; + } + } + public static > @Nullable InnerScore extractScore(SolverMetric metric, ScoreDefinition scoreDefinition, Function scoreLevelFunction) { var levelLabels = getLevelLabels(scoreDefinition); diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java index 61aa00fa4c6..2c0682f3957 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveBestScoreDiffStatistic.java @@ -81,7 +81,7 @@ private void localSearchStepEnded(LocalSearchStepScope stepScope) { var newBestScore = stepScope. getScore().raw(); var bestScoreDiff = newBestScore.subtract(oldBestScore); oldBestScore = newBestScore; - Tags tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() + var tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() .and("move.type", moveType); SolverMetricUtil.registerScore(SolverMetric.PICKED_MOVE_TYPE_BEST_SCORE_DIFF, tags, scoreDefinition, tagsToMoveScoreMap, InnerScore.fullyAssigned(bestScoreDiff)); diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java index 8c3e47d7406..abc61379a5f 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/monitoring/statistic/PickedMoveStepScoreDiffStatistic.java @@ -83,7 +83,7 @@ private void localSearchStepEnded(LocalSearchStepScope stepScope) { var stepScoreDiff = newStepScore.subtract(oldStepScore); oldStepScore = newStepScore; - Tags tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() + var tags = stepScope.getPhaseScope().getSolverScope().getMonitoringTags() .and("move.type", moveType); SolverMetricUtil.registerScore(SolverMetric.PICKED_MOVE_TYPE_STEP_SCORE_DIFF, tags, scoreDefinition, tagsToMoveScoreMap, InnerScore.fullyAssigned(stepScoreDiff)); diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/recaller/BestSolutionRecaller.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/recaller/BestSolutionRecaller.java index ac7fe8b5808..401309d08c6 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/recaller/BestSolutionRecaller.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/recaller/BestSolutionRecaller.java @@ -54,7 +54,7 @@ public void solvingStarted(SolverScope solverScope) { solverScope.setBestSolutionTimeMillis(solverScope.getClock().millis()); // The original bestSolution might be the final bestSolution and should have an accurate Score solverScope.getSolutionDescriptor().setScore(solverScope.getBestSolution(), score); - if (innerScore.fullyAssigned()) { + if (innerScore.isFullyAssigned()) { solverScope.setStartingInitializedScore(innerScore.raw()); } else { solverScope.setStartingInitializedScore(null); @@ -145,7 +145,7 @@ private void updateBestSolutionWithoutFiring(SolverScope solverScope) private void updateBestSolutionWithoutFiring(SolverScope solverScope, InnerScore bestScore, Solution_ bestSolution) { - if (bestScore.fullyAssigned() && !solverScope.isBestSolutionInitialized()) { + if (bestScore.isFullyAssigned() && !solverScope.isBestSolutionInitialized()) { solverScope.setStartingInitializedScore(bestScore.raw()); } diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java index 4c0de5ae169..ca578f2b720 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/scope/SolverScope.java @@ -290,7 +290,7 @@ public void endingNow() { } public boolean isBestSolutionInitialized() { - return getBestScore().fullyAssigned(); + return getBestScore().isFullyAssigned(); } public long calculateTimeMillisSpentUpToNow() { diff --git a/core/src/main/java/ai/timefold/solver/core/impl/solver/termination/BestScoreFeasibleTermination.java b/core/src/main/java/ai/timefold/solver/core/impl/solver/termination/BestScoreFeasibleTermination.java index a66eb29f4ac..b87259e83af 100644 --- a/core/src/main/java/ai/timefold/solver/core/impl/solver/termination/BestScoreFeasibleTermination.java +++ b/core/src/main/java/ai/timefold/solver/core/impl/solver/termination/BestScoreFeasibleTermination.java @@ -40,7 +40,7 @@ public boolean isPhaseTerminated(AbstractPhaseScope phaseScope) { } private static boolean isTerminated(InnerScore innerScore) { - return innerScore.fullyAssigned() && innerScore.raw().isFeasible(); + return innerScore.isFullyAssigned() && innerScore.raw().isFeasible(); } @SuppressWarnings({ "unchecked", "rawtypes" }) @@ -57,7 +57,7 @@ public double calculatePhaseTimeGradient(AbstractPhaseScope phaseScop > double calculateFeasibilityTimeGradient(@Nullable InnerScore innerStartScore, Score_ score) { - if (innerStartScore == null || !innerStartScore.fullyAssigned()) { + if (innerStartScore == null || !innerStartScore.isFullyAssigned()) { return 0.0; } var startScore = innerStartScore.raw(); diff --git a/core/src/test/java/ai/timefold/solver/core/config/solver/EnvironmentModeTest.java b/core/src/test/java/ai/timefold/solver/core/config/solver/EnvironmentModeTest.java index 84fcaa38f08..9441f71dc2f 100644 --- a/core/src/test/java/ai/timefold/solver/core/config/solver/EnvironmentModeTest.java +++ b/core/src/test/java/ai/timefold/solver/core/config/solver/EnvironmentModeTest.java @@ -289,7 +289,7 @@ public void changeWorkingSolution(ScoreDirector scoreDirector, (InnerScoreDirector) scoreDirector; var score = innerScoreDirector.calculateScore(); - if (!score.fullyAssigned()) { + if (!score.isFullyAssigned()) { throw new IllegalStateException("The solution (" + TestdataEntity.class.getSimpleName() + ") was not fully initialized by CustomSolverPhase: (" + this.getClass().getCanonicalName() + ")"); diff --git a/core/src/test/java/ai/timefold/solver/core/impl/score/director/AbstractScoreDirectorSemanticsTest.java b/core/src/test/java/ai/timefold/solver/core/impl/score/director/AbstractScoreDirectorSemanticsTest.java index a113a937744..42fcd2a2da5 100644 --- a/core/src/test/java/ai/timefold/solver/core/impl/score/director/AbstractScoreDirectorSemanticsTest.java +++ b/core/src/test/java/ai/timefold/solver/core/impl/score/director/AbstractScoreDirectorSemanticsTest.java @@ -145,7 +145,7 @@ void constraintPresentEvenIfNoMatches() { scoreDirector.setWorkingSolution(solution); var score1 = scoreDirector.calculateScore(); assertSoftly(softly -> { - softly.assertThat(score1.fullyAssigned()).isTrue(); + softly.assertThat(score1.isFullyAssigned()).isTrue(); softly.assertThat(score1.raw().score()).isEqualTo(1); softly.assertThat(scoreDirector.getConstraintMatchTotalMap()) .containsOnlyKeys("ai.timefold.solver.core.testdomain.constraintconfiguration/First weight"); @@ -158,7 +158,7 @@ void constraintPresentEvenIfNoMatches() { scoreDirector.afterVariableChanged(entity, "value"); var score2 = scoreDirector.calculateScore(); assertSoftly(softly -> { - softly.assertThat(score2.fullyAssigned()).isFalse(); + softly.assertThat(score2.isFullyAssigned()).isFalse(); softly.assertThat(score2.raw().score()).isZero(); softly.assertThat(scoreDirector.getConstraintMatchTotalMap()) .containsOnlyKeys("ai.timefold.solver.core.testdomain.constraintconfiguration/First weight");