feat: new SolutionUpdatePolicy to force update of shadow vars

Author: triceo

core/src/main/java/ai/timefold/solver/core/api/solver/SolutionManager.java

@@ -111,8 +111,8 @@ static <Solution_> void updateShadowVariables(@NonNull Class<Solution_> solution
     }
 
     /**
-     * Same as {@link #updateShadowVariables(Class, Object...)},
-     * this method accepts a solution rather than a list of entities.
+     * Equivalent to {@link SolutionManager#update(Object, SolutionUpdatePolicy)}
+     * with {@link SolutionUpdatePolicy#RESET_SHADOW_VARIABLES_ONLY}.
      *
      * @param solution the solution
      */

core/src/main/java/ai/timefold/solver/core/api/solver/SolutionUpdatePolicy.java

@@ -40,11 +40,23 @@ public enum SolutionUpdatePolicy {
     /**
      * Runs variable listeners on all planning entities and problem facts,
      * updates shadow variables.
+     * Assumes an uninitialized solution;
+     * for solutions where some shadow variables are already filled in,
+     * use {@link #RESET_SHADOW_VARIABLES_ONLY} instead.
+     * <p>
      * Does not update score;
      * the solution will keep the current score, even if it is stale or null.
      * To avoid this, use {@link #UPDATE_ALL} instead.
      */
     UPDATE_SHADOW_VARIABLES_ONLY(false, true),
+    /**
+     * A specialized variant of {@link #UPDATE_SHADOW_VARIABLES_ONLY}, which clears the existing shadow variables first.
+     * This is significantly slower than the former,
+     * but it avoids an issue with dependent shadow variables not being recomputed
+     * if their source value did not change.
+     * This is only useful for solutions where some shadow variables are already filled in.
+     */
+    RESET_SHADOW_VARIABLES_ONLY(false, true),
     /**
      * Does not run anything.
      * Improves performance during {@link SolutionManager#analyze(Object, ScoreAnalysisFetchPolicy, SolutionUpdatePolicy)}
@@ -75,4 +87,9 @@ public boolean isScoreUpdateEnabled() {
     public boolean isShadowVariableUpdateEnabled() {
         return shadowVariableUpdateEnabled;
     }
+
+    public boolean isShadowVariableUpdateForced() {
+        return isShadowVariableUpdateEnabled() && this == RESET_SHADOW_VARIABLES_ONLY;
+    }
+
 }

core/src/main/java/ai/timefold/solver/core/impl/domain/solution/descriptor/SolutionDescriptor.java

@@ -99,6 +99,16 @@ public class SolutionDescriptor<Solution_> {
             PlanningEntityCollectionProperty.class,
             PlanningScore.class };
 
+    @SuppressWarnings("unchecked")
+    public static <Solution_> SolutionDescriptor<Solution_> buildSolutionDescriptor(Solution_ solution) {
+        var enabledPreviewFeatures = EnumSet.of(PreviewFeature.DECLARATIVE_SHADOW_VARIABLES);
+        var solutionClass = (Class<Solution_>) solution.getClass();
+        var initialSolutionDescriptor = buildSolutionDescriptor(enabledPreviewFeatures, solutionClass);
+        var entityClassSet = new LinkedHashSet<Class<?>>();
+        initialSolutionDescriptor.visitAllEntities(solution, e -> entityClassSet.add(e.getClass()));
+        return buildSolutionDescriptor(enabledPreviewFeatures, solutionClass, entityClassSet.toArray(new Class<?>[0]));
+    }
+
     public static <Solution_> SolutionDescriptor<Solution_> buildSolutionDescriptor(Class<Solution_> solutionClass,
             Class<?>... entityClasses) {
         return buildSolutionDescriptor(solutionClass, Arrays.asList(entityClasses));

core/src/main/java/ai/timefold/solver/core/impl/domain/variable/ShadowVariableUpdateHelper.java

@@ -52,6 +52,13 @@
  */
 public final class ShadowVariableUpdateHelper<Solution_> {
 
+    /**
+     * Deducing the solution descriptors from the solution instance is very expensive.
+     * Cache the solution descriptors to avoid this,
+     * even at the unlikely expense of keeping solution descriptors in memory for solutions which are no longer used.
+     */
+    private static final IdentityHashMap<Class<?>, SolutionDescriptor<?>> SOLUTION_DESCRIPTOR_CACHE = new IdentityHashMap<>();
+
     private static final EnumSet<ShadowVariableType> SUPPORTED_TYPES =
             EnumSet.of(BASIC, CUSTOM_LISTENER, CASCADING_UPDATE, DECLARATIVE);
 
@@ -74,21 +81,12 @@ private ShadowVariableUpdateHelper(EnumSet<ShadowVariableType> supportedShadowVa
 
     @SuppressWarnings("unchecked")
     public void updateShadowVariables(Solution_ solution) {
-        var enabledPreviewFeatures = EnumSet.of(PreviewFeature.DECLARATIVE_SHADOW_VARIABLES);
-        var solutionClass = (Class<Solution_>) solution.getClass();
-        var initialSolutionDescriptor = SolutionDescriptor.buildSolutionDescriptor(
-                enabledPreviewFeatures, solutionClass);
-        var entityClassArray = initialSolutionDescriptor.getAllEntitiesAndProblemFacts(solution)
-                .stream()
-                .map(Object::getClass)
-                .distinct()
-                .toArray(Class[]::new);
-        var solutionDescriptor = SolutionDescriptor.buildSolutionDescriptor(enabledPreviewFeatures, solutionClass,
-                entityClassArray);
+        var solutionDescriptor = (SolutionDescriptor<Solution_>) SOLUTION_DESCRIPTOR_CACHE.computeIfAbsent(solution.getClass(),
+                solutionClass -> SolutionDescriptor.buildSolutionDescriptor(solution));
         try (var scoreDirector = new InternalScoreDirector<>(solutionDescriptor)) {
             // When we have a solution, we can reuse the logic from VariableListenerSupport to update all variable types
             scoreDirector.setWorkingSolution(solution);
-            scoreDirector.forceTriggerVariableListeners();
+            scoreDirector.forceTriggerVariableListeners(true);
         }
     }
 

core/src/main/java/ai/timefold/solver/core/impl/score/director/AbstractScoreDirector.java

@@ -2,6 +2,7 @@
 
 import static java.util.Objects.requireNonNull;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.IdentityHashMap;
 import java.util.LinkedHashSet;
@@ -27,6 +28,7 @@
 import ai.timefold.solver.core.impl.domain.variable.ListVariableStateSupply;
 import ai.timefold.solver.core.impl.domain.variable.descriptor.ListVariableDescriptor;
 import ai.timefold.solver.core.impl.domain.variable.descriptor.VariableDescriptor;
+import ai.timefold.solver.core.impl.domain.variable.inverserelation.InverseRelationShadowVariableDescriptor;
 import ai.timefold.solver.core.impl.domain.variable.listener.support.VariableListenerSupport;
 import ai.timefold.solver.core.impl.domain.variable.listener.support.violation.SolutionTracker;
 import ai.timefold.solver.core.impl.domain.variable.supply.SupplyManager;
@@ -358,8 +360,52 @@ protected void clearVariableListenerEvents() {
     }
 
     @Override
-    public void forceTriggerVariableListeners() {
-        variableListenerSupport.forceTriggerAllVariableListeners(getWorkingSolution());
+    public void forceTriggerVariableListeners(boolean resetShadowVariablesFirst) {
+        var solution = getWorkingSolution();
+        if (resetShadowVariablesFirst) {
+            var solutionDescriptor = getSolutionDescriptor();
+            solutionDescriptor.visitAllEntities(solution, entity -> {
+                var entityDescriptor = solutionDescriptor.findEntityDescriptor(entity.getClass());
+                var shadowVariableDescriptors = entityDescriptor.getShadowVariableDescriptors();
+                if (shadowVariableDescriptors.isEmpty()) {
+                    return;
+                }
+                for (var shadowVariableDescriptor : shadowVariableDescriptors) {
+                    var value = shadowVariableDescriptor.getValue(entity);
+                    if (value == null) {
+                        continue;
+                    }
+                    if (shadowVariableDescriptor instanceof InverseRelationShadowVariableDescriptor
+                            && value instanceof Collection<?> collection) {
+                        // Inverse collection must never be null, so just empty it.
+                        collection.clear();
+                        continue;
+                    }
+                    var propertyType = shadowVariableDescriptor.getVariablePropertyType();
+                    if (propertyType.isPrimitive()) {
+                        if (propertyType == boolean.class) {
+                            shadowVariableDescriptor.setValue(entity, false);
+                        } else if (propertyType == byte.class) {
+                            shadowVariableDescriptor.setValue(entity, (byte) 0);
+                        } else if (propertyType == char.class) {
+                            shadowVariableDescriptor.setValue(entity, '\u0000');
+                        } else if (propertyType == short.class) {
+                            shadowVariableDescriptor.setValue(entity, (short) 0);
+                        } else if (propertyType == int.class) {
+                            shadowVariableDescriptor.setValue(entity, 0);
+                        } else if (propertyType == long.class) {
+                            shadowVariableDescriptor.setValue(entity, 0L);
+                        } else {
+                            throw new IllegalStateException(
+                                    "Impossible state: unknown primitive type %s".formatted(propertyType));
+                        }
+                    } else {
+                        shadowVariableDescriptor.setValue(entity, null);
+                    }
+                }
+            });
+        }
+        variableListenerSupport.forceTriggerAllVariableListeners(solution);
     }
 
     protected void setCalculatedScore(Score_ score) {

core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScoreDirector.java

@@ -349,8 +349,11 @@ void assertExpectedUndoMoveScore(Move<Solution_> move, InnerScore<Score_> before
      * Unlike {@link #triggerVariableListeners()} which only triggers notifications already in the queue,
      * this triggers every variable listener on every genuine variable.
      * This is useful in {@link SolutionManager#update(Object)} to fill in shadow variable values.
+     * 
+     * @param resetShadowVariablesFirst true if the shadow variables should be reset first,
+     *        slowing down the operation but preventing stale dependencies.
      */
-    void forceTriggerVariableListeners();
+    void forceTriggerVariableListeners(boolean resetShadowVariablesFirst);
 
     /**
      * A derived score director is created from a root score director.

core/src/main/java/ai/timefold/solver/core/impl/solver/DefaultSolutionManager.java

@@ -78,7 +78,7 @@ private <Result_> Result_ callScoreDirector(Solution_ solution,
                         Maybe use Constraint Streams instead of Easy or Incremental score calculator?""");
             }
             if (isShadowVariableUpdateEnabled) {
-                scoreDirector.forceTriggerVariableListeners();
+                scoreDirector.forceTriggerVariableListeners(solutionUpdatePolicy.isShadowVariableUpdateForced());
             }
             if (solutionUpdatePolicy.isScoreUpdateEnabled()) {
                 scoreDirector.calculateScore();

core/src/test/java/ai/timefold/solver/core/impl/domain/variable/ListVariableListenerTest.java

@@ -122,7 +122,7 @@ void addAndRemoveEntity() {
         var ann = new TestdataListEntityWithShadowHistory("Ann", a, b, c);
 
         scoreDirector.setWorkingSolution(buildSolution(ann));
-        scoreDirector.forceTriggerVariableListeners();
+        scoreDirector.forceTriggerVariableListeners(false);
 
         // Assert inverse entity.
         assertEntityHistory(a, ann);

core/src/test/java/ai/timefold/solver/core/impl/domain/variable/custom/CustomVariableListenerTest.java

@@ -185,7 +185,7 @@ void manyToManyRequiresUniqueEntityEvents() {
         solution.setEntityList(new ArrayList<>());
         solution.setValueList(List.of(val1));
         scoreDirector.setWorkingSolution(solution);
-        scoreDirector.forceTriggerVariableListeners();
+        scoreDirector.forceTriggerVariableListeners(true);
 
         scoreDirector.beforeEntityAdded(b);
         scoreDirector.getWorkingSolution().getEntityList().add(b);

test/src/main/java/ai/timefold/solver/test/impl/score/stream/DefaultShadowVariableAwareMultiConstraintAssertion.java

@@ -6,6 +6,7 @@
 
 import ai.timefold.solver.core.api.score.Score;
 import ai.timefold.solver.core.api.score.stream.ConstraintProvider;
+import ai.timefold.solver.core.api.solver.SolutionManager;
 import ai.timefold.solver.core.impl.score.constraint.ConstraintMatchPolicy;
 import ai.timefold.solver.core.impl.score.stream.common.AbstractConstraintStreamScoreDirectorFactory;
 import ai.timefold.solver.test.api.score.stream.MultiConstraintAssertion;
@@ -28,11 +29,11 @@ public final class DefaultShadowVariableAwareMultiConstraintAssertion<Solution_,
     @Override
     public MultiConstraintAssertion settingAllShadowVariables() {
         // Most score directors don't need derived status; CS will override this.
+        SolutionManager.updateShadowVariables(solution);
         try (var scoreDirector = scoreDirectorFactory.createScoreDirectorBuilder()
                 .withConstraintMatchPolicy(ConstraintMatchPolicy.ENABLED)
                 .buildDerived()) {
             scoreDirector.setWorkingSolution(solution);
-            scoreDirector.forceTriggerVariableListeners();
             update(scoreDirector.calculateScore(), scoreDirector.getConstraintMatchTotalMap(),
                     scoreDirector.getIndictmentMap());
             toggleInitialized();