Pre-validate measure libraries before per-subject CQL evaluation (#1024)

Measures whose CQL referenced a missing ValueSet (or had other fatal
compile errors) failed in two awkward ways. With a function-based
component stratifier, FunctionEvaluationHandler.processNonSubValueStratifier
observed missing expression results when the IP failed and threw
"Expression result: Initial Population is missing" — masking the
underlying CqlException. In multi-measure evaluations, that masking
exception then escaped the per-library loop in MeasureEvaluationResultHandler
and was caught by the outer subject-scoped try/catch, which wrote the
error to every measure def — poisoning unrelated libraries that would
otherwise have evaluated cleanly.

Two complementary layers:

- MeasureLibraryPreValidator resolves each library before the per-subject
  loop, collects fatal compile errors via LibraryManager.resolveLibrary's
  error-collecting overload, and walks the compiled ELM ValueSet refs
  against the terminology provider. Failed libraries drop from the
  evaluation set with the engine-level diagnostic ("Unable to locate
  ValueSet …", or the original compile message) recorded against their
  measure defs. Hard library-resolution failures still propagate so
  existing throw-based contracts in InvalidMeasureTest are preserved.

- MeasureEvaluationResultHandler.getEvaluationResults now checks
  getExceptionFor(libraryId) before invoking cqlFunctionEvaluation, so
  any per-subject CqlException that slips past pre-validation is
  surfaced through the existing per-subject error path instead of being
  masked by the function-evaluation handler.

Tests:
- MeasureLibraryPreValidatorTest exercises compile-error, missing-
  ValueSet, fall-through, short-circuit, and multi-library isolation
  cases against real MultiLibraryIdMeasureEngineDetails / MeasureDef /
  CompositeEvaluationResultsPerMeasure instances.
- MultiMeasureIsolationTest evaluates three measures (clean / broken /
  clean) and asserts the broken one's failure does not poison the
  siblings' MeasureReports.
- Four cohortEncounterMissingValueSet* integration tests live in
  InvalidMeasureTest covering no-stratifier, scalar, value-component,
  and function-component stratifier shapes — all surfacing the same
  "Unable to locate ValueSet" diagnostic.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: lukedegruchy

cqf-fhir-cr/src/main/java/org/opencds/cqf/fhir/cr/measure/common/MeasureEvaluationResultHandler.java

@@ -7,7 +7,6 @@
 import java.time.ZonedDateTime;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import org.apache.commons.lang3.tuple.Pair;
 import org.hl7.elm.r1.VersionedIdentifier;
 import org.opencds.cqf.cql.engine.execution.CqlEngine;
@@ -111,15 +110,31 @@ public static CompositeEvaluationResultsPerMeasure getEvaluationResults(
         // measure -> subject -> results
         var resultsBuilder = CompositeEvaluationResultsPerMeasure.builder();
 
+        // Pre-validate ValueSet references against the terminology provider before any
+        // per-subject evaluation runs. Libraries with unresolvable ValueSets are recorded with
+        // an error and excluded from the per-subject loop entirely — keeping the original
+        // diagnostic ("Unable to locate ValueSet …") and avoiding wasted per-subject work.
+        var preValidationFailedLibraries =
+                new MeasureLibraryPreValidator(context).validate(multiLibraryIdMeasureEngineDetails, resultsBuilder);
+
+        final List<VersionedIdentifier> activeLibraryIdentifiers =
+                multiLibraryIdMeasureEngineDetails.getLibraryIdentifiers().stream()
+                        .filter(id -> !preValidationFailedLibraries.contains(id))
+                        .toList();
+
         // Library $evaluate each subject
         // The goal here is to do each measure/library evaluation within the context of a single subject.
         // This means that we will not switch between subject contexts while evaluating measures.
         // Once we've switched to a different subject context, the previous expression cache is dropped.
 
-        final List<String> libraryIdentIds = multiLibraryIdMeasureEngineDetails.getLibraryIdentifiers().stream()
+        final List<String> libraryIdentIds = activeLibraryIdentifiers.stream()
                 .map(VersionedIdentifier::getId)
                 .toList();
 
+        if (activeLibraryIdentifiers.isEmpty()) {
+            return resultsBuilder.build();
+        }
+
         logger.atDebug()
                 .setMessage(
                         "START: Evaluate measure for library idents: (count:{}): [{}], and subjects (count={}): [{}]")
@@ -150,7 +165,6 @@ public static CompositeEvaluationResultsPerMeasure getEvaluationResults(
             String subjectIdPart = subjectInfo.getRight();
             context.getState().setContextValue(subjectTypePart, subjectIdPart);
             try {
-                var libraryIdentifiers = multiLibraryIdMeasureEngineDetails.getLibraryIdentifiers();
 
                 final long startPerLibraryPerSubject = System.currentTimeMillis();
                 throttledDebug(shouldLog)
@@ -161,7 +175,7 @@ public static CompositeEvaluationResultsPerMeasure getEvaluationResults(
                 var evaluationResultsForMultiLib = multiLibraryIdMeasureEngineDetails
                         .getLibraryEngine()
                         .getEvaluationResult(
-                                libraryIdentifiers,
+                                activeLibraryIdentifiers,
                                 subjectId,
                                 null,
                                 parametersMap,
@@ -177,7 +191,7 @@ public static CompositeEvaluationResultsPerMeasure getEvaluationResults(
                         .addArgument(() -> showSubsetOfTotal(libraryIdentIds))
                         .log();
 
-                for (var libraryVersionedIdentifier : libraryIdentifiers) {
+                for (var libraryVersionedIdentifier : activeLibraryIdentifiers) {
                     validateEvaluationResultExistsForIdentifier(
                             libraryVersionedIdentifier, evaluationResultsForMultiLib);
 
@@ -186,24 +200,28 @@ public static CompositeEvaluationResultsPerMeasure getEvaluationResults(
                     var measureDefs =
                             multiLibraryIdMeasureEngineDetails.getMeasureDefsForLibrary(libraryVersionedIdentifier);
 
-                    // function evaluation
-                    final List<EvaluationResult> functionEvaluationResults =
-                            FunctionEvaluationHandler.cqlFunctionEvaluation(
-                                    context,
-                                    measureDefs,
-                                    libraryVersionedIdentifier,
-                                    evaluationResult,
-                                    subjectTypePart);
+                    final var libraryException =
+                            evaluationResultsForMultiLib.getExceptionFor(libraryVersionedIdentifier);
+
+                    // Skip function evaluation when the base CQL eval already failed for this
+                    // library/subject. Otherwise the function path (e.g. processNonSubValueStratifier)
+                    // observes missing expression results and throws a higher-level
+                    // "Expression result: <pop> is missing" error that masks the underlying
+                    // CqlException ("Unable to locate ValueSet", etc.) and — by escaping the inner
+                    // loop into the outer catch — pollutes sibling libraries' measure defs.
+                    final List<EvaluationResult> functionEvaluationResults = (libraryException == null)
+                            ? FunctionEvaluationHandler.cqlFunctionEvaluation(
+                                    context, measureDefs, libraryVersionedIdentifier, evaluationResult, subjectTypePart)
+                            : List.of();
 
                     resultsBuilder.addResults(measureDefs, subjectId, evaluationResult, functionEvaluationResults);
 
-                    Optional.ofNullable(evaluationResultsForMultiLib.getExceptionFor(libraryVersionedIdentifier))
-                            .ifPresent(exception -> {
-                                var error = EXCEPTION_FOR_SUBJECT_ID_MESSAGE_TEMPLATE.formatted(
-                                        subjectId, exception.getMessage());
-                                resultsBuilder.addErrors(measureDefs, error);
-                                logger.error(error, exception);
-                            });
+                    if (libraryException != null) {
+                        var error = EXCEPTION_FOR_SUBJECT_ID_MESSAGE_TEMPLATE.formatted(
+                                subjectId, libraryException.getMessage());
+                        resultsBuilder.addErrors(measureDefs, error);
+                        logger.error(error, libraryException);
+                    }
                 }
 
             } catch (Exception e) {

cqf-fhir-cr/src/main/java/org/opencds/cqf/fhir/cr/measure/common/MeasureLibraryPreValidator.java

@@ -0,0 +1,150 @@
+package org.opencds.cqf.fhir.cr.measure.common;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import org.cqframework.cql.cql2elm.CqlCompilerException;
+import org.cqframework.cql.cql2elm.LibraryManager;
+import org.cqframework.cql.cql2elm.model.CompiledLibrary;
+import org.hl7.elm.r1.ValueSetDef;
+import org.hl7.elm.r1.VersionedIdentifier;
+import org.opencds.cqf.cql.engine.execution.CqlEngine;
+import org.opencds.cqf.cql.engine.terminology.TerminologyProvider;
+import org.opencds.cqf.cql.engine.terminology.ValueSetInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Static pre-validation pass for measure libraries. Resolves each library, captures fatal compile
+ * errors, and walks the compiled ELM ValueSet references to confirm they exist in the terminology
+ * provider — all before any per-subject CQL evaluation runs.
+ *
+ * <p>Libraries with fatal compile errors or unresolvable ValueSet references are recorded as
+ * errors against their bound measure defs and returned in the failed-set so the caller can
+ * exclude them from per-subject CQL evaluation. This avoids two failure modes that would
+ * otherwise occur at runtime:
+ * <ul>
+ *   <li><b>Diagnostic masking</b>: function-based stratifier evaluation observes missing
+ *       expression results when the IP fails and throws a higher-level
+ *       "Expression result: &lt;pop&gt; is missing" error that hides the underlying cause.</li>
+ *   <li><b>Multi-measure pollution</b>: the masked exception escapes the inner library loop in
+ *       {@link MeasureEvaluationResultHandler} and is caught by the outer subject-scoped
+ *       try/catch, which writes the error to <em>every</em> measure def — including unrelated
+ *       libraries that compiled and would have evaluated cleanly.</li>
+ * </ul>
+ *
+ * <p>Hard library-resolution failures (e.g. {@code CqlIncludeException} for a missing CQL source)
+ * still propagate to the runtime CQL path so existing throw-based contracts in
+ * {@code InvalidMeasureTest} are preserved.
+ */
+public class MeasureLibraryPreValidator {
+
+    private static final Logger logger = LoggerFactory.getLogger(MeasureLibraryPreValidator.class);
+    private static final String EXCEPTION_FOR_LIBRARY_TEMPLATE = "Exception for library: %s, Message: %s";
+
+    private final CqlEngine engine;
+
+    public MeasureLibraryPreValidator(CqlEngine engine) {
+        this.engine = engine;
+    }
+
+    /**
+     * Validate compile-time integrity for every library bound to the given measure defs.
+     *
+     * @param details measure-engine details holding the library-to-measure binding
+     * @param resultsBuilder builder to record per-library validation errors against measure defs
+     * @return library identifiers that failed validation (caller should exclude them from
+     *         per-subject CQL evaluation)
+     */
+    public Set<VersionedIdentifier> validate(
+            MultiLibraryIdMeasureEngineDetails details, CompositeEvaluationResultsPerMeasure.Builder resultsBuilder) {
+
+        var libraryManager = engine.getEnvironment().getLibraryManager();
+        var terminologyProvider = engine.getEnvironment().getTerminologyProvider();
+        var failed = new HashSet<VersionedIdentifier>();
+
+        for (var libraryId : details.getLibraryIdentifiers()) {
+            findFirstFailure(libraryManager, terminologyProvider, libraryId)
+                    .ifPresent(error -> recordLibraryFailure(details, resultsBuilder, libraryId, error, failed));
+        }
+        return failed;
+    }
+
+    private static Optional<RuntimeException> findFirstFailure(
+            LibraryManager libraryManager, TerminologyProvider terminologyProvider, VersionedIdentifier libraryId) {
+
+        var compileErrors = new ArrayList<CqlCompilerException>();
+        CompiledLibrary compiled;
+        try {
+            compiled = libraryManager.resolveLibrary(libraryId, compileErrors);
+        } catch (RuntimeException exception) {
+            // Hard failures (missing library source, etc.) preserve the existing throw-at-runtime
+            // contract for callers that rely on the exception type.
+            return Optional.empty();
+        }
+
+        var firstCompileError = firstFatalCompileError(compileErrors);
+        if (firstCompileError.isPresent()) {
+            return firstCompileError;
+        }
+
+        return findFirstUnresolvableValueSet(terminologyProvider, compiled);
+    }
+
+    private static Optional<RuntimeException> firstFatalCompileError(List<CqlCompilerException> errors) {
+        return errors.stream()
+                .filter(exception -> exception.getSeverity() == CqlCompilerException.ErrorSeverity.Error)
+                .map(RuntimeException.class::cast)
+                .findFirst();
+    }
+
+    private static Optional<RuntimeException> findFirstUnresolvableValueSet(
+            TerminologyProvider terminologyProvider, CompiledLibrary compiled) {
+
+        for (var vsDef : valueSetDefs(compiled)) {
+            var info = toValueSetInfo(vsDef);
+            if (info != null) {
+                try {
+                    terminologyProvider.expand(info);
+                } catch (RuntimeException e) {
+                    return Optional.of(e);
+                }
+            }
+        }
+        return Optional.empty();
+    }
+
+    private static List<ValueSetDef> valueSetDefs(CompiledLibrary compiled) {
+        if (compiled == null || compiled.getLibrary() == null) {
+            return List.of();
+        }
+        var valueSets = compiled.getLibrary().getValueSets();
+        return valueSets == null ? List.of() : valueSets.getDef();
+    }
+
+    private static ValueSetInfo toValueSetInfo(ValueSetDef vsDef) {
+        if (vsDef.getId() == null) {
+            return null;
+        }
+        var info = new ValueSetInfo().withId(vsDef.getId());
+        if (vsDef.getVersion() != null) {
+            info.setVersion(vsDef.getVersion());
+        }
+        return info;
+    }
+
+    private static void recordLibraryFailure(
+            MultiLibraryIdMeasureEngineDetails details,
+            CompositeEvaluationResultsPerMeasure.Builder resultsBuilder,
+            VersionedIdentifier libraryId,
+            RuntimeException error,
+            Set<VersionedIdentifier> failed) {
+        var measureDefs = details.getMeasureDefsForLibrary(libraryId);
+        var formatted = EXCEPTION_FOR_LIBRARY_TEMPLATE.formatted(libraryId.getId(), error.getMessage());
+        resultsBuilder.addErrors(measureDefs, formatted);
+        logger.error(formatted, error);
+        failed.add(libraryId);
+    }
+}