ci/eval: Catch non-top-level internally disallowed problems

Before, something like

  nativeBuildInputs = [
    (runCommand "dep" { meta.description = "Dep"; } "")
  ];

would cause an eval warning without failing CI

After this change, CI ensures that when evaluating the set of packages
that don't have internally disallowed problem kinds, no such problem
kinds are present.

Author: infinisil

ci/eval/chunk.nix

@@ -8,6 +8,7 @@
   myChunk,
   includeBroken,
   systems,
+  defaultProblemHandler,
   extraNixpkgsConfigJson,
 }:
 
@@ -17,7 +18,7 @@ let
 
   unfiltered = import ./outpaths.nix {
     inherit path;
-    inherit includeBroken systems;
+    inherit includeBroken systems defaultProblemHandler;
     inherit (preEvalResult) attrPathsDisallowedForInternalUse;
     extraNixpkgsConfig = builtins.fromJSON extraNixpkgsConfigJson;
   };

ci/eval/default.nix

@@ -101,6 +101,7 @@ let
         system=$3
         outputDir=$4
         preEvalFile=$5
+        defaultProblemHandler=$6
 
         # Default is 5, higher values effectively disable the warning.
         # This randomly breaks Eval.
@@ -124,6 +125,7 @@ let
           --arg myChunk "$myChunk" \
           --arg preEvalFile "$preEvalFile" \
           --arg systems "[ \"$system\" ]" \
+          --argstr defaultProblemHandler "$defaultProblemHandler" \
           --arg includeBroken ${lib.boolToString includeBroken} \
           --argstr extraNixpkgsConfigJson ${lib.escapeShellArg (builtins.toJSON extraNixpkgsConfig)} \
           -I ${nixpkgs} \
@@ -188,6 +190,7 @@ let
         chunkedEval() {
           local chunkOutputDir=$1
           local preEvalFile=$2
+          local defaultProblemHandler=$3
 
           local attrCount=$(jq '.paths | length' "$preEvalFile")
           echo "Attribute count: $attrCount"
@@ -205,7 +208,7 @@ let
 
           seq -w 0 "$seq_end" |
             xargs -I{} -P"$cores" \
-            ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir" "$preEvalFile"
+            ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir" "$preEvalFile" "$defaultProblemHandler"
 
           if (( chunkSize * chunkCount != attrCount )); then
             # A final incomplete chunk would mess up the stats, don't include it
@@ -225,13 +228,15 @@ let
         startEpoch=$(date +%s)
 
         # The first eval evaluates only attributes that are not disallowed for internal Nixpkgs use, ensuring that they don't depend on disallowed attributes
-        # Because the first eval doesn't evaluate the disallowed attributes themselves, but we still want to check that they don't fail evaluation, we evaluate them separately in a second eval
+        # It also uses "warn" as the default problem handler for internally disallowed problem kinds, which ensures no such warnings are thrown at all, because any stderr output fails CI.
+        # Because the first eval doesn't evaluate the disallowed attributes themselves, but we still want to check that they don't fail evaluation, we evaluate them separately in a second eval.
+        # The second eval uses "ignore" as the default problem handler for internally disallowed problem kinds, because we don't care about transitive warnings then.
         # The reason we need two evals is because we want disallowed attributes to be able to depend on other disallowed attributes, which inherently needs a separate Nixpkgs instantiation
         # And while we could interleave that instantiation into a single eval, that would ~double memory usage for all chunks, while doing it separately doesn't
         echo "Evaluating the internally allowed attributes"
-        chunkedEval "$chunkOutputDirs"/allowed ${preEvalFile}
+        chunkedEval "$chunkOutputDirs"/allowed ${preEvalFile} warn
         echo "Evaluating the internally disallowed attributes"
-        chunkedEval "$chunkOutputDirs"/disallowed "$disallowedAttributesPreEvalFile"
+        chunkedEval "$chunkOutputDirs"/disallowed "$disallowedAttributesPreEvalFile" ignore
 
         echo $(( $(date +%s) - startEpoch )) > "$out/${evalSystem}/total-time"
 

ci/eval/outpaths.nix

@@ -16,6 +16,8 @@
 
   attrPathsDisallowedForInternalUse ? [ ],
 
+  defaultProblemHandler ? "ignore",
+
   # Customize the config used to evaluate nixpkgs
   extraNixpkgsConfig ? { },
 }:
@@ -37,19 +39,27 @@ let
             allowVariants = !attrNamesOnly;
             checkMeta = true;
 
-            # We don't need to care about problems being caught using the
-            # standard mechanism, because any problems whose kind is not
-            # nixpkgsInternalUseAllowed cause the corresponding attributes to
-            # be disallowed entirely for internal use with
-            # attrPathsDisallowedForInternalUse, see also ./pre-eval.nix
+            # TODO: This should be synchronised with the problem kinds nixpkgsInternalUseAllowed:
+            # If allowed -> "error" (traditional default for broken) or "ignore"
+            # If not allowed -> defaultProblemHandler
             problems.matchers = lib.mkForce [
-              # We only need to set the broken handler to error, so that CI
-              # doesn't evaluate those. No reason it couldn't evaluate them
-              # afaik, but this is how it's been before.
               {
                 kind = "broken";
                 handler = "error";
               }
+              {
+                kind = "maintainerless";
+                handler = defaultProblemHandler;
+              }
+              {
+                kind = "removal";
+                handler = defaultProblemHandler;
+              }
+              # TODO: What to do about this? Doesn't currently give even a warning
+              #{
+              #  kind = "deprecated";
+              #  handler = defaultProblemHandler;
+              #}
             ];
             inherit attrPathsDisallowedForInternalUse;