TOOLS-4235 Move evergreen validate to precious config (#1019)

Author: autarch

build.go

@@ -51,9 +51,6 @@ func init() {
 	taskRegistry.Declare("sa:modtidy").
 		Description("runs go mod tidy").
 		Do(buildscript.SAModTidy)
-	taskRegistry.Declare("sa:evgvalidate").
-		Description("runs evergreen validate").
-		Do(buildscript.SAEvergreenValidate)
 
 	// Testing
 	taskRegistry.Declare("test:unit").

buildscript/sa.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"strings"
 
 	"github.com/craiggwilson/goke/pkg/sh"
 	"github.com/craiggwilson/goke/task"
@@ -69,39 +68,3 @@ func SAModTidy(ctx *task.Context) error {
 
 	return nil
 }
-
-// SAEvergreenValidate runs `evergreen validate` on common.yml and ensures the file is valid.
-func SAEvergreenValidate(ctx *task.Context) error {
-	output, err := sh.RunOutput(
-		ctx,
-		"evergreen",
-		"validate",
-		"--file",
-		"common.yml",
-		"-p",
-		"mongo-tools",
-	)
-	if err != nil {
-		return fmt.Errorf("error from `evergreen validate`: %s: %w", output, err)
-	}
-
-	if strings.HasSuffix(output, "is valid with warnings") {
-		for _, line := range strings.Split(output, "\n") {
-			if strings.HasPrefix(line, "WARNING: ") &&
-				strings.HasSuffix(
-					line,
-					"defined but not used by any variants; consider using or disabling",
-				) {
-				continue
-			}
-
-			if strings.HasSuffix(line, "is valid with warnings") {
-				continue
-			}
-
-			return fmt.Errorf("error from `evergreen validate`: %s", output)
-		}
-	}
-
-	return nil
-}

common.yml

@@ -229,8 +229,6 @@ functions:
 
           ${_set_shell_env}
           ${_maybe_enable_devtoolset_7}
-          # The evergreen client binary lives in $HOME, and `sa:evgvalidate` shells out to it.
-          export PATH="$PATH:$HOME"
           $GO_EXEC_PREFIX go run build.go -v ${target}
 
   "download mongod and shell":
@@ -2375,20 +2373,12 @@ tasks:
         vars:
           target: sa:modtidy
 
-  - name: evergreen-validate
-    commands:
-      - func: "fetch source"
-      - func: "install mise and go"
-      - func: "create evergreen config file"
-      - func: "run make target"
-        vars:
-          target: sa:evgvalidate
-
   - name: lint
     commands:
       - func: "fetch source"
       - func: "install mise and go"
       - func: "install mise-managed tools"
+      - func: "create evergreen config file"
       - command: shell.exec
         type: test
         params:
@@ -2400,6 +2390,9 @@ tasks:
             set -o verbose
 
             ${_set_shell_env}
+            # The evergreen client binary lives in $HOME, and the evergreen-validate precious
+            # command shells out to it.
+            export PATH="$PATH:$HOME"
             mise exec 'github:houseabsolute/precious' -- precious lint --all
 
   - name: qa-tests-5.0
@@ -2691,7 +2684,6 @@ buildvariants:
     tasks:
       - name: lint
       - name: mod-tidy
-      - name: evergreen-validate
       - name: check-third-party-notices
       - name: check-sbom-lite
       - name: check-sarif-report

etc/evergreen-validate-wrapper.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# `evergreen validate` exits 0 even when it reports warnings, and it always writes to stdout, not
+# stderr. We want to fail on any warning except the specific pre-existing ones listed below, but
+# precious's ignore-stderr can't do that: it only looks at stderr, and it's all-or-nothing (matching
+# one pattern suppresses the whole output, not just that line). So we do the line-by-line filtering
+# here instead.
+#
+# Each accepted warning is matched exactly, by task name, rather than by a wildcard "defined but
+# not used" pattern. A new unused-task warning is a real problem to fix or explicitly allow here,
+# not something to pass silently just because it looks like the others.
+
+set -o pipefail
+
+TEMP_FILE=$(mktemp "${TMPDIR:-/tmp}/evergreen-validate.tmp.XXXXXXXXXX")
+trap 'rm -f "$TEMP_FILE"' EXIT
+
+# The tee lets us capture the output for filtering below while still printing it to the console, so
+# it shows up when precious is run in verbose mode.
+evergreen validate --file common.yml -p mongo-tools 2>&1 | tee "$TEMP_FILE"
+STATUS=${PIPESTATUS[0]}
+
+if [ "$STATUS" -ne 0 ]; then
+    exit "$STATUS"
+fi
+
+if grep --quiet "is valid with warnings" "$TEMP_FILE"; then
+    while IFS= read -r line; do
+        case "$line" in
+        "WARNING: task 'commit-queue-workaround' defined but not used by any variants; consider using or disabling")
+            continue
+            ;;
+        "WARNING: task 'deliberate_fail_to_test_build_baron_context_rules' defined but not used by any variants; consider using or disabling")
+            continue
+            ;;
+        "WARNING: task 't_resmoke_setup' defined but not used by any variants; consider using or disabling")
+            continue
+            ;;
+        "WARNING: task 'generate_mongodump_fuzz_tasks' defined but not used by any variants; consider using or disabling")
+            continue
+            ;;
+        *"is valid with warnings")
+            continue
+            ;;
+        *)
+            exit 1
+            ;;
+        esac
+    done <"$TEMP_FILE"
+fi

precious.toml

@@ -169,6 +169,15 @@ ok-exit-codes = 0
 lint-failure-exit-codes = 1
 labels = [ "default", "fast-tidy" ]
 
+[commands.evergreen-validate]
+type = "lint"
+invoke = "once"
+path-args = "none"
+include = [ "common.yml", "mongodump_passthrough/*.yml" ]
+cmd = [ "$PRECIOUS_ROOT/etc/evergreen-validate-wrapper.sh" ]
+ok-exit-codes = [0]
+lint-failure-exit-codes = [1]
+
 [commands.github-codeowners]
 type = "lint"
 invoke = "once"