Unify baseline checking behind a single checkBaseline helper (#19888)

Introduce checkBaseline/checkBaselineWith in the Compiler test module and
route the syntax-tree tests, surface-area verification, and the .err.bsl /
.il.bsl / .vsbsl component-test baselines through it, replacing each site's
own shouldUpdateBaselines check and ad-hoc file writes.

The helper reads the baseline file, compares via a pluggable comparer
(default exact equality; surface area and error baselines pass a set-based
comparer, IL passes an ILChecker-based one), and on mismatch writes the
produced content to an output file or updates the baseline when
TEST_UPDATE_BSL is set. The output file extension is replaced for .bsl
(foo.bsl -> foo.out) and appended otherwise (neg78.vsbsl -> neg78.vsbsl.out)
so the .bsl/.vsbsl baselines a single test can have no longer collide.

Removes the now-redundant updateBaseLineIfEnvironmentSaysSo,
createBaselineErrors, convenienceBaselineInstructions, and assertBaseline,
and drops the unused BaselineFile.FilePath field. This also fixes the IL
baseline update path, which previously copied the previous run's IL to the
baseline because the copy ran before the fresh actual was written.

The SyntaxTree-local .gitignore (*.actual) is removed since those tests now
write .out, covered by the global *.out rule.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: auduchinok

tests/FSharp.Compiler.Service.Tests/SurfaceArea.fs

@@ -28,5 +28,4 @@ type SurfaceAreaTest() =
             Assembly.LoadFrom path
 
         let baseline = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Compiler.Service.SurfaceArea.{platform}.bsl")
-        let outFileName = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Compiler.Service.SurfaceArea.{platform}.out")
-        FSharp.Test.SurfaceArea.verify assembly baseline outFileName
+        FSharp.Test.SurfaceArea.verify assembly baseline

tests/FSharp.Compiler.Service.Tests/SyntaxTreeTests.fs

@@ -178,25 +178,7 @@ let ParseFile fileName =
         |> normalize
         |> sprintf "%s\n"
 
-    let bslPath = $"{fullPath}.bsl"
-    let actualPath = $"{fullPath}.actual"
-
-    let expected =
-        if File.Exists bslPath then
-            File.ReadAllText bslPath |> normalize
-        else
-            "No baseline was found"
-
-    if expected = actual then
-        File.Delete(actualPath)
-    else
-        if shouldUpdateBaselines then
-            File.Delete(actualPath)
-            File.WriteAllText(bslPath, actual)
-        else
-            File.WriteAllText(actualPath, actual)
-
-    Assert.Equal(expected, actual)
+    checkBaseline actual $"{fullPath}.bsl"
 
     // Run type checker to assert that it doesn't fail with the tree produced by the parser
     CompilerAssert.ParseAndTypeCheck([|"--langversion:preview"|], fileName, contents) |> ignore

tests/FSharp.Core.UnitTests/SurfaceArea.fs

@@ -38,5 +38,4 @@ type SurfaceAreaTest() =
 #endif
         let assembly = typeof<int list>.Assembly
         let baseline = Path.Combine(__SOURCE_DIRECTORY__, $"FSharp.Core.SurfaceArea.{platform}.{flavor}.bsl")
-        let outFileName = Path.Combine(Path.GetDirectoryName(assembly.Location), $"FSharp.Core.SurfaceArea.{platform}.{flavor}.out")
-        SurfaceArea.verify assembly baseline outFileName
+        SurfaceArea.verify assembly baseline

tests/FSharp.Test.Utilities/Compiler.fs

@@ -30,13 +30,54 @@ module rec Compiler =
     let shouldUpdateBaselines =
         Environment.GetEnvironmentVariable("TEST_UPDATE_BSL") <> null
 
+    let private baselineFailureMessage (expectedFile: string) (outFile: string) (diff: string) =
+        $"""Baseline mismatch for {expectedFile}
+to update the baseline:
+$ cp {outFile} {expectedFile}
+to compare:
+$ code --diff {outFile} {expectedFile}
+(or set TEST_UPDATE_BSL=1 and re-run to update the baseline automatically)
+{diff}"""
+
+    let private baselineOutputFile (expectedFile: string) =
+        if Path.GetExtension(expectedFile) = ".bsl" then
+            Path.ChangeExtension(expectedFile, ".out")
+        else
+            expectedFile + ".out"
+
+    let checkBaselineWith (compare: string -> string -> string option) (expected: string) (expectedFile: string) =
+        let outFile = baselineOutputFile expectedFile
+        let baselineContent =
+            if FileSystem.FileExistsShim expectedFile then File.ReadAllText expectedFile else ""
+        let diff = compare baselineContent expected
+
+        match diff with
+        | None ->
+            if FileSystem.FileExistsShim outFile then
+                FileSystem.FileDeleteShim outFile
+        | Some diff ->
+            if shouldUpdateBaselines then
+                if FileSystem.FileExistsShim outFile then
+                    FileSystem.FileDeleteShim outFile
+                File.WriteAllText(expectedFile, expected)
+            else
+                File.WriteAllText(outFile, expected)
+
+            Assert.True(false, baselineFailureMessage expectedFile outFile diff)
+
+    let checkBaseline (expected: string) (expectedFile: string) =
+        let compare fileContent produced =
+            let e = normalizeNewlines fileContent
+            let a = normalizeNewlines produced
+            if e = a then None else Some $"Expected:\n{e}\nActual:\n{a}"
+        checkBaselineWith compare expected expectedFile
+
     [<AutoOpen>]
     type SourceUtilities () =
         static member getCurrentMethodName([<CallerMemberName; Optional; DefaultParameterValue("")>] memberName: string) = memberName
 
     type BaselineFile =
         {
-            FilePath: string
             BslSource: string
             Content: string option
         }
@@ -251,8 +292,6 @@ module rec Compiler =
             | Some s -> s
             | None -> sourceFilePath + sourceBaselineSuffix + ".il.bsl"
 
-        let fsOutFilePath = normalizePathSeparator (Path.ChangeExtension(outputDirectoryPath ++ filename, ".err"))
-        let ilOutFilePath = normalizePathSeparator (Path.ChangeExtension(outputDirectoryPath ++ filename, ".il"))
         let fsBslSource = readFileOrDefault fsBslFilePath
         let ilBslSource = readFileOrDefault ilBslFilePath
 
@@ -262,8 +301,8 @@ module rec Compiler =
                 Some
                     {
                         SourceFilename = Some sourceFilePath
-                        FSBaseline = { FilePath = fsOutFilePath; BslSource = fsBslFilePath; Content = fsBslSource }
-                        ILBaseline = { FilePath = ilOutFilePath; BslSource = ilBslFilePath; Content = ilBslSource }
+                        FSBaseline = { BslSource = fsBslFilePath; Content = fsBslSource }
+                        ILBaseline = { BslSource = ilBslFilePath; Content = ilBslSource }
                     }
             Options           = Compiler.defaultOptions
             OutputType        = Library
@@ -1220,36 +1259,6 @@ module rec Compiler =
         | _ -> failwith "FSI running only supports F#."
 
 
-    let convenienceBaselineInstructions baseline expected actual =
-        $"""to update baseline:
-$ cp {baseline.FilePath} {baseline.BslSource}
-to compare baseline:
-$ code --diff {baseline.FilePath} {baseline.BslSource}
-Expected:
-{expected}
-Actual:
-{actual}"""
-    let updateBaseLineIfEnvironmentSaysSo baseline =
-        if shouldUpdateBaselines then
-            if FileSystem.FileExistsShim baseline.FilePath then
-                FileSystem.CopyShim(baseline.FilePath, baseline.BslSource, true)
-
-    let assertBaseline expected actual baseline fOnFail =
-        if expected <> actual then
-            fOnFail()
-            updateBaseLineIfEnvironmentSaysSo baseline
-            createBaselineErrors baseline actual
-            Assert.True((expected = actual), convenienceBaselineInstructions baseline expected actual)
-        elif FileSystem.FileExistsShim baseline.FilePath then
-            FileSystem.FileDeleteShim baseline.FilePath
-
-
-    let private createBaselineErrors (baselineFile: BaselineFile) (actualErrors: string) : unit =
-        printfn $"creating baseline error file for convenience: {baselineFile.FilePath}, expected: {baselineFile.BslSource}"
-        let file = FileSystem.OpenFileForWriteShim(baselineFile.FilePath)
-        file.SetLength(0)
-        file.WriteAllText(actualErrors)
-
     /// Turn our ErrorInfo back into a genuine FSharpDiagnostic
     let private toFSharpDiagnostic (ei: ErrorInfo) : FSharpDiagnostic =
 
@@ -1309,19 +1318,8 @@ Actual:
                 match o.Compilation with
                 | FS fs -> fs
                 | _     -> failwith "verifyBaseline only supports F#"
-        let expected =
-            fsSource.Baseline.Value.FSBaseline.Content
-            |> Option.defaultValue ""
-            |> normalizeNewlines
-
         // 4) Compare or update
-        if expected <> formattedActual then
-            // same update mechanism you already have:
-            fsSource.CreateOutputDirectory()
-            createBaselineErrors fsSource.Baseline.Value.FSBaseline formattedActual
-            updateBaseLineIfEnvironmentSaysSo fsSource.Baseline.Value.FSBaseline
-            let msg = convenienceBaselineInstructions fsSource.Baseline.Value.FSBaseline expected formattedActual
-            Assert.True(false, msg)
+        checkBaseline formattedActual fsSource.Baseline.Value.FSBaseline.BslSource
 
         // 5) Return the original result for fluent chaining
         cResult
@@ -1387,14 +1385,8 @@ Actual:
                 | None ->  String.Empty
             let success, errorMsg, actualIL = ILChecker.verifyILAndReturnActual [] p [expectedIL]
 
-            if not success then
-                // Failed try update baselines if required
-                // If we are here then the il file has been produced we can write it back to the baseline location
-                // if the environment variable TEST_UPDATE_BSL has been set
-                updateBaseLineIfEnvironmentSaysSo baseline.ILBaseline
-                createBaselineErrors baseline.ILBaseline actualIL
-                let errorMsg = (convenienceBaselineInstructions baseline.ILBaseline expectedIL actualIL) + errorMsg
-                Assert.Fail(errorMsg)
+            let compare _ _ = if success then None else Some errorMsg
+            checkBaselineWith compare actualIL baseline.ILBaseline.BslSource
 
     let verifyILBaseline (compilationResult: CompilationResult) : CompilationResult =
         match compilationResult with
@@ -2011,22 +2003,9 @@ Actual:
                 |> String.Concat
 
             let withResultsMatchingFile (path:string) (result:CompilationResult) =
-                let expectedContent = File.ReadAllText(path) |> normalizeNewLines
-                let actualErrors = renderToString result
-
-                match Assert.shouldBeSameMultilineStringSets expectedContent actualErrors with
-                | None -> ()
-                | Some diff ->
-                    if shouldUpdateBaselines then
-                        File.WriteAllText(path, actualErrors)
-
-                    printfn $"{Path.GetFullPath path} \n {diff}"
-                    printfn "==========================EXPECTED==========================="
-                    printfn "%s" expectedContent
-                    printfn "===========================ACTUAL============================"
-                    printfn "%s" actualErrors
-                    Assert.True(String.IsNullOrEmpty(diff), path)
-
+                let compare fileContent produced =
+                    Assert.shouldBeSameMultilineStringSets (normalizeNewLines fileContent) produced
+                checkBaselineWith compare (renderToString result) path
                 result
 
         let checkCodes (expected: int list) (selector: CompilationOutput -> ErrorInfo list) (result: CompilationResult) : CompilationResult =

tests/FSharp.Test.Utilities/SurfaceArea.fs

@@ -42,42 +42,26 @@ module FSharp.Test.SurfaceArea
             |]
         assembly, actual
 
-    let private appendNewLine str = str + System.Environment.NewLine
-
     // verify public surface area matches expected, handles baseline update when TEST_UPDATE_BSL is set
-    let verify assembly baselinePath outFilePath : unit =
-        let expected =
-            File.ReadAllLines(baselinePath)
-            |> String.concat System.Environment.NewLine
-            |> appendNewLine
-
+    let verify assembly baselinePath : unit =
         let normalize (s:string) = Regex.Replace(s, "(\\r\\n|\\n|\\r)+", Environment.NewLine).Trim()
         let asm, actualNotNormalized = getSurfaceAreaForAssembly (assembly)
-        let actual = 
-            actualNotNormalized 
-            |> Seq.map normalize 
+        let actual =
+            actualNotNormalized
+            |> Seq.map normalize
             |> Seq.filter (String.IsNullOrWhiteSpace >> not)
             |> Seq.sort
             |> String.concat Environment.NewLine
 
-        let expected = normalize expected
-
-        match Assert.shouldBeSameMultilineStringSets expected actual with
-        | None ->
-            File.Delete(outFilePath)
-
-        | Some diff ->
-            if shouldUpdateBaselines then
-                File.Delete(outFilePath)
-                File.WriteAllText(baselinePath, actual)
-            else
-                File.WriteAllText(outFilePath, actual)
-
-                let msg = $"""Assembly: %A{asm}
+        let compare fileContent produced =
+            match Assert.shouldBeSameMultilineStringSets (normalize fileContent) produced with
+            | None -> None
+            | Some diff ->
+                Some $"""Assembly: %A{asm}
 
                   Expected and actual surface area don't match. To see the delta, run:
-                      windiff {baselinePath} {outFilePath}
+                      windiff {baselinePath} {Path.ChangeExtension(baselinePath, ".out")}
 
                   {diff}"""
 
-                failwith msg
+        checkBaselineWith compare actual baselinePath