Fix benchmark (#4928)

Three things have been done.

1. enhence the checking before benchmark, ensure the target repo actually buildable and the targets file actually exist.
2. we no longer generating the visualisations for the memory usage.
3. bump ghc and cabal version and update the target repos.

Instead of leaving the benchmark stale, we leave just enough functionalities for our CI.

---------

Co-authored-by: fendor <fendor@users.noreply.github.com>

Author: soulomoon

.github/workflows/bench.yml

@@ -19,7 +19,20 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       should_skip: ${{ steps.skip_check.outputs.should_skip }}
+      bench_ghc_versions: ${{ steps.set_ghc_versions.outputs.versions }}
     steps:
+      - id: set_ghc_versions
+        run: |
+          # benching the two latest GHCs we support now
+          # since benchmark are expansive.
+          # choosing the two latest are easier to maintain and more forward looking
+          # see discussion https://github.com/haskell/haskell-language-server/pull/4118
+          # Benchmarking only the two latest GHCs we support now
+          # since benchmark are expensive.
+          # Choosing the two latest is easier to maintain and more forward looking.
+          # See discussion https://github.com/haskell/haskell-language-server/pull/4118
+          # also possible to add more GHCs if we perform better in the future.
+          echo 'versions=["9.12", "9.14"]' >> $GITHUB_OUTPUT
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@v5.3.1
         with:
@@ -45,14 +58,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # benching the two latest GHCs we support now
-        # since benchmark are expansive.
-        # choosing the two latest are easier to maintain and more forward looking
-        # see discussion https://github.com/haskell/haskell-language-server/pull/4118
-        # also possible to add more GHCs if we performs better in the future.
-        ghc:
-          - '9.8'
-          - '9.10'
+        ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }}
         os:
           - ubuntu-latest
 
@@ -120,9 +126,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ghc: ['9.8', '9.10']
+        ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }}
         os: [ubuntu-latest]
-        cabal: ['3.14']
+        cabal: ['3.16']
         example: ['cabal', 'lsp-types']
 
     steps:
@@ -161,7 +167,7 @@ jobs:
         column -s, -t < bench-results/unprofiled/${{ matrix.example }}/resultDiff.csv | tee bench-results/unprofiled/${{ matrix.example }}/resultDiff.txt
 
     - name: tar benchmarking artifacts
-      run:  find bench-results -name "*.csv" -or -name "*.svg" -or -name "*.html" | xargs tar -czf benchmark-artifacts.tar.gz
+      run:  find bench-results -name "*.csv" | xargs tar -czf benchmark-artifacts.tar.gz
 
     - name: Archive benchmarking artifacts
       uses: actions/upload-artifact@v6

bench/Main.hs

@@ -20,14 +20,12 @@
     |       └── <HLS plugin>
     │           ├── <experiment>.gcStats.log          - RTS -s output
     │           ├── <experiment>.csv                  - stats for the experiment
-    │           ├── <experiment>.svg                  - Graph of bytes over elapsed time
-    │           ├── <experiment>.diff.svg             - idem, including the previous version
     │           ├── <experiment>.log                  - ghcide-bench output
     │           └── results.csv                       - results of all the experiments for the example
     ├── results.csv        - aggregated results of all the experiments and versions
-    └── <experiment>.svg   - graph of bytes over elapsed time, for all the included versions
+    └── resultDiff.csv     - diff of aggregated results compared with previous version
 
-   For diff graphs, the "previous version" is the preceding entry in the list of versions
+   For diff results, the "previous version" is the preceding entry in the list of versions
    in the config file. A possible improvement is to obtain this info via `git rev-list`.
 
    To execute the script:
@@ -36,8 +34,8 @@
 
    To build a specific analysis, enumerate the desired file artifacts
 
-   > stack bench --ba "bench-results/HEAD/results.csv bench-results/HEAD/edit.diff.svg"
-   > cabal bench --benchmark-options "bench-results/HEAD/results.csv bench-results/HEAD/edit.diff.svg"
+   > stack bench --ba "bench-results/HEAD/results.csv"
+   > cabal bench --benchmark-options "bench-results/HEAD/results.csv"
 
  -}
 {-# LANGUAGE DeriveAnyClass     #-}
@@ -165,8 +163,6 @@ createBuildSystem config = do
   benchRules build (MkBenchRules (askOracle $ GetSamples ()) benchHls warmupHls "haskell-language-server" (parallelism configStatic))
   addGetParentOracle
   csvRules build
-  svgRules build
-  heapProfileRules build
   phonyRules "" binaryName NoProfiling build (examples configStatic)
 
   whenJust (profileInterval configStatic) $ \i -> do

bench/README.md

@@ -41,30 +41,30 @@ The Shake script supports a number of phony targets that allow running a subset
 ```
 Targets:
   - bench-results/binaries/*/commitid
-  - bench-results/binaries/HEAD/ghcide
+  - bench-results/binaries/HEAD/haskell-language-server
   - bench-results/binaries/HEAD/ghc.path
-  - bench-results/binaries/*/ghcide
+  - bench-results/binaries/*/haskell-language-server
   - bench-results/binaries/*/ghc.path
   - bench-results/binaries/*/*.warmup
-  - bench-results/*/*/*/*.csv
-  - bench-results/*/*/*/*.gcStats.log
-  - bench-results/*/*/*/*.output.log
-  - bench-results/*/*/*/*.eventlog
-  - bench-results/*/*/*/*.hp
+  - bench-results/*/*/*/*/*.csv
+  - bench-results/*/*/*/*/*.gcStats.log
+  - bench-results/*/*/*/*/*.output.log
+  - bench-results/*/*/*/*/*.eventlog
+  - bench-results/*/*/*/*/*.hp
+  - bench-results/*/*/*/*/results.csv
+  - bench-results/*/*/*/*/resultDiff.csv
   - bench-results/*/*/*/results.csv
-  - bench-results/*/*/results.csv
-  - bench-results/*/results.csv
   - bench-results/*/*/*/resultDiff.csv
+  - bench-results/*/*/results.csv
   - bench-results/*/*/resultDiff.csv
+  - bench-results/*/results.csv
   - bench-results/*/resultDiff.csv
-  - bench-results/*/*/*/*.svg
-  - bench-results/*/*/*/*.diff.svg
-  - bench-results/*/*/*.svg
-  - bench-results/*/*/*/*.heap.svg
-  - Cabal-3.0.0.0
-  - lsp-types-1.0.0.1
+  - cabal
+  - lsp-types
   - all
-  - profiled-Cabal-3.0.0.0
-  - profiled-lsp-types-1.0.0.1
+  - all-binaries
+  - profiled-cabal
+  - profiled-lsp-types
   - profiled-all
+  - profiled-all-binaries
   ```

bench/config.yaml

@@ -21,15 +21,15 @@ examples:
   # Medium-sized project without TH
   - name: cabal
     package: Cabal
-    version: 3.10.2.1
+    version: 3.16.1.0
     modules:
         - src/Distribution/Simple.hs
         - src/Distribution/Types/ComponentLocalBuildInfo.hs
     extra-args: [] # extra HLS command line args
   # Small-sized project with TH
   - name: lsp-types
     package: lsp-types
-    version: 2.1.1.0
+    version: 2.4.0.0
     modules:
         - src/Language/LSP/Protocol/Types/SemanticTokens.hs
         - generated/Language/LSP/Protocol/Internal/Types/NotebookDocumentChangeEvent.hs

cabal.project

@@ -51,8 +51,8 @@ constraints:
 allow-newer:
   cabal-install-parsers:Cabal-syntax,
 
-if impl(ghc >= 9.11)
-  benchmarks: False
+
+if impl(ghc >= 9.13)
   allow-newer:
     cabal-install-parsers:base,
     cabal-install-parsers:time,

ghcide-bench/src/Experiments.hs

@@ -708,21 +708,20 @@ setup = do
             package = packageName <> "-" <> showVersion packageVersion
             hieYamlPath = path </> "hie.yaml"
         alreadySetup <- doesDirectoryExist path
-        unless alreadySetup $
-          case buildTool ?config of
+        case buildTool ?config of
             Cabal -> do
                 let cabalVerbosity = "-v" ++ show (fromEnum (verbose ?config))
-                callCommandLogging $ "cabal get " <> cabalVerbosity <> " " <> package <> " -d " <> examplesPath
-                let hieYamlPath = path </> "hie.yaml"
+                unless alreadySetup $
+                    callCommandLogging $ "cabal get " <> cabalVerbosity <> " " <> package <> " -d " <> examplesPath
                 writeFile hieYamlPath simpleCabalCradleContent
                 -- Need this in case there is a parent cabal.project somewhere
                 writeFile
                     (path </> "cabal.project")
-                    "packages: ."
+                    (cabalProjectForPackage ExamplePackage{..})
                 writeFile
                     (path </> "cabal.project.local")
                     ""
-            Stack -> do
+            Stack -> unless alreadySetup $ do
                 let stackVerbosity = case verbosity ?config of
                         Quiet  -> "--silent"
                         Normal -> ""
@@ -745,6 +744,11 @@ setup = do
                 writeFile hieYamlPath simpleStackCradleContent
         return path
 
+  checkExampleModulesExist benchDir (example ?config)
+  case (buildTool ?config, exampleDetails (example ?config)) of
+      (Cabal, ExampleHackage{}) -> buildCabalExample benchDir
+      _                         -> return ()
+
   whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True
 
   let cleanUp = case exampleDetails (example ?config) of
@@ -756,6 +760,30 @@ setup = do
 
   return SetupResult{..}
 
+checkExampleModulesExist :: FilePath -> Example -> IO ()
+checkExampleModulesExist benchDir Example{..} =
+    forM_ exampleModules $ \target -> do
+        let fullPath = benchDir </> target
+        exists <- doesFileExist fullPath
+        unless exists $
+            fail $ "Benchmark example " <> show exampleName
+                <> " is missing target file " <> show target
+                <> " at " <> fullPath
+
+buildCabalExample :: HasConfig => FilePath -> IO ()
+buildCabalExample path = do
+    output $ "cabal build all -j in " <> path
+    cmd_ (Cwd path) ("cabal" :: String) (["build", "all", "-j"] :: [String])
+
+cabalProjectForPackage :: ExamplePackage -> String
+cabalProjectForPackage ExamplePackage{packageName = "lsp-types"} =
+    unlines
+        [ "packages: ."
+        , "allow-newer: boring:base"
+        ]
+cabalProjectForPackage _ =
+    "packages: ."
+
 setupDocumentContents :: Config -> Session [DocumentPositions]
 setupDocumentContents config =
         forM (exampleModules $ example config) $ \m -> do

haskell-language-server.cabal

@@ -1979,8 +1979,7 @@ benchmark benchmark
     main-is: Main.hs
     hs-source-dirs: bench
     build-tool-depends:
-        haskell-language-server:ghcide-bench,
-        eventlog2html:eventlog2html,
+        haskell-language-server:ghcide-bench
     default-extensions:
         LambdaCase
         RecordWildCards

shake-bench/shake-bench.cabal

@@ -16,20 +16,12 @@ source-repository head
     location: https://github.com/haskell/haskell-language-server.git
 
 library
-  if impl(ghc > 9.11)
-    buildable: False
   exposed-modules:  Development.Benchmark.Rules
   hs-source-dirs:   src
   build-depends:
         aeson,
         base == 4.*,
         bytestring,
-        Chart,
-        Chart-diagrams,
-        diagrams-contrib,
-        diagrams-core,
-        diagrams-lib,
-        diagrams-svg,
         directory,
         extra >= 1.7.2,
         filepath,