Fixbench

.github/workflows/bench.yml

@@ -19,7 +19,20 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       should_skip: ${{ steps.skip_check.outputs.should_skip }}
+      bench_ghc_versions: ${{ steps.set_ghc_versions.outputs.versions }}
     steps:
+      - id: set_ghc_versions
+        run: |
+          # benching the two latest GHCs we support now
+          # since benchmark are expansive.
+          # choosing the two latest are easier to maintain and more forward looking
+          # see discussion https://github.com/haskell/haskell-language-server/pull/4118
+          # Benchmarking only the two latest GHCs we support now
+          # since benchmark are expensive.
+          # Choosing the two latest is easier to maintain and more forward looking.
+          # See discussion https://github.com/haskell/haskell-language-server/pull/4118
+          # also possible to add more GHCs if we perform better in the future.
+          echo 'versions=["9.12", "9.14"]' >> $GITHUB_OUTPUT
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@v5.3.1
         with:
@@ -45,14 +58,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # benching the two latest GHCs we support now
-        # since benchmark are expansive.
-        # choosing the two latest are easier to maintain and more forward looking
-        # see discussion https://github.com/haskell/haskell-language-server/pull/4118
-        # also possible to add more GHCs if we performs better in the future.
-        ghc:
-          - '9.8'
-          - '9.10'
+        ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }}
         os:
           - ubuntu-latest
 
@@ -120,9 +126,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ghc: ['9.8', '9.10']
+        ghc: ${{ fromJSON(needs.pre_job.outputs.bench_ghc_versions) }}
         os: [ubuntu-latest]
-        cabal: ['3.14']
+        cabal: ['3.16']
         example: ['cabal', 'lsp-types']
 
     steps:
@@ -161,7 +167,7 @@ jobs:
         column -s, -t < bench-results/unprofiled/${{ matrix.example }}/resultDiff.csv | tee bench-results/unprofiled/${{ matrix.example }}/resultDiff.txt
 
     - name: tar benchmarking artifacts
-      run:  find bench-results -name "*.csv" -or -name "*.svg" -or -name "*.html" | xargs tar -czf benchmark-artifacts.tar.gz
+      run:  find bench-results -name "*.csv" | xargs tar -czf benchmark-artifacts.tar.gz
 
     - name: Archive benchmarking artifacts
       uses: actions/upload-artifact@v6

bench/Main.hs

@@ -20,14 +20,12 @@
     |       └── <HLS plugin>
     │           ├── <experiment>.gcStats.log          - RTS -s output
     │           ├── <experiment>.csv                  - stats for the experiment
-    │           ├── <experiment>.svg                  - Graph of bytes over elapsed time
-    │           ├── <experiment>.diff.svg             - idem, including the previous version
     │           ├── <experiment>.log                  - ghcide-bench output
     │           └── results.csv                       - results of all the experiments for the example
     ├── results.csv        - aggregated results of all the experiments and versions
-    └── <experiment>.svg   - graph of bytes over elapsed time, for all the included versions
+    └── resultDiff.csv     - diff of aggregated results compared with previous version
 
-   For diff graphs, the "previous version" is the preceding entry in the list of versions
+   For diff results, the "previous version" is the preceding entry in the list of versions
    in the config file. A possible improvement is to obtain this info via `git rev-list`.
 
    To execute the script:
@@ -36,8 +34,8 @@
 
    To build a specific analysis, enumerate the desired file artifacts
 
-   > stack bench --ba "bench-results/HEAD/results.csv bench-results/HEAD/edit.diff.svg"
-   > cabal bench --benchmark-options "bench-results/HEAD/results.csv bench-results/HEAD/edit.diff.svg"
+   > stack bench --ba "bench-results/HEAD/results.csv"
+   > cabal bench --benchmark-options "bench-results/HEAD/results.csv"
 
  -}
 {-# LANGUAGE DeriveAnyClass     #-}
@@ -165,8 +163,6 @@ createBuildSystem config = do
   benchRules build (MkBenchRules (askOracle $ GetSamples ()) benchHls warmupHls "haskell-language-server" (parallelism configStatic))
   addGetParentOracle
   csvRules build
-  svgRules build
-  heapProfileRules build
   phonyRules "" binaryName NoProfiling build (examples configStatic)
 
   whenJust (profileInterval configStatic) $ \i -> do

bench/README.md

@@ -41,30 +41,30 @@ The Shake script supports a number of phony targets that allow running a subset
 ```
 Targets:
   - bench-results/binaries/*/commitid
-  - bench-results/binaries/HEAD/ghcide
+  - bench-results/binaries/HEAD/haskell-language-server
   - bench-results/binaries/HEAD/ghc.path
-  - bench-results/binaries/*/ghcide
+  - bench-results/binaries/*/haskell-language-server
   - bench-results/binaries/*/ghc.path
   - bench-results/binaries/*/*.warmup
-  - bench-results/*/*/*/*.csv
-  - bench-results/*/*/*/*.gcStats.log
-  - bench-results/*/*/*/*.output.log
-  - bench-results/*/*/*/*.eventlog
-  - bench-results/*/*/*/*.hp
+  - bench-results/*/*/*/*/*.csv
+  - bench-results/*/*/*/*/*.gcStats.log
+  - bench-results/*/*/*/*/*.output.log
+  - bench-results/*/*/*/*/*.eventlog
+  - bench-results/*/*/*/*/*.hp
+  - bench-results/*/*/*/*/results.csv
+  - bench-results/*/*/*/*/resultDiff.csv
   - bench-results/*/*/*/results.csv
-  - bench-results/*/*/results.csv
-  - bench-results/*/results.csv
   - bench-results/*/*/*/resultDiff.csv
+  - bench-results/*/*/results.csv
   - bench-results/*/*/resultDiff.csv
+  - bench-results/*/results.csv
   - bench-results/*/resultDiff.csv
-  - bench-results/*/*/*/*.svg
-  - bench-results/*/*/*/*.diff.svg
-  - bench-results/*/*/*.svg
-  - bench-results/*/*/*/*.heap.svg
-  - Cabal-3.0.0.0
-  - lsp-types-1.0.0.1
+  - cabal
+  - lsp-types
   - all
-  - profiled-Cabal-3.0.0.0
-  - profiled-lsp-types-1.0.0.1
+  - all-binaries
+  - profiled-cabal
+  - profiled-lsp-types
   - profiled-all
+  - profiled-all-binaries
   ```

bench/config.yaml

@@ -21,15 +21,15 @@ examples:
   # Medium-sized project without TH
   - name: cabal
     package: Cabal
-    version: 3.10.2.1
+    version: 3.16.1.0
     modules:
         - src/Distribution/Simple.hs
         - src/Distribution/Types/ComponentLocalBuildInfo.hs
     extra-args: [] # extra HLS command line args
   # Small-sized project with TH
   - name: lsp-types
     package: lsp-types
-    version: 2.1.1.0
+    version: 2.4.0.0
     modules:
         - src/Language/LSP/Protocol/Types/SemanticTokens.hs
         - generated/Language/LSP/Protocol/Internal/Types/NotebookDocumentChangeEvent.hs

cabal.project

@@ -51,8 +51,8 @@ constraints:
 allow-newer:
   cabal-install-parsers:Cabal-syntax,
 
-if impl(ghc >= 9.11)
-  benchmarks: False
+
+if impl(ghc >= 9.13)
   allow-newer:
     cabal-install-parsers:base,
     cabal-install-parsers:time,

ghcide-bench/src/Experiments.hs

@@ -708,21 +708,20 @@ setup = do
             package = packageName <> "-" <> showVersion packageVersion
             hieYamlPath = path </> "hie.yaml"
         alreadySetup <- doesDirectoryExist path
-        unless alreadySetup $
-          case buildTool ?config of
+        case buildTool ?config of
             Cabal -> do
                 let cabalVerbosity = "-v" ++ show (fromEnum (verbose ?config))
-                callCommandLogging $ "cabal get " <> cabalVerbosity <> " " <> package <> " -d " <> examplesPath
-                let hieYamlPath = path </> "hie.yaml"
+                unless alreadySetup $
+                    callCommandLogging $ "cabal get " <> cabalVerbosity <> " " <> package <> " -d " <> examplesPath
                 writeFile hieYamlPath simpleCabalCradleContent
                 -- Need this in case there is a parent cabal.project somewhere
                 writeFile
                     (path </> "cabal.project")
-                    "packages: ."
+                    (cabalProjectForPackage ExamplePackage{..})
                 writeFile
                     (path </> "cabal.project.local")
                     ""
-            Stack -> do
+            Stack -> unless alreadySetup $ do
                 let stackVerbosity = case verbosity ?config of
                         Quiet  -> "--silent"
                         Normal -> ""
@@ -745,6 +744,11 @@ setup = do
                 writeFile hieYamlPath simpleStackCradleContent
         return path
 
+  checkExampleModulesExist benchDir (example ?config)
+  case (buildTool ?config, exampleDetails (example ?config)) of
+      (Cabal, ExampleHackage{}) -> buildCabalExample benchDir
+      _                         -> return ()
+
   whenJust (shakeProfiling ?config) $ createDirectoryIfMissing True
 
   let cleanUp = case exampleDetails (example ?config) of
@@ -756,6 +760,30 @@ setup = do
 
   return SetupResult{..}
 
+checkExampleModulesExist :: FilePath -> Example -> IO ()
+checkExampleModulesExist benchDir Example{..} =
+    forM_ exampleModules $ \target -> do
+        let fullPath = benchDir </> target
+        exists <- doesFileExist fullPath
+        unless exists $
+            fail $ "Benchmark example " <> show exampleName
+                <> " is missing target file " <> show target
+                <> " at " <> fullPath
+
+buildCabalExample :: HasConfig => FilePath -> IO ()
+buildCabalExample path = do
+    output $ "cabal build all -j in " <> path
+    cmd_ (Cwd path) ("cabal" :: String) (["build", "all", "-j"] :: [String])
+
+cabalProjectForPackage :: ExamplePackage -> String
+cabalProjectForPackage ExamplePackage{packageName = "lsp-types"} =
+    unlines
+        [ "packages: ."
+        , "allow-newer: boring:base"
+        ]
+cabalProjectForPackage _ =
+    "packages: ."
+
 setupDocumentContents :: Config -> Session [DocumentPositions]
 setupDocumentContents config =
         forM (exampleModules $ example config) $ \m -> do

haskell-language-server.cabal

@@ -1979,8 +1979,7 @@ benchmark benchmark
     main-is: Main.hs
     hs-source-dirs: bench
     build-tool-depends:
-        haskell-language-server:ghcide-bench,
-        eventlog2html:eventlog2html,
+        haskell-language-server:ghcide-bench
     default-extensions:
         LambdaCase
         RecordWildCards

shake-bench/shake-bench.cabal

@@ -16,20 +16,12 @@ source-repository head
     location: https://github.com/haskell/haskell-language-server.git
 
 library
-  if impl(ghc > 9.11)
-    buildable: False
   exposed-modules:  Development.Benchmark.Rules
   hs-source-dirs:   src
   build-depends:
         aeson,
         base == 4.*,
         bytestring,
-        Chart,
-        Chart-diagrams,
-        diagrams-contrib,
-        diagrams-core,
-        diagrams-lib,
-        diagrams-svg,
         directory,
         extra >= 1.7.2,
         filepath,