Add typing burst benchmark coverage

bench/config.yaml

@@ -95,16 +95,22 @@ experiments:
     - "edit"
     - "hover"
     - "semanticTokens"
+    - "semanticTokens after typing burst"
     - "hover after edit"
+    - "hover after typing burst"
     # - "hover after cradle edit"
     - "getDefinition"
     - "getDefinition after edit"
+    - "getDefinition after typing burst"
     - "completions"
     - "completions after edit"
+    - "completions after typing burst"
     - "code actions"
     - "code actions after edit"
+    - "code actions after typing burst"
     - "code actions after cradle edit"
     - "documentSymbols after edit"
+    - "documentSymbols after typing burst"
     - "hole fit suggestions"
     - "eval execute single-line code lens"
     - "eval execute multi-line code lens"

ghcide-bench/README.md

@@ -47,6 +47,8 @@ Currently the following experiments are defined:
 - *code actions*: makes an edit that breaks typechecking and asks for code actions
 - *hole fit suggestions*: measures the performance of hole fits
 - *X after edit*: combines the *edit* and X experiments
+- *X after typing burst*: makes five hygienic edits with a 250 ms delay between
+  edits, then waits for the X response
 - *X after cradle edit*: combines the X experiments with an edit to the `hie.yaml` file
 
 One can define additional experiments easily, for e.g. formatting, code lenses, renames, etc.

ghcide-bench/src/Experiments.hs

@@ -83,6 +83,12 @@ headerEdit =
         , _text = "-- header comment \n"
         }
 
+typingBurstEditCount :: Int
+typingBurstEditCount = 5
+
+typingBurstDelay :: Seconds
+typingBurstDelay = 0.25
+
 data DocumentPositions = DocumentPositions {
     -- | A position that can be used to generate non null goto-def and completion responses
     identifierP    :: Maybe Position,
@@ -100,6 +106,14 @@ allWithIdentifierPos f docs = case applicableDocs of
   where
     applicableDocs = filter (isJust . identifierP) docs
 
+applyTypingBurst :: [DocumentPositions] -> Session ()
+applyTypingBurst docs =
+    forM_ [1..typingBurstEditCount] $ \n -> do
+        forM_ docs $ \DocumentPositions{..} ->
+            changeDoc doc [charEdit stringLiteralP]
+        when (n < typingBurstEditCount) $
+            liftIO $ sleep typingBurstDelay
+
 experiments :: HasConfig => [Bench]
 experiments =
     [
@@ -115,6 +129,15 @@ experiments =
                 Nothing -> return False
         return $ and r,
       ---------------------------------------------------------------------------------------
+      bench "semanticTokens after typing burst" $ \docs -> do
+        applyTypingBurst docs
+        r <- forM docs $ \DocumentPositions{..} -> do
+            tks <- getSemanticTokens doc
+            case tks ^? LSP._L of
+                Just _  -> return True
+                Nothing -> return False
+        return $ and r,
+      ---------------------------------------------------------------------------------------
       bench "hover" $ allWithIdentifierPos $ \DocumentPositions{..} ->
         isJust <$> getHover doc (fromJust identifierP),
       ---------------------------------------------------------------------------------------
@@ -124,6 +147,11 @@ experiments =
         flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
           isJust <$> getHover doc (fromJust identifierP),
       ---------------------------------------------------------------------------------------
+      bench "hover after typing burst" $ \docs -> do
+        applyTypingBurst docs
+        flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
+          isJust <$> getHover doc (fromJust identifierP),
+      ---------------------------------------------------------------------------------------
       bench
         "hover after cradle edit"
         (\docs -> do
@@ -158,10 +186,15 @@ experiments =
         hasDefinitions <$> getDefinitions doc (fromJust identifierP),
       ---------------------------------------------------------------------------------------
       bench "getDefinition after edit" $ \docs -> do
-          forM_ docs $ \DocumentPositions{..} ->
-            changeDoc doc [charEdit stringLiteralP]
-          flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
-            hasDefinitions <$> getDefinitions doc (fromJust identifierP),
+        forM_ docs $ \DocumentPositions{..} ->
+          changeDoc doc [charEdit stringLiteralP]
+        flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
+          hasDefinitions <$> getDefinitions doc (fromJust identifierP),
+      ---------------------------------------------------------------------------------------
+      bench "getDefinition after typing burst" $ \docs -> do
+        applyTypingBurst docs
+        flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
+          hasDefinitions <$> getDefinitions doc (fromJust identifierP),
       ---------------------------------------------------------------------------------------
       bench "documentSymbols" $ allM $ \DocumentPositions{..} -> do
         fmap (either (not . null) (not . null)) . getDocumentSymbols $ doc,
@@ -172,6 +205,11 @@ experiments =
         flip allM docs $ \DocumentPositions{..} ->
           either (not . null) (not . null) <$> getDocumentSymbols doc,
       ---------------------------------------------------------------------------------------
+      bench "documentSymbols after typing burst" $ \docs -> do
+        applyTypingBurst docs
+        flip allM docs $ \DocumentPositions{..} ->
+          either (not . null) (not . null) <$> getDocumentSymbols doc,
+      ---------------------------------------------------------------------------------------
       bench "completions" $ \docs -> do
         flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
           not . null <$> getCompletions doc (fromJust identifierP),
@@ -182,6 +220,11 @@ experiments =
         flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
           not . null <$> getCompletions doc (fromJust identifierP),
       ---------------------------------------------------------------------------------------
+      bench "completions after typing burst" $ \docs -> do
+        applyTypingBurst docs
+        flip allWithIdentifierPos docs $ \DocumentPositions{..} ->
+          not . null <$> getCompletions doc (fromJust identifierP),
+      ---------------------------------------------------------------------------------------
       bench
         "code actions"
         ( \docs -> do
@@ -206,6 +249,17 @@ experiments =
                 getCodeActions doc (Range p p))
         ),
       ---------------------------------------------------------------------------------------
+      bench
+        "code actions after typing burst"
+        ( \docs -> do
+            unless (any (isJust . identifierP) docs) $
+                error "None of the example modules is suitable for this experiment"
+            applyTypingBurst docs
+            not . null . catMaybes <$> forM docs (\DocumentPositions{..} -> do
+              forM identifierP $ \p ->
+                getCodeActions doc (Range p p))
+        ),
+      ---------------------------------------------------------------------------------------
       bench
         "code actions after cradle edit"
         ( \docs -> do
@@ -383,7 +437,7 @@ configP =
 
       packageP = ExamplePackage
             <$> strOption (long "example-package-name" <> value "Cabal")
-            <*> option versionP (long "example-package-version" <> value (makeVersion [3,6,0,0]))
+            <*> option versionP (long "example-package-version" <> value (makeVersion [3,16,1,0]))
       pathOrScriptP = ExamplePath   <$> strOption (long "example-path")
                   <|> ExampleScript <$> strOption (long "example-script") <*> many (strOption (long "example-script-args" <> help "arguments for the example generation script"))
 

ghcide-bench/test/Main.hs

@@ -36,6 +36,8 @@ benchmarkTests =
         | e <- Bench.experiments
         , Bench.name e /= "edit" -- the edit experiment does not ever fail
         , Bench.name e /= "hole fit suggestions" -- is too slow!
+        , not ("semanticTokens" `isInfixOf` Bench.name e) -- ghcide does not load the semantic-tokens plugin
+        , not ("code actions" `isInfixOf` Bench.name e) -- ghcide does not load the code-action plugin
         -- the cradle experiments are way too slow
         , not ("cradle" `isInfixOf` Bench.name e)
     ]