perf(robot): remove unnecessary token list caching from document cache

Token lists were cached separately in document._cache, but only used
as intermediate input for AST model building. No external caller ever
accessed the cached token lists directly.

Move token generation into model builder methods so tokens are only
produced on cache miss. On cache hit, neither tokenization nor model
building occurs.

- Eliminates up to 6 cached token lists per document in Language Server
- No behavior change for CLI (tokens were already uncached for version=None)
- Remove ~80 lines of redundant caching infrastructure

Author: d-biehl

packages/robot/src/robotcode/robot/diagnostics/document_cache_helper.py

@@ -8,7 +8,6 @@
 from pathlib import Path
 from typing import (
     Any,
-    Callable,
     Iterable,
     Iterator,
     List,
@@ -176,43 +175,16 @@ def __get_document_type(self, document: TextDocument) -> DocumentType:
         return DocumentType.UNKNOWN
 
     def get_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
-        if data_only:
-            return self.__get_tokens_data_only(document)
-        return self.__get_tokens(document)
-
-    def __get_tokens_data_only(self, document: TextDocument) -> List[Token]:
-        document_type = self.get_document_type(document)
-        if document_type == DocumentType.INIT:
-            return self.get_init_tokens(document, True)
-        if document_type == DocumentType.GENERAL:
-            return self.get_general_tokens(document, True)
-        if document_type == DocumentType.RESOURCE:
-            return self.get_resource_tokens(document, True)
-
-        raise UnknownFileTypeError(str(document.uri))
-
-    def __get_tokens(self, document: TextDocument) -> List[Token]:
         document_type = self.get_document_type(document)
         if document_type == DocumentType.INIT:
-            return self.get_init_tokens(document)
+            return self.__get_init_tokens(document, data_only)
         if document_type == DocumentType.GENERAL:
-            return self.get_general_tokens(document)
+            return self.__get_general_tokens(document, data_only)
         if document_type == DocumentType.RESOURCE:
-            return self.get_resource_tokens(document)
+            return self.__get_resource_tokens(document, data_only)
 
         raise UnknownFileTypeError(str(document.uri))
 
-    def get_general_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
-        if document.version is None:
-            if data_only:
-                return self.__get_general_tokens_data_only(document)
-
-            return self.__get_general_tokens(document)
-
-        if data_only:
-            return document.get_cache(self.__get_general_tokens_data_only)
-        return document.get_cache(self.__get_general_tokens)
-
     def __internal_get_tokens(
         self,
         source: Any,
@@ -270,85 +242,20 @@ def __internal_get_init_tokens(
 
         return robot.api.get_init_tokens(source, data_only=data_only, tokenize_variables=tokenize_variables)
 
-    def __get_general_tokens_data_only(self, document: TextDocument) -> List[Token]:
-        lang = self.get_languages_for_document(document)
-
-        def get(text: str) -> List[Token]:
-            with io.StringIO(text) as content:
-                return [e for e in self.__internal_get_tokens(content, True, lang=lang)]
-
-        return self.__get_tokens_internal(document, get)
-
-    def __get_general_tokens(self, document: TextDocument) -> List[Token]:
-        lang = self.get_languages_for_document(document)
-
-        def get(text: str) -> List[Token]:
-            with io.StringIO(text) as content:
-                return [e for e in self.__internal_get_tokens(content, lang=lang)]
-
-        return self.__get_tokens_internal(document, get)
-
-    def __get_tokens_internal(self, document: TextDocument, get: Callable[[str], List[Token]]) -> List[Token]:
-        return get(document.text())
-
-    def get_resource_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
-        if document.version is None:
-            if data_only:
-                return self.__get_resource_tokens_data_only(document)
-
-            return self.__get_resource_tokens(document)
-
-        if data_only:
-            return document.get_cache(self.__get_resource_tokens_data_only)
-
-        return document.get_cache(self.__get_resource_tokens)
-
-    def __get_resource_tokens_data_only(self, document: TextDocument) -> List[Token]:
-        lang = self.get_languages_for_document(document)
-
-        def get(text: str) -> List[Token]:
-            with io.StringIO(text) as content:
-                return [e for e in self.__internal_get_resource_tokens(content, True, lang=lang)]
-
-        return self.__get_tokens_internal(document, get)
-
-    def __get_resource_tokens(self, document: TextDocument) -> List[Token]:
+    def __get_general_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
         lang = self.get_languages_for_document(document)
+        with io.StringIO(document.text()) as content:
+            return [e for e in self.__internal_get_tokens(content, data_only, lang=lang)]
 
-        def get(text: str) -> List[Token]:
-            with io.StringIO(text) as content:
-                return [e for e in self.__internal_get_resource_tokens(content, lang=lang)]
-
-        return self.__get_tokens_internal(document, get)
-
-    def get_init_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
-        if document.version is None:
-            if data_only:
-                return self.__get_init_tokens_data_only(document)
-
-            return self.__get_init_tokens(document)
-
-        if data_only:
-            return document.get_cache(self.__get_init_tokens_data_only)
-        return document.get_cache(self.__get_init_tokens)
-
-    def __get_init_tokens_data_only(self, document: TextDocument) -> List[Token]:
+    def __get_resource_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
         lang = self.get_languages_for_document(document)
+        with io.StringIO(document.text()) as content:
+            return [e for e in self.__internal_get_resource_tokens(content, data_only, lang=lang)]
 
-        def get(text: str) -> List[Token]:
-            with io.StringIO(text) as content:
-                return [e for e in self.__internal_get_init_tokens(content, True, lang=lang)]
-
-        return self.__get_tokens_internal(document, get)
-
-    def __get_init_tokens(self, document: TextDocument) -> List[Token]:
+    def __get_init_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
         lang = self.get_languages_for_document(document)
-
-        def get(text: str) -> List[Token]:
-            with io.StringIO(text) as content:
-                return [e for e in self.__internal_get_init_tokens(content, lang=lang)]
-
-        return self.__get_tokens_internal(document, get)
+        with io.StringIO(document.text()) as content:
+            return [e for e in self.__internal_get_init_tokens(content, data_only, lang=lang)]
 
     def get_model(self, document: TextDocument, data_only: bool = True) -> ast.AST:
         document_type = self.get_document_type(document)
@@ -387,59 +294,56 @@ def get_tokens(source: str, data_only: bool = False, lang: Any = None) -> Iterat
     def get_general_model(self, document: TextDocument, data_only: bool = True) -> ast.AST:
         if document.version is None:
             if data_only:
-                return self.__get_general_model_data_only(document, self.get_general_tokens(document, True))
+                return self.__get_general_model_data_only(document)
 
-            return self.__get_general_model(document, self.get_general_tokens(document))
+            return self.__get_general_model(document)
 
         if data_only:
-            return document.get_cache(self.__get_general_model_data_only, self.get_general_tokens(document, True))
+            return document.get_cache(self.__get_general_model_data_only)
 
-        return document.get_cache(self.__get_general_model, self.get_general_tokens(document))
+        return document.get_cache(self.__get_general_model)
 
-    def __get_general_model_data_only(self, document: TextDocument, tokens: Iterable[Any]) -> ast.AST:
-        return self.__get_model(document, tokens, DocumentType.GENERAL)
+    def __get_general_model_data_only(self, document: TextDocument) -> ast.AST:
+        return self.__get_model(document, self.__get_general_tokens(document, True), DocumentType.GENERAL)
 
-    def __get_general_model(self, document: TextDocument, tokens: Iterable[Any]) -> ast.AST:
-        return self.__get_model(document, tokens, DocumentType.GENERAL)
+    def __get_general_model(self, document: TextDocument) -> ast.AST:
+        return self.__get_model(document, self.__get_general_tokens(document), DocumentType.GENERAL)
 
     def get_resource_model(self, document: TextDocument, data_only: bool = True) -> ast.AST:
         if document.version is None:
             if data_only:
-                return self.__get_resource_model_data_only(document, self.get_resource_tokens(document, True))
+                return self.__get_resource_model_data_only(document)
 
-            return self.__get_resource_model(document, self.get_resource_tokens(document))
+            return self.__get_resource_model(document)
 
         if data_only:
-            return document.get_cache(
-                self.__get_resource_model_data_only,
-                self.get_resource_tokens(document, True),
-            )
+            return document.get_cache(self.__get_resource_model_data_only)
 
-        return document.get_cache(self.__get_resource_model, self.get_resource_tokens(document))
+        return document.get_cache(self.__get_resource_model)
 
-    def __get_resource_model_data_only(self, document: TextDocument, tokens: Iterable[Any]) -> ast.AST:
-        return self.__get_model(document, tokens, DocumentType.RESOURCE)
+    def __get_resource_model_data_only(self, document: TextDocument) -> ast.AST:
+        return self.__get_model(document, self.__get_resource_tokens(document, True), DocumentType.RESOURCE)
 
-    def __get_resource_model(self, document: TextDocument, tokens: Iterable[Any]) -> ast.AST:
-        return self.__get_model(document, tokens, DocumentType.RESOURCE)
+    def __get_resource_model(self, document: TextDocument) -> ast.AST:
+        return self.__get_model(document, self.__get_resource_tokens(document), DocumentType.RESOURCE)
 
     def get_init_model(self, document: TextDocument, data_only: bool = True) -> ast.AST:
         if document.version is None:
             if data_only:
-                return self.__get_init_model_data_only(document, self.get_init_tokens(document, True))
+                return self.__get_init_model_data_only(document)
 
-            return self.__get_init_model(document, self.get_init_tokens(document))
+            return self.__get_init_model(document)
 
         if data_only:
-            return document.get_cache(self.__get_init_model_data_only, self.get_init_tokens(document, True))
+            return document.get_cache(self.__get_init_model_data_only)
 
-        return document.get_cache(self.__get_init_model, self.get_init_tokens(document))
+        return document.get_cache(self.__get_init_model)
 
-    def __get_init_model_data_only(self, document: TextDocument, tokens: Iterable[Any]) -> ast.AST:
-        return self.__get_model(document, tokens, DocumentType.INIT)
+    def __get_init_model_data_only(self, document: TextDocument) -> ast.AST:
+        return self.__get_model(document, self.__get_init_tokens(document, True), DocumentType.INIT)
 
-    def __get_init_model(self, document: TextDocument, tokens: Iterable[Any]) -> ast.AST:
-        return self.__get_model(document, tokens, DocumentType.INIT)
+    def __get_init_model(self, document: TextDocument) -> ast.AST:
+        return self.__get_model(document, self.__get_init_tokens(document), DocumentType.INIT)
 
     def get_namespace(self, document: TextDocument) -> Namespace:
         document_type = self.get_document_type(document)