Speed up options snapshot calculation (#20797)

The `options_snapshot` function was using up to 19% CPU in small
incremental runs in a very large codebase, when not using orjson.
Micro-optimize it, and use binary serialization instead of json.

We can still make this faster, but first let's see how much this helps.

Author: JukkaL

mypy/build.py

@@ -37,6 +37,7 @@
     TextIO,
     TypeAlias as _TypeAlias,
     TypedDict,
+    cast,
 )
 
 from librt.base64 import b64encode
@@ -59,6 +60,7 @@
     LIST_GEN,
     LITERAL_NONE,
     CacheMeta,
+    JsonValue,
     ReadBuffer,
     SerializedError,
     Tag,
@@ -74,6 +76,7 @@
     write_int,
     write_int_list,
     write_int_opt,
+    write_json_value,
     write_str,
     write_str_list,
     write_str_opt,
@@ -108,6 +111,7 @@
     MypyFile,
     SymbolTable,
 )
+from mypy.options import OPTIONS_AFFECTING_CACHE_NO_PLATFORM
 from mypy.partially_defined import PossiblyUndefinedVariableVisitor
 from mypy.semanal import SemanticAnalyzer
 from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis
@@ -1616,11 +1620,17 @@ def options_snapshot(id: str, manager: BuildManager) -> dict[str, object]:
     Separately store only the options we may compare individually, and take a hash
     of everything else. If --debug-cache is specified, fall back to full snapshot.
     """
-    snapshot = manager.options.clone_for_module(id).select_options_affecting_cache()
+    platform_opt, values = manager.options.clone_for_module(id).select_options_affecting_cache()
     if manager.options.debug_cache:
-        return snapshot
-    platform_opt = snapshot.pop("platform")
-    return {"platform": platform_opt, "other_options": hash_digest(json_dumps(snapshot))}
+        # Build full options snapshot for debugging purposes.
+        result: dict[str, object] = {"platform": platform_opt}
+        for key, val in zip(OPTIONS_AFFECTING_CACHE_NO_PLATFORM, values):
+            result[key] = val
+        return result
+    # Process most options quickly, since this is performance critical.
+    buf = WriteBuffer()
+    write_json_value(buf, cast(JsonValue, values))
+    return {"platform": platform_opt, "other_options": hash_digest(buf.getvalue())}
 
 
 def find_cache_meta(

mypy/options.py

@@ -77,6 +77,10 @@ class BuildType:
     }
 ) - {"debug_cache"}
 
+# OPTIONS_AFFECTING_CACHE without "platform", as a sorted tuple for fast iteration.
+# "platform" is handled separately in options_snapshot().
+OPTIONS_AFFECTING_CACHE_NO_PLATFORM: Final = tuple(sorted(OPTIONS_AFFECTING_CACHE - {"platform"}))
+
 # Features that are currently (or were recently) incomplete/experimental
 TYPE_VAR_TUPLE: Final = "TypeVarTuple"
 UNPACK: Final = "Unpack"
@@ -608,14 +612,21 @@ def compile_glob(self, s: str) -> Pattern[str]:
             expr += re.escape("." + part) if part != "*" else r"(\..*)?"
         return re.compile(expr + "\\Z")
 
-    def select_options_affecting_cache(self) -> dict[str, object]:
-        result: dict[str, object] = {}
-        for opt in OPTIONS_AFFECTING_CACHE:
+    def select_options_affecting_cache(self) -> tuple[str, list[object]]:
+        """Return (platform, [values...]) for options that affect the cache.
+
+        The list contains values for OPTIONS_AFFECTING_CACHE_NO_PLATFORM
+        in sorted attribute name order. Keys are omitted since the cache
+        is invalidated when the mypy version changes, and keys are constant
+        on any specific mypy version.
+        """
+        result: list[object] = []
+        for opt in OPTIONS_AFFECTING_CACHE_NO_PLATFORM:
             val = getattr(self, opt)
             if opt in ("disabled_error_codes", "enabled_error_codes"):
                 val = sorted([code.code for code in val])
-            result[opt] = val
-        return result
+            result.append(val)
+        return self.platform, result
 
     def dep_import_options(self) -> dict[str, object]:
         # These are options that can affect dependent modules as well.