fix: normalize HTML responses for deterministic Django replay (#58)

Author: jy-tan

drift/core/communication/communicator.py

@@ -612,31 +612,7 @@ def _extract_response_data(self, struct: Any) -> dict[str, Any]:
             return {}
 
         try:
-
-            def value_to_python(value):
-                """Convert protobuf Value to Python type."""
-                if hasattr(value, "null_value"):
-                    return None
-                elif hasattr(value, "number_value"):
-                    return value.number_value
-                elif hasattr(value, "string_value"):
-                    return value.string_value
-                elif hasattr(value, "bool_value"):
-                    return value.bool_value
-                elif hasattr(value, "struct_value") and value.struct_value:
-                    return struct_to_dict(value.struct_value)
-                elif hasattr(value, "list_value") and value.list_value:
-                    return [value_to_python(v) for v in value.list_value.values]
-                return None
-
-            def struct_to_dict(s):
-                """Convert protobuf Struct to Python dict."""
-                if not hasattr(s, "fields"):
-                    return {}
-                result = {}
-                for key, value in s.fields.items():
-                    result[key] = value_to_python(value)
-                return result
+            from ..protobuf_utils import struct_to_dict
 
             data = struct_to_dict(struct)
 

drift/core/communication/types.py

@@ -379,6 +379,8 @@ def extract_response_data(struct: Any) -> dict[str, Any]:
 
     The CLI returns response data wrapped in a Struct with a "response" field.
     """
+    from ..protobuf_utils import value_to_python
+
     try:
         # Handle betterproto dict-like struct
         if hasattr(struct, "items"):
@@ -391,8 +393,8 @@ def extract_response_data(struct: Any) -> dict[str, Any]:
         if hasattr(struct, "fields"):
             fields = struct.fields
             if "response" in fields:
-                return _value_to_python(fields["response"])
-            return {k: _value_to_python(v) for k, v in fields.items()}
+                return value_to_python(fields["response"])
+            return {k: value_to_python(v) for k, v in fields.items()}
 
         # Direct dict access
         if isinstance(struct, dict):
@@ -403,20 +405,3 @@ def extract_response_data(struct: Any) -> dict[str, Any]:
         return {}
     except Exception:
         return {}
-
-
-def _value_to_python(value: Any) -> Any:
-    """Convert a protobuf Value to Python native type."""
-    if hasattr(value, "null_value"):
-        return None
-    if hasattr(value, "number_value"):
-        return value.number_value
-    if hasattr(value, "string_value"):
-        return value.string_value
-    if hasattr(value, "bool_value"):
-        return value.bool_value
-    if hasattr(value, "struct_value"):
-        return {k: _value_to_python(v) for k, v in value.struct_value.fields.items()}
-    if hasattr(value, "list_value"):
-        return [_value_to_python(v) for v in value.list_value.values]
-    return value

drift/core/protobuf_utils.py

@@ -0,0 +1,102 @@
+"""Protobuf conversion utilities.
+
+This module provides utilities for converting protobuf Value and Struct objects
+to Python native types. Handles both Google protobuf and betterproto variants.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def value_to_python(value: Any) -> Any:
+    """Convert protobuf Value or Python native type to Python native type.
+
+    Handles:
+    1. Python native types (from betterproto's Struct which stores values directly)
+    2. Google protobuf Value objects (from google.protobuf.struct_pb2)
+    3. betterproto Value objects (from betterproto.lib.google.protobuf)
+
+    Args:
+        value: A protobuf Value, native Python type, or nested structure
+
+    Returns:
+        Python native type (None, bool, int, float, str, list, or dict)
+    """
+    # Handle Python native types (from betterproto or already converted)
+    if value is None:
+        return None
+    if isinstance(value, (bool, int, float, str)):
+        # Note: bool must be checked before int since bool is a subclass of int
+        return value
+    if isinstance(value, list):
+        return [value_to_python(v) for v in value]
+    if isinstance(value, dict):
+        return {k: value_to_python(v) for k, v in value.items()}
+
+    # Handle Google protobuf Value objects using WhichOneof
+    if hasattr(value, "WhichOneof"):
+        kind = value.WhichOneof("kind")
+        if kind == "null_value":
+            return None
+        elif kind == "number_value":
+            return value.number_value
+        elif kind == "string_value":
+            return value.string_value
+        elif kind == "bool_value":
+            return value.bool_value
+        elif kind == "struct_value":
+            return struct_to_dict(value.struct_value)
+        elif kind == "list_value":
+            return [value_to_python(v) for v in value.list_value.values]
+
+    # Handle betterproto Value objects using is_set method
+    if hasattr(value, "is_set"):
+        try:
+            if value.is_set("null_value"):
+                return None
+            elif value.is_set("number_value"):
+                return value.number_value
+            elif value.is_set("string_value"):
+                return value.string_value
+            elif value.is_set("bool_value"):
+                return value.bool_value
+            elif value.is_set("struct_value"):
+                sv = value.struct_value
+                if isinstance(sv, dict):
+                    return {k: value_to_python(v) for k, v in sv.get("fields", sv).items()}
+                return struct_to_dict(sv)
+            elif value.is_set("list_value"):
+                lv = value.list_value
+                # Handle dict-style list_value (betterproto can store dicts)
+                if isinstance(lv, dict):
+                    return [value_to_python(v) for v in lv.get("values", [])]
+                return [value_to_python(v) for v in lv.values]
+        except (AttributeError, TypeError):
+            pass  # Not a betterproto Value, fall through
+
+    # Fallback: return the value as-is
+    return value
+
+
+def struct_to_dict(struct: Any) -> dict[str, Any]:
+    """Convert protobuf Struct to Python dict.
+
+    Args:
+        struct: A protobuf Struct object (Google or betterproto)
+
+    Returns:
+        Python dict with converted values
+    """
+    if not struct:
+        return {}
+
+    # Handle dict-like struct (betterproto sometimes returns dicts directly)
+    if isinstance(struct, dict):
+        return {k: value_to_python(v) for k, v in struct.items()}
+
+    # Handle struct with fields attribute
+    if hasattr(struct, "fields"):
+        return {k: value_to_python(v) for k, v in struct.fields.items()}
+
+    return {}