feat: extend tokenizer cache APIs

Add compute_cache_token() alongside compute_data_token() and
compute_behavior_token(), refactor cache_key() to delegate to it, and
rename the cached class attribute to __ccflow_tokenizer_cache__ so it
matches __ccflow_tokenizer_deps__.

This commit also keeps class support in __ccflow_tokenizer_deps__,
including recursive class-dependency detection, and adds regression
coverage for combined cache tokens and cache-key integration.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Pascal Tomecek <pascal.tomecek@cubistsystematic.com>

Author: ptomecek

ccflow/evaluators/common.py

@@ -58,22 +58,6 @@ def combine_evaluators(first: Optional[EvaluatorBase], second: Optional[Evaluato
         return MultiEvaluator(evaluators=[first, second])
 
 
-def _model_tokens(model: BaseModel, *, include_data: bool = True) -> List[str]:
-    from ..utils.tokenize import compute_behavior_token, compute_data_token
-
-    tokens = [compute_data_token(model.model_dump(mode="python"))] if include_data else []
-    behavior = compute_behavior_token(type(model))
-    if behavior is not None:
-        tokens.append(behavior)
-    return tokens
-
-
-def _combine_tokens(tokens: List[str]) -> bytes:
-    from ..utils.tokenize import compute_data_token
-
-    return compute_data_token(tuple(tokens)).encode("utf-8")
-
-
 def _flatten_cache_key_context(flow_obj: ModelEvaluationContext) -> tuple[ModelEvaluationContext, str, List[EvaluatorBase]]:
     fn = flow_obj.fn
     non_transparent: List[EvaluatorBase] = []
@@ -256,17 +240,22 @@ def cache_key(flow_obj: Union[ModelEvaluationContext, ContextBase, CallableModel
     Args:
         flow_obj: The object to be tokenized to form the cache key.
     """
-    from ..utils.tokenize import compute_data_token
+    from ..utils.tokenize import compute_cache_token
 
     if isinstance(flow_obj, ModelEvaluationContext):
         flow_obj, fn, non_transparent = _flatten_cache_key_context(flow_obj)
-        tokens = [compute_data_token({**flow_obj.model_dump(mode="python"), "fn": fn})]
-        tokens.extend(_model_tokens(flow_obj.model, include_data=False))
-        for evaluator in non_transparent:
-            tokens.extend(_model_tokens(evaluator))
-        return _combine_tokens(tokens)
+        return compute_cache_token(
+            data_values=[
+                {**flow_obj.model_dump(mode="python"), "fn": fn},
+                *(evaluator.model_dump(mode="python") for evaluator in non_transparent),
+            ],
+            behavior_classes=[type(flow_obj.model), *(type(evaluator) for evaluator in non_transparent)],
+        ).encode("utf-8")
     elif isinstance(flow_obj, (ContextBase, CallableModel)):
-        return _combine_tokens(_model_tokens(flow_obj))
+        return compute_cache_token(
+            data_values=[flow_obj.model_dump(mode="python")],
+            behavior_classes=[type(flow_obj)],
+        ).encode("utf-8")
     else:
         raise TypeError(f"object of type {type(flow_obj)} cannot be serialized by this function!")
 

ccflow/tests/utils/test_behavior_hash.py

@@ -1,10 +1,12 @@
 """Tests for tokenize helpers used by cache_key()."""
 
+import pytest
+
 from ccflow.callable import CallableModel, ContextBase, EvaluatorBase, ModelEvaluationContext
 from ccflow.context import NullContext
 from ccflow.evaluators.common import cache_key
 from ccflow.result import GenericResult
-from ccflow.utils.tokenize import compute_behavior_token, compute_data_token
+from ccflow.utils.tokenize import compute_behavior_token, compute_cache_token, compute_data_token
 
 # ---------------------------------------------------------------------------
 # Data token
@@ -21,6 +23,39 @@ def test_different_values_different_tokens(self):
         assert compute_data_token({"x": 1}) != compute_data_token({"x": 2})
 
 
+class TestComputeCacheToken:
+    def test_deterministic(self):
+        class Helper:
+            def f(self):
+                return 1
+
+        token1 = compute_cache_token(data_values=[{"x": 1}], behavior_classes=[Helper])
+        token2 = compute_cache_token(data_values=[{"x": 1}], behavior_classes=[Helper])
+        assert token1 == token2
+
+    def test_data_changes_token(self):
+        class Helper:
+            def f(self):
+                return 1
+
+        token1 = compute_cache_token(data_values=[{"x": 1}], behavior_classes=[Helper])
+        token2 = compute_cache_token(data_values=[{"x": 2}], behavior_classes=[Helper])
+        assert token1 != token2
+
+    def test_behavior_changes_token(self):
+        class HelperA:
+            def f(self):
+                return 1
+
+        class HelperB:
+            def f(self):
+                return 2
+
+        token1 = compute_cache_token(data_values=[{"x": 1}], behavior_classes=[HelperA])
+        token2 = compute_cache_token(data_values=[{"x": 1}], behavior_classes=[HelperB])
+        assert token1 != token2
+
+
 # ---------------------------------------------------------------------------
 # Basic behavior
 # ---------------------------------------------------------------------------
@@ -77,7 +112,7 @@ def f(self):
                 return 1
 
         token = compute_behavior_token(M)
-        assert M.__behavior_token_cache__ == token
+        assert M.__ccflow_tokenizer_cache__ == token
         # Second call returns cached value
         assert compute_behavior_token(M) is token
 
@@ -294,6 +329,29 @@ def f(self):
 
         assert compute_behavior_token(A) != compute_behavior_token(B)
 
+    def test_class_dep_included(self):
+        class HelperA:
+            def f(self):
+                return 1
+
+        class HelperB:
+            def f(self):
+                return 2
+
+        class A:
+            __ccflow_tokenizer_deps__ = [HelperA]
+
+            def f(self):
+                return 1
+
+        class B:
+            __ccflow_tokenizer_deps__ = [HelperB]
+
+            def f(self):
+                return 1
+
+        assert compute_behavior_token(A) != compute_behavior_token(B)
+
     def test_subclass_deps_extend_inherited_deps(self):
         def base_a():
             return 1
@@ -324,6 +382,21 @@ class SubB(BaseB):
 
         assert compute_behavior_token(SubA) != compute_behavior_token(SubB)
 
+    def test_recursive_class_deps_raise(self):
+        class A:
+            def f(self):
+                return 1
+
+        class B:
+            def g(self):
+                return 2
+
+        A.__ccflow_tokenizer_deps__ = [B]
+        B.__ccflow_tokenizer_deps__ = [A]
+
+        with pytest.raises(TypeError, match="Recursive __ccflow_tokenizer_deps__ class dependency"):
+            compute_behavior_token(A)
+
 
 # ---------------------------------------------------------------------------
 # Integration with cache_key()
@@ -412,6 +485,33 @@ def helper(self, x=2):
 
         assert cache_key(A()) != cache_key(B())
 
+    def test_class_dep_changes_key(self):
+        from ccflow import Flow
+
+        class HelperA:
+            def f(self):
+                return 1
+
+        class HelperB:
+            def f(self):
+                return 2
+
+        class A(CallableModel):
+            __ccflow_tokenizer_deps__ = [HelperA]
+
+            @Flow.call
+            def __call__(self, context: NullContext) -> GenericResult:
+                return GenericResult(value=1)
+
+        class B(CallableModel):
+            __ccflow_tokenizer_deps__ = [HelperB]
+
+            @Flow.call
+            def __call__(self, context: NullContext) -> GenericResult:
+                return GenericResult(value=1)
+
+        assert cache_key(A()) != cache_key(B())
+
     def test_opaque_evaluator_behavior_changes_key(self):
         from ccflow import Flow
 
@@ -512,7 +612,7 @@ def __call__(self):
         assert compute_behavior_token(Sub) != compute_behavior_token(Base)
 
     def test_subclass_cache_independent(self):
-        """Parent and subclass don't share __behavior_token_cache__."""
+        """Parent and subclass don't share __ccflow_tokenizer_cache__."""
 
         class Base:
             def f(self):

ccflow/utils/__init__.py

@@ -1,4 +1,4 @@
 from .chunker import *
 from .core import *
 from .logging import *
-from .tokenize import compute_behavior_token, compute_data_token, normalize_token, tokenize
+from .tokenize import compute_behavior_token, compute_cache_token, compute_data_token, normalize_token, tokenize