Implement caching for system prompt in AsyncHTTPClient and HTTPClient to reduce payload size

Author: devasheeshG

recallrai/async_session.py

@@ -136,13 +136,18 @@ async def get_context(
             TimeoutError: If the request times out.
             RecallrAIError: For other API-related errors.
         """
+        # Fetch and cache the system prompt client-side to avoid sending ~20 KB on every request
+        system_prompt_text: Optional[str] = None
+        if include_system_prompt:
+            system_prompt_text = await self._http.get_cached_system_prompt()
+
         params = {
             "recall_strategy": recall_strategy.value,
             "min_top_k": min_top_k,
             "max_top_k": max_top_k,
             "memories_threshold": memories_threshold,
             "summaries_threshold": summaries_threshold,
-            "include_system_prompt": include_system_prompt,
+            "include_system_prompt": False,
             "include_metadata_ids": include_metadata_ids,
         }
         if last_n_messages is not None:
@@ -173,7 +178,10 @@ async def get_context(
             logger.warning("You are trying to get context for a processed session. Why do you need it?")
         elif self.status == SessionStatus.PROCESSING:
             logger.warning("You are trying to get context for a processing session. Why do you need it?")
-        return ContextResponse.from_api_response(response.json())
+        result = ContextResponse.from_api_response(response.json())
+        if system_prompt_text is not None and result.context is not None:
+            result = result.model_copy(update={"context": system_prompt_text + "\n\n\n" + result.context})
+        return result
 
     async def get_context_stream(
         self,
@@ -206,13 +214,18 @@ async def get_context_stream(
         Yields:
             Streaming ContextResponse objects with status updates and final context.
         """
+        # Fetch and cache the system prompt client-side to avoid sending ~20 KB on every request
+        system_prompt_text: Optional[str] = None
+        if include_system_prompt:
+            system_prompt_text = await self._http.get_cached_system_prompt()
+
         params = {
             "recall_strategy": recall_strategy.value,
             "min_top_k": min_top_k,
             "max_top_k": max_top_k,
             "memories_threshold": memories_threshold,
             "summaries_threshold": summaries_threshold,
-            "include_system_prompt": include_system_prompt,
+            "include_system_prompt": False,
             "include_metadata_ids": include_metadata_ids,
             "stream": True,
         }
@@ -233,7 +246,10 @@ async def get_context_stream(
             if not payload:
                 continue
             data = json.loads(payload)
-            yield ContextResponse.from_api_response(data)
+            event = ContextResponse.from_api_response(data)
+            if system_prompt_text is not None and event.is_final and event.context is not None:
+                event = event.model_copy(update={"context": system_prompt_text + "\n\n\n" + event.context})
+            yield event
 
     async def update(self, new_metadata: Optional[Dict[str, Any]] = None) -> None:
         """

recallrai/session.py

@@ -135,13 +135,18 @@ def get_context(
             TimeoutError: If the request times out.
             RecallrAIError: For other API-related errors.
         """
+        # Fetch and cache the system prompt client-side to avoid sending ~20 KB on every request
+        system_prompt_text: Optional[str] = None
+        if include_system_prompt:
+            system_prompt_text = self._http.get_cached_system_prompt()
+
         params = {
             "recall_strategy": recall_strategy.value,
             "min_top_k": min_top_k,
             "max_top_k": max_top_k,
             "memories_threshold": memories_threshold,
             "summaries_threshold": summaries_threshold,
-            "include_system_prompt": include_system_prompt,
+            "include_system_prompt": False,
             "include_metadata_ids": include_metadata_ids,
         }
         if last_n_messages is not None:
@@ -172,7 +177,10 @@ def get_context(
             logger.warning("You are trying to get context for a processed session. Why do you need it?")
         elif self.status == SessionStatus.PROCESSING:
             logger.warning("You are trying to get context for a processing session. Why do you need it?")
-        return ContextResponse.from_api_response(response.json())
+        result = ContextResponse.from_api_response(response.json())
+        if system_prompt_text is not None and result.context is not None:
+            result = result.model_copy(update={"context": system_prompt_text + "\n\n\n" + result.context})
+        return result
 
     def get_context_stream(
         self,
@@ -205,13 +213,18 @@ def get_context_stream(
         Yields:
             Streaming ContextResponse objects with status updates and final context.
         """
+        # Fetch and cache the system prompt client-side to avoid sending ~20 KB on every request
+        system_prompt_text: Optional[str] = None
+        if include_system_prompt:
+            system_prompt_text = self._http.get_cached_system_prompt()
+
         params = {
             "recall_strategy": recall_strategy.value,
             "min_top_k": min_top_k,
             "max_top_k": max_top_k,
             "memories_threshold": memories_threshold,
             "summaries_threshold": summaries_threshold,
-            "include_system_prompt": include_system_prompt,
+            "include_system_prompt": False,
             "stream": True,
             "include_metadata_ids": include_metadata_ids,
         }
@@ -232,7 +245,10 @@ def get_context_stream(
             if not payload:
                 continue
             data = json.loads(payload)
-            yield ContextResponse.from_api_response(data)
+            event = ContextResponse.from_api_response(data)
+            if system_prompt_text is not None and event.is_final and event.context is not None:
+                event = event.model_copy(update={"context": system_prompt_text + "\n\n\n" + event.context})
+            yield event
 
     def delete(self) -> None:
         """

recallrai/utils/async_http_client.py

@@ -2,6 +2,7 @@
 Async HTTP client for making requests to the RecallrAI API.
 """
 
+import time
 from json import JSONDecodeError
 from typing import Any, AsyncIterator, Dict, Optional
 from httpx import Response, AsyncClient, TimeoutException, ConnectError, Limits
@@ -40,6 +41,22 @@ def __init__(
         self.base_url = base_url.rstrip("/")
         self.timeout = timeout
         self._client: Optional[AsyncClient] = None
+        self._system_prompt_cache: Optional[str] = None
+        self._system_prompt_cache_expires_at: float = 0.0
+
+    async def get_cached_system_prompt(self) -> str:
+        """Fetch the global system prompt, using a 1-hour in-memory cache to
+        avoid shipping the ~20 KB prompt on every request."""
+        now = time.monotonic()
+        if (
+            self._system_prompt_cache is not None
+            and now < self._system_prompt_cache_expires_at
+        ):
+            return self._system_prompt_cache  # type: ignore[return-value]
+        response = await self.get("/api/v1/system-prompt")
+        self._system_prompt_cache = response.json()["system_prompt"]
+        self._system_prompt_cache_expires_at = now + 3600
+        return self._system_prompt_cache  # type: ignore[return-value]
 
     async def __aenter__(self):
         """Async context manager entry."""

recallrai/utils/http_client.py

@@ -2,6 +2,7 @@
 HTTP client for making requests to the RecallrAI API.
 """
 
+import time
 from json import JSONDecodeError
 from typing import Any, Dict, Iterator, Optional
 from httpx import Response, Client, TimeoutException, ConnectError, Limits
@@ -56,6 +57,22 @@ def __init__(
                 "User-Agent": "RecallrAI-Python-SDK/0.5.8",
             },
         )
+        self._system_prompt_cache: Optional[str] = None
+        self._system_prompt_cache_expires_at: float = 0.0
+
+    def get_cached_system_prompt(self) -> str:
+        """Fetch the global system prompt, using a 1-hour in-memory cache to
+        avoid shipping the ~20 KB prompt on every request."""
+        now = time.monotonic()
+        if (
+            self._system_prompt_cache is not None
+            and now < self._system_prompt_cache_expires_at
+        ):
+            return self._system_prompt_cache  # type: ignore[return-value]
+        response = self.get("/api/v1/system-prompt")
+        self._system_prompt_cache = response.json()["system_prompt"]
+        self._system_prompt_cache_expires_at = now + 3600
+        return self._system_prompt_cache  # type: ignore[return-value]
 
     def request(
         self,