feat: support timestamp="unset" to retain content without a date (#465)

* feat: support timestamp="unset" to retain content without a date

When callers retain timeless content (e.g. fictional documents, static
reference material), passing timestamp="unset" now skips the utcnow()
default so mentioned_at is stored as NULL instead of an artificial date.

- HTTP: validate_timestamp recognises "unset" sentinel and threads it
  through api_retain as event_date=None (key present, value None), which
  the orchestrator distinguishes from key-absent (still defaults to now)
- Orchestrator: new branching logic separates "key absent" → utcnow()
  from "key present but None" → no date
- types.py: RetainContent.event_date and ProcessedFact.mentioned_at are
  now datetime | None; removed the unused _now_utc factory
- fact_extraction.py: all event_date params accept datetime | None;
  _build_user_message emits "Event Date: Unknown" when None; removed
  mentioned_at from the Fact LLM response model (LLM never sets it)
- embedding_processing: skip date suffix when fact_date is None
- entity_resolver: COALESCE(event_date, now()) for first_seen/last_seen
  so entities table NOT NULL constraint is preserved
- link_utils: skip temporal linking for units without event_date
- Migration aa2b3c4d5e6f: DROP NOT NULL on memory_units.event_date
- Tests: test_retain_no_timestamp and test_retain_omit_timestamp_defaults_to_now
- Docs + OpenAPI + TypeScript client updated

* refactor: replace _TIMESTAMP_UNKNOWN sentinel with plain string comparison

The sentinel object() was only needed to distinguish "unset" from None
at the boundary — but since the field type is datetime | str | None,
"unset" can pass through the validator unchanged and be compared directly.

* chore: regenerate OpenAPI spec and clients after timestamp type change

timestamp field is now datetime | str | None to accept the "unset" sentinel value.

Author: nicoloboschi

hindsight-api/hindsight_api/alembic/versions/aa2b3c4d5e6f_nullable_event_date.py

@@ -0,0 +1,36 @@
+"""Make event_date nullable in memory_units to support timestamp-free content
+
+Revision ID: aa2b3c4d5e6f
+Revises: z1u2v3w4x5y6
+Create Date: 2026-03-02
+
+When callers retain content without a timestamp (e.g. fictional documents, static text),
+the event_date column should be allowed to be NULL rather than defaulting to utcnow().
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "aa2b3c4d5e6f"
+down_revision: str | Sequence[str] | None = "z1u2v3w4x5y6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+    op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date DROP NOT NULL")
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+    # Backfill NULLs with now() before restoring the NOT NULL constraint
+    op.execute(f"UPDATE {schema}memory_units SET event_date = now() WHERE event_date IS NULL")
+    op.execute(f"ALTER TABLE {schema}memory_units ALTER COLUMN event_date SET NOT NULL")

hindsight-api/hindsight_api/api/http.py

@@ -383,7 +383,15 @@ class MemoryItem(BaseModel):
     )
 
     content: str
-    timestamp: datetime | None = None
+    timestamp: datetime | str | None = Field(
+        default=None,
+        description=(
+            "When the content occurred. "
+            "Accepts an ISO 8601 datetime string (e.g. '2024-01-15T10:30:00Z'), null/omitted (defaults to now), "
+            "or the special string 'unset' to explicitly store without any timestamp "
+            "(use this for timeless content such as fictional documents or static reference material)."
+        ),
+    )
     context: str | None = None
     metadata: dict[str, str] | None = None
     document_id: str | None = Field(default=None, description="Optional document ID for this memory item.")
@@ -414,12 +422,14 @@ def validate_timestamp(cls, v):
         if isinstance(v, datetime):
             return v
         if isinstance(v, str):
+            if v.lower() == "unset":
+                return "unset"
             try:
                 # Try parsing as ISO format
                 return datetime.fromisoformat(v.replace("Z", "+00:00"))
             except ValueError as e:
                 raise ValueError(
-                    f"Invalid timestamp/event_date format: '{v}'. Expected ISO format like '2024-01-15T10:30:00' or '2024-01-15T10:30:00Z'"
+                    f"Invalid timestamp/event_date format: '{v}'. Expected ISO format like '2024-01-15T10:30:00' or '2024-01-15T10:30:00Z', or the special value 'unset' to store without a timestamp."
                 ) from e
         raise ValueError(f"timestamp must be a string or datetime, got {type(v).__name__}")
 
@@ -3878,7 +3888,9 @@ async def api_retain(
             contents = []
             for item in request.items:
                 content_dict = {"content": item.content}
-                if item.timestamp:
+                if item.timestamp == "unset":
+                    content_dict["event_date"] = None
+                elif item.timestamp:
                     content_dict["event_date"] = item.timestamp
                 if item.context:
                     content_dict["context"] = item.context

hindsight-api/hindsight_api/engine/entity_resolver.py

@@ -237,7 +237,7 @@ async def _resolve_entities_batch_impl(
             rows = await conn.fetch(
                 f"""
                 INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count)
-                SELECT $1, name, event_date, event_date, cnt
+                SELECT $1, name, COALESCE(event_date, now()), COALESCE(event_date, now()), cnt
                 FROM unnest($2::text[], $3::timestamptz[], $4::int[]) AS t(name, event_date, cnt)
                 ON CONFLICT (bank_id, LOWER(canonical_name))
                 DO UPDATE SET
@@ -408,7 +408,7 @@ async def _create_entity(
         entity_id = await conn.fetchval(
             f"""
             INSERT INTO {fq_table("entities")} (bank_id, canonical_name, first_seen, last_seen, mention_count)
-            VALUES ($1, $2, $3, $4, 1)
+            VALUES ($1, $2, COALESCE($3, now()), COALESCE($4, now()), 1)
             ON CONFLICT (bank_id, LOWER(canonical_name))
             DO UPDATE SET
                 mention_count = {fq_table("entities")}.mention_count + 1,

hindsight-api/hindsight_api/engine/retain/embedding_processing.py

@@ -29,9 +29,12 @@ def augment_texts_with_dates(facts: list[ExtractedFact], format_date_fn) -> list
     for fact in facts:
         # Use occurred_start as the representative date
         fact_date = fact.occurred_start or fact.mentioned_at
-        readable_date = format_date_fn(fact_date)
-        # Augment text with date for embedding (but store original text in DB)
-        augmented_text = f"{fact.fact_text} (happened in {readable_date})"
+        if fact_date is not None:
+            readable_date = format_date_fn(fact_date)
+            # Augment text with date for embedding (but store original text in DB)
+            augmented_text = f"{fact.fact_text} (happened in {readable_date})"
+        else:
+            augmented_text = fact.fact_text
         augmented_texts.append(augmented_text)
     return augmented_texts
 

hindsight-api/hindsight_api/engine/retain/fact_extraction.py

@@ -19,13 +19,16 @@
 from ..response_models import TokenUsage
 
 
-def _infer_temporal_date(fact_text: str, event_date: datetime) -> str | None:
+def _infer_temporal_date(fact_text: str, event_date: datetime | None) -> str | None:
     """
     Infer a temporal date from fact text when LLM didn't provide occurred_start.
 
     This is a fallback for when the LLM fails to extract temporal information
     from relative time expressions like "last night", "yesterday", etc.
     """
+    if event_date is None:
+        return None
+
     fact_lower = fact_text.lower()
 
     # Map relative time expressions to day offsets
@@ -100,7 +103,6 @@ class Fact(BaseModel):
     # Optional temporal fields
     occurred_start: str | None = None
     occurred_end: str | None = None
-    mentioned_at: str | None = None
 
     # Optional location field
     where: str | None = Field(
@@ -747,7 +749,7 @@ def _build_user_message(
     chunk: str,
     chunk_index: int,
     total_chunks: int,
-    event_date: datetime,
+    event_date: datetime | None,
     context: str,
     metadata: dict[str, str] | None = None,
 ) -> str:
@@ -756,8 +758,12 @@ def _build_user_message(
 
     sanitized_chunk = _sanitize_text(chunk)
     sanitized_context = _sanitize_text(context) if context else "none"
-    event_date = parse_datetime_flexible(event_date)
-    event_date_formatted = event_date.strftime("%A, %B %d, %Y")
+
+    if event_date is not None:
+        event_date = parse_datetime_flexible(event_date)
+        event_date_str = f"{event_date.strftime('%A, %B %d, %Y')} ({event_date.isoformat()})"
+    else:
+        event_date_str = "Unknown"
 
     metadata_section = ""
     if metadata:
@@ -767,7 +773,7 @@ def _build_user_message(
     return f"""Extract facts from the following text chunk.
 
 Chunk: {chunk_index + 1}/{total_chunks}
-Event Date: {event_date_formatted} ({event_date.isoformat()})
+Event Date: {event_date_str}
 Context: {sanitized_context}{metadata_section}
 
 Text:
@@ -805,7 +811,7 @@ async def _extract_facts_from_chunk(
     chunk: str,
     chunk_index: int,
     total_chunks: int,
-    event_date: datetime,
+    event_date: datetime | None,
     context: str,
     llm_config: "LLMConfig",
     config,
@@ -1043,8 +1049,9 @@ def get_value(field_name):
                     if validated_relations:
                         fact_data["causal_relations"] = validated_relations
 
-                # Always set mentioned_at to the event_date (when the conversation/document occurred)
-                fact_data["mentioned_at"] = event_date.isoformat()
+                # Set mentioned_at to the event_date (when the conversation/document occurred),
+                # or None when the caller opted into no timestamp.
+                fact_data["mentioned_at"] = event_date.isoformat() if event_date is not None else None
 
                 # Build Fact model instance
                 try:
@@ -1107,7 +1114,7 @@ async def _extract_facts_with_auto_split(
     chunk: str,
     chunk_index: int,
     total_chunks: int,
-    event_date: datetime,
+    event_date: datetime | None,
     context: str,
     llm_config: LLMConfig,
     config,
@@ -1226,7 +1233,7 @@ async def _extract_facts_with_auto_split(
 
 async def extract_facts_from_text(
     text: str,
-    event_date: datetime,
+    event_date: datetime | None,
     llm_config: LLMConfig,
     agent_name: str,
     config,
@@ -1641,8 +1648,9 @@ def get_value(field_name):
                 if validated_relations:
                     fact_data["causal_relations"] = validated_relations
 
-            # Always set mentioned_at
-            fact_data["mentioned_at"] = event_date.isoformat()
+            # Set mentioned_at to the event_date (when the conversation/document occurred),
+            # or None when the caller opted into no timestamp.
+            fact_data["mentioned_at"] = event_date.isoformat() if event_date is not None else None
 
             try:
                 fact = Fact(fact=combined_text, fact_type=fact_type, **fact_data)

hindsight-api/hindsight_api/engine/retain/link_utils.py

@@ -47,6 +47,9 @@ def compute_temporal_links(
 
     links = []
     for unit_id, unit_event_date in new_units.items():
+        # Units without event_date can't form temporal links
+        if unit_event_date is None:
+            continue
         # Normalize unit_event_date for consistent comparison
         unit_event_date_norm = _normalize_datetime(unit_event_date)
 
@@ -96,7 +99,11 @@ def compute_temporal_query_bounds(
         return None, None
 
     # Normalize all dates to be timezone-aware to avoid comparison issues
-    all_dates = [_normalize_datetime(d) for d in new_units.values()]
+    # Filter out None values — units without event_date can't form temporal links
+    all_dates = [_normalize_datetime(d) for d in new_units.values() if d is not None]
+
+    if not all_dates:
+        return None, None
 
     try:
         min_date = min(all_dates) - timedelta(hours=time_window_hours)
@@ -432,20 +439,23 @@ async def create_temporal_links_batch_per_fact(
         min_date, max_date = compute_temporal_query_bounds(new_units, time_window_hours)
 
         fetch_neighbors_start = time_mod.time()
-        all_candidates = await conn.fetch(
-            f"""
-            SELECT id, event_date
-            FROM {fq_table("memory_units")}
-            WHERE bank_id = $1
-              AND event_date BETWEEN $2 AND $3
-              AND id::text != ALL($4)
-            ORDER BY event_date DESC
-            """,
-            bank_id,
-            min_date,
-            max_date,
-            unit_ids,
-        )
+        if min_date is not None and max_date is not None:
+            all_candidates = await conn.fetch(
+                f"""
+                SELECT id, event_date
+                FROM {fq_table("memory_units")}
+                WHERE bank_id = $1
+                  AND event_date BETWEEN $2 AND $3
+                  AND id::text != ALL($4)
+                ORDER BY event_date DESC
+                """,
+                bank_id,
+                min_date,
+                max_date,
+                unit_ids,
+            )
+        else:
+            all_candidates = []
         _log(
             log_buffer,
             f"      [7.2] Fetch {len(all_candidates)} candidate neighbors (1 query): {time_mod.time() - fetch_neighbors_start:.3f}s",
@@ -460,11 +470,15 @@ async def create_temporal_links_batch_per_fact(
             # Convert new_units dict to candidate format for within-batch linking
             new_unit_items = list(new_units.items())
             for i, (unit_id, event_date) in enumerate(new_unit_items):
+                if event_date is None:
+                    continue  # Skip units without event_date for temporal linking
                 unit_event_date_norm = _normalize_datetime(event_date)
 
                 # Compare with other new units (only those after this one to avoid duplicates)
                 for j in range(i + 1, len(new_unit_items)):
                     other_id, other_event_date = new_unit_items[j]
+                    if other_event_date is None:
+                        continue  # Skip units without event_date
                     other_event_date_norm = _normalize_datetime(other_event_date)
 
                     # Check if within time window

hindsight-api/hindsight_api/engine/retain/orchestrator.py

@@ -128,12 +128,14 @@ async def retain_batch(
         item_tags = item.get("tags", []) or []
         merged_tags = list(set(item_tags + (document_tags or [])))
 
-        # Handle event_date: parse flexibly (handles both datetime objects and ISO strings)
-        event_date_value = item.get("event_date")
-        if event_date_value:
-            event_date_value = parse_datetime_flexible(event_date_value)
+        # Handle event_date: distinguish "not provided" (default to now) from
+        # "explicitly None" (caller opted into no timestamp).
+        if "event_date" in item and item["event_date"] is None:
+            event_date_value = None  # Caller explicitly signalled "unknown date"
+        elif item.get("event_date"):
+            event_date_value = parse_datetime_flexible(item["event_date"])
         else:
-            event_date_value = utcnow()
+            event_date_value = utcnow()  # Backward-compatible default
 
         content = RetainContent(
             content=item["content"],

hindsight-api/hindsight_api/engine/retain/types.py

@@ -6,7 +6,7 @@
 """
 
 from dataclasses import dataclass, field
-from datetime import UTC, datetime
+from datetime import datetime
 from typing import Literal, TypedDict
 from uuid import UUID
 
@@ -29,7 +29,7 @@ class RetainContentDict(TypedDict, total=False):
 
     content: str  # Required
     context: str
-    event_date: datetime
+    event_date: datetime | None
     metadata: dict[str, str]
     document_id: str
     entities: list[dict[str, str]]  # [{"text": "...", "type": "..."}]
@@ -39,11 +39,6 @@ class RetainContentDict(TypedDict, total=False):
     )  # Observation scopes for consolidation
 
 
-def _now_utc() -> datetime:
-    """Factory function for default event_date."""
-    return datetime.now(UTC)
-
-
 @dataclass
 class RetainContent:
     """
@@ -54,7 +49,7 @@ class RetainContent:
 
     content: str
     context: str = ""
-    event_date: datetime = field(default_factory=_now_utc)
+    event_date: datetime | None = None
     metadata: dict[str, str] = field(default_factory=dict)
     entities: list[dict[str, str]] = field(default_factory=list)  # User-provided entities
     tags: list[str] = field(default_factory=list)  # Visibility scope tags
@@ -147,7 +142,7 @@ class ProcessedFact:
     # Temporal data
     occurred_start: datetime | None
     occurred_end: datetime | None
-    mentioned_at: datetime
+    mentioned_at: datetime | None
 
     # Context and metadata
     context: str
@@ -200,12 +195,10 @@ def from_extracted_fact(
         Returns:
             ProcessedFact ready for storage
         """
-        from datetime import datetime
-
         # Use occurred dates only if explicitly provided by LLM
         occurred_start = extracted_fact.occurred_start
         occurred_end = extracted_fact.occurred_end
-        mentioned_at = extracted_fact.mentioned_at or datetime.now(UTC)
+        mentioned_at = extracted_fact.mentioned_at  # May be None when caller opted into no timestamp
 
         # Convert entity strings to EntityRef objects
         entities = [EntityRef(name=name) for name in extracted_fact.entities]