Hashed blob names

Author: andystaples

CHANGELOG.md

@@ -20,10 +20,15 @@ ADDED
   (tail terminal instances indefinitely until stopped via `delete_job`).
   Exported blobs are self-describing: each blob carries an explicit
   `schema_version`, the orchestration's `OrchestrationState` metadata, and
-  the full ordered event list. Each exported blob also carries
-  `{"instance_id": <id>}` as destination-side metadata (the Azure writer
-  persists this as Azure Blob metadata) so consumers can scan a container
-  without parsing each blob body. The export workflow retries each instance up
+  the full ordered event list. Blob names are a lowercase-hex SHA-256 of
+  ``{last_updated_at}|{instance_id}`` with the format extension appended
+  (matches the .NET `ExportInstanceHistoryActivity` naming scheme), so
+  re-exporting an instance after a later terminal update lands at a new
+  blob path rather than overwriting the previous one, and instance IDs
+  that differ only by `/` no longer collide. Each exported blob also
+  carries `{"instance_id": <id>}` as destination-side metadata (the Azure
+  writer persists this as Azure Blob metadata) so consumers can scan a
+  container without parsing each blob body. The export workflow retries each instance up
   to 3 times with exponential backoff (15s/30s/60s), retries failed batches
   up to 3 times, caps in-flight exports via `max_parallel_exports`
   (default 32), continues-as-new every 5 page cycles to bound orchestrator

durabletask/extensions/history_export/activities.py

@@ -24,9 +24,10 @@
 
 from __future__ import annotations
 
+import hashlib
 from collections.abc import Mapping
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, cast
 
 from durabletask import client as client_module
@@ -222,7 +223,21 @@ def export_instance_history(
             fmt=fmt,
             metadata=metadata,
         )
-        blob_name = _blob_name_for(instance_id=instance_id, prefix=prefix, fmt=fmt)
+        # Blob name is a SHA-256 hash of the instance's terminal
+        # timestamp + instance ID (matches the .NET
+        # ``ExportInstanceHistoryActivity`` scheme).  This means:
+        # • Two exports of the *same* completion produce the same
+        #   blob name (idempotent under retry when ``overwrite=True``).
+        # • An instance re-exported after a later completion lands
+        #   at a new path rather than overwriting the previous one.
+        # • Instance IDs that differ only by ``/`` no longer collide
+        #   under the old ``.replace("/", "_")`` transform.
+        blob_name = _blob_name_for(
+            instance_id=instance_id,
+            last_updated_at=state.last_updated_at,
+            prefix=prefix,
+            fmt=fmt,
+        )
         ctx.writer.write(
             instance_id=instance_id,
             container=container,
@@ -249,12 +264,56 @@ def export_instance_history(
 # Helpers
 # ----------------------------------------------------------------------
 
-def _blob_name_for(*, instance_id: str, prefix: str | None, fmt: ExportFormat) -> str:
+def _blob_name_for(
+    *,
+    instance_id: str,
+    last_updated_at: datetime,
+    prefix: str | None,
+    fmt: ExportFormat,
+) -> str:
+    """Return the destination blob name for one exported instance.
+
+    Matches the .NET ``ExportInstanceHistoryActivity.GenerateBlobFileName``
+    scheme: lowercase-hex SHA-256 of
+    ``f"{last_updated_at:O}|{instance_id}"`` with the format-appropriate
+    extension appended, optionally namespaced under the configured
+    destination prefix.  Hash byte-equivalence with .NET output
+    requires matching the .NET ``DateTimeOffset.ToString("O")`` format
+    exactly (see :func:`_dotnet_o_format`).
+    """
+    timestamp_str = _dotnet_o_format(last_updated_at)
+    hash_input = f"{timestamp_str}|{instance_id}"
+    digest = hashlib.sha256(hash_input.encode("utf-8")).hexdigest()
     ext = file_extension_for(fmt)
-    safe_id = instance_id.replace("/", "_")
+    blob_name = f"{digest}{ext}"
     if prefix:
-        return f"{prefix.rstrip('/')}/{safe_id}{ext}"
-    return f"{safe_id}{ext}"
+        return f"{prefix.rstrip('/')}/{blob_name}"
+    return blob_name
+
+
+def _dotnet_o_format(dt: datetime) -> str:
+    """Format *dt* to match .NET ``DateTimeOffset.ToString("O")``.
+
+    .NET's round-trip format is ``yyyy-MM-ddTHH:mm:ss.fffffffK`` for
+    ``DateTimeOffset``, where ``K`` resolves to ``+HH:MM`` / ``-HH:MM``
+    and fractional seconds always render with seven digits (100-ns
+    ticks resolution).  Python :class:`datetime.datetime` only carries
+    microsecond precision (six digits), so the seventh digit is always
+    a trailing zero.  Naive datetimes are assumed UTC.
+    """
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    base = dt.strftime("%Y-%m-%dT%H:%M:%S")
+    fractional = f"{dt.microsecond:06d}0"
+    offset = dt.utcoffset()
+    if offset is None:
+        offset_str = "+00:00"
+    else:
+        total_minutes = int(offset.total_seconds() // 60)
+        sign = "+" if total_minutes >= 0 else "-"
+        total_minutes = abs(total_minutes)
+        offset_str = f"{sign}{total_minutes // 60:02d}:{total_minutes % 60:02d}"
+    return f"{base}.{fractional}{offset_str}"
 
 
 def register(worker_instance: worker_module.TaskHubGrpcWorker) -> None:

tests/durabletask/extensions/history_export/test_activities.py

@@ -318,5 +318,89 @@ class _State:
     assert result["success"] is False
     assert "no longer terminal" in result["error"]
     assert "RUNNING" in result["error"]
-    assert writer.calls == []
-    assert stub_client.history_calls == 0
+
+
+# ---------------------------------------------------------------------
+# Unit tests for the N-8 blob-naming scheme
+# ---------------------------------------------------------------------
+
+
+def test_blob_name_matches_dotnet_hash_scheme():
+    """N-8: blob name is lowercase-hex sha256 of '{:O}|{instance_id}'.
+
+    Pins the exact .NET-aligned scheme so any future drift (timestamp
+    format, hash function, casing) breaks loudly.  The expected hash
+    is computed by hand from the same inputs the activity would use.
+    """
+    import hashlib
+
+    from durabletask.extensions.history_export.activities import (
+        _blob_name_for,
+        _dotnet_o_format,
+    )
+
+    last_updated = datetime(2026, 6, 4, 17, 9, 9, 420990, tzinfo=timezone.utc)
+    instance_id = "inst-1"
+    fmt = ExportFormat(kind=ExportFormatKind.JSON)
+
+    # The seven-digit fractional-seconds format is what .NET emits for
+    # the same instant.
+    assert _dotnet_o_format(last_updated) == "2026-06-04T17:09:09.4209900+00:00"
+
+    expected_hash = hashlib.sha256(
+        f"{_dotnet_o_format(last_updated)}|{instance_id}".encode("utf-8")
+    ).hexdigest()
+
+    assert _blob_name_for(
+        instance_id=instance_id,
+        last_updated_at=last_updated,
+        prefix=None,
+        fmt=fmt,
+    ) == f"{expected_hash}.json"
+
+    assert _blob_name_for(
+        instance_id=instance_id,
+        last_updated_at=last_updated,
+        prefix="exports/run-1/",
+        fmt=ExportFormat(kind=ExportFormatKind.JSONL_GZIP),
+    ) == f"exports/run-1/{expected_hash}.jsonl.gz"
+
+
+def test_blob_name_isolates_instance_ids_that_differ_only_by_slash():
+    """N-8: instance IDs containing '/' no longer collide.
+
+    The old scheme used ``instance_id.replace(\"/\", \"_\")`` which
+    collapsed ``v1/x`` and ``v1_x`` to the same blob name.  Hashing
+    isolates them.
+    """
+    from durabletask.extensions.history_export.activities import _blob_name_for
+
+    last_updated = datetime(2026, 6, 4, 17, 9, 9, 420990, tzinfo=timezone.utc)
+    fmt = ExportFormat(kind=ExportFormatKind.JSON)
+
+    name_a = _blob_name_for(
+        instance_id="v1/x", last_updated_at=last_updated, prefix=None, fmt=fmt,
+    )
+    name_b = _blob_name_for(
+        instance_id="v1_x", last_updated_at=last_updated, prefix=None, fmt=fmt,
+    )
+    assert name_a != name_b
+
+
+def test_blob_name_changes_when_instance_terminal_timestamp_changes():
+    """N-8: re-export at a different terminal time lands at a new blob."""
+    from durabletask.extensions.history_export.activities import _blob_name_for
+
+    fmt = ExportFormat(kind=ExportFormatKind.JSON)
+    instance_id = "inst-x"
+
+    earlier = datetime(2026, 6, 4, 17, 0, 0, 0, tzinfo=timezone.utc)
+    later = datetime(2026, 6, 4, 18, 0, 0, 0, tzinfo=timezone.utc)
+
+    name_earlier = _blob_name_for(
+        instance_id=instance_id, last_updated_at=earlier, prefix=None, fmt=fmt,
+    )
+    name_later = _blob_name_for(
+        instance_id=instance_id, last_updated_at=later, prefix=None, fmt=fmt,
+    )
+    assert name_earlier != name_later