fix(ingest): scope document uniqueness to the ingestion space so each config gets its own chunks

Author: codebybilal18

app/db/migrations/versions/0002_document_unique_per_space.py

@@ -0,0 +1,43 @@
+"""scope document uniqueness to the ingestion space
+
+A document was unique per (tenant_id, sha256), so re-uploading the same PDF
+under a different ingestion space returned the existing row and skipped
+ingestion entirely. Config A's space therefore stayed empty while config B's
+filled — and the A-vs-B comparison the benchmark exists to make was impossible.
+
+Chunks were already keyed per space. This brings documents in line: the same
+file ingested under two chunking strategies is two ingestion records, because
+it produces two independent sets of chunks and embeddings.
+
+The cost is a duplicated blob per space. At demo scale that is a few megabytes,
+and the alternative — tracking ingestion state per (document, space) in a
+separate table — is the right answer only once storage actually matters.
+
+Revision ID: 0002
+Revises: 0001
+Create Date: 2026-07-26
+"""
+
+from __future__ import annotations
+
+from alembic import op
+
+revision = "0002"
+down_revision = "0001"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.drop_constraint("uq_documents_tenant_sha", "documents", type_="unique")
+    op.create_unique_constraint(
+        "uq_documents_tenant_sha_space", "documents", ["tenant_id", "sha256", "space"]
+    )
+
+
+def downgrade() -> None:
+    # Collapsing back to one row per file requires deleting every duplicate
+    # first; doing that silently would destroy ingested chunks, so this is left
+    # to a deliberate manual step.
+    op.drop_constraint("uq_documents_tenant_sha_space", "documents", type_="unique")
+    op.create_unique_constraint("uq_documents_tenant_sha", "documents", ["tenant_id", "sha256"])

app/db/models.py

@@ -120,7 +120,13 @@ class Document(Base):
     pages = relationship("Page", back_populates="document", cascade="all, delete-orphan")
     chunks = relationship("Chunk", back_populates="document", cascade="all, delete-orphan")
 
-    __table_args__ = (UniqueConstraint("tenant_id", "sha256", name="uq_documents_tenant_sha"),)
+    # Scoped to `space`: the same file ingested under two chunking strategies is
+    # two ingestion records, because it yields two independent sets of chunks
+    # and embeddings. Keying on the hash alone made the second ingest a no-op
+    # and left config A's space empty.
+    __table_args__ = (
+        UniqueConstraint("tenant_id", "sha256", "space", name="uq_documents_tenant_sha_space"),
+    )
 
 
 class DocumentBlob(Base):

app/ingest/pipeline.py

@@ -115,8 +115,17 @@ async def create_document(
     paying to parse and embed the same corpus twice.
     """
     digest = content_sha256(data)
+    space = space_for(config)
+
+    # Scoped to the space. Matching on the hash alone would return the copy
+    # ingested under a different chunking strategy and skip the work, leaving
+    # this space with no chunks at all.
     existing = await session.scalar(
-        select(Document).where(Document.tenant_id == tenant_id, Document.sha256 == digest)
+        select(Document).where(
+            Document.tenant_id == tenant_id,
+            Document.sha256 == digest,
+            Document.space == space,
+        )
     )
     if existing is not None:
         return existing, False
@@ -128,7 +137,7 @@ async def create_document(
         size_bytes=len(data),
         sha256=digest,
         status=DocumentStatus.UPLOADED,
-        space=space_for(config),
+        space=space,
         meta={"chunk_strategy": config.chunk_strategy.value, "config": config.name},
     )
     session.add(doc)

tests/test_schema_contract.py

@@ -76,3 +76,24 @@ def test_enum_members_serialise_as_their_lowercase_value():
     assert DocumentStatus.READY == "ready"
     assert DocumentStatus.READY.value == "ready"
     assert ExtractionMethod.VISION == "vision"
+
+
+def test_documents_are_unique_per_ingestion_space():
+    """Keyed on (tenant, sha256) alone, re-ingesting the same PDF under a
+    different chunking strategy returned the existing row and did nothing — so
+    config A's space stayed empty and the A-vs-B comparison, which is the whole
+    point of the benchmark, could not be made.
+
+    Chunks were already per-space; documents have to agree.
+    """
+    constraints = {
+        c.name: sorted(col.name for col in c.columns)
+        for c in Document.__table__.constraints
+        if c.__class__.__name__ == "UniqueConstraint"
+    }
+    assert "uq_documents_tenant_sha_space" in constraints, constraints
+    assert constraints["uq_documents_tenant_sha_space"] == ["sha256", "space", "tenant_id"]
+
+    migrations = sorted(MIGRATION.parent.glob("*.py"))
+    source = "\n".join(p.read_text() for p in migrations)
+    assert "uq_documents_tenant_sha_space" in source, "the ORM constraint has no matching migration"