adding roundtrip tests to assert documents metadata is correctly written and retrieved

Author: davidsbatista

integrations/opensearch/src/haystack_integrations/document_stores/opensearch/document_store.py

@@ -352,29 +352,10 @@ async def count_documents_async(self) -> int:
     @staticmethod
     def _deserialize_search_hits(hits: list[dict[str, Any]]) -> list[Document]:
         out = []
-
         for hit in hits:
-            data = hit["_source"].copy()
-
-            # Reconstruct metadata dict from flattened fields
-            meta = {}
-            fields_to_remove = []
-            for key, value in data.items():
-                if key not in SPECIAL_FIELDS:
-                    meta[key] = value
-                    fields_to_remove.append(key)
-
-            # Remove metadata fields from top level and add them to meta
-            for key in fields_to_remove:
-                data.pop(key, None)
-
-            if meta:
-                data["meta"] = meta
-
+            data = hit["_source"]
             if "highlight" in hit:
-                if "meta" not in data:
-                    data["meta"] = {}
-                data["meta"]["highlighted"] = hit["highlight"]
+                data["metadata"]["highlighted"] = hit["highlight"]
             data["score"] = hit["_score"]
             out.append(Document.from_dict(data))
 

integrations/opensearch/tests/test_bm25_retriever.py

@@ -424,6 +424,60 @@ def test_bm25_retriever_runtime_document_store_switching(
     assert len(results_1_again["documents"]) == 1
 
 
+@pytest.mark.integration
+def test_bm25_retriever_document_structure_with_metadata(document_store):
+    """
+    Test document structure with complex metadata (nested values, lists, etc.)
+    """
+    docs = [
+        Document(
+            content="Python is versatile",
+            meta={
+                "category": "programming",
+                "tags": ["python", "general-purpose"],
+                "rating": 4.5,
+                "active": True,
+                "author": {"name": "John", "role": "developer"},
+            },
+            id="python_doc",
+        ),
+        Document(
+            content="JavaScript is dynamic",
+            meta={
+                "category": "programming",
+                "tags": ["javascript", "web"],
+                "rating": 4.8,
+                "active": True,
+            },
+            id="js_doc",
+        ),
+    ]
+    document_store.write_documents(docs, refresh=True)
+    retriever = OpenSearchBM25Retriever(document_store=document_store)
+
+    results = retriever.run(query="programming", top_k=2)
+    assert len(results["documents"]) == 2
+
+    for doc in results["documents"]:
+        # Verify structure
+        assert hasattr(doc, "content")
+        assert hasattr(doc, "meta")
+        assert isinstance(doc.meta, dict)
+
+        # Verify complex metadata is preserved
+        assert "category" in doc.meta
+        assert "tags" in doc.meta
+        assert isinstance(doc.meta["tags"], list)
+        assert "rating" in doc.meta
+
+        # Verify document can be serialized/deserialized
+        doc_dict = doc.to_dict()
+        doc_from_dict = Document.from_dict(doc_dict)
+        assert doc_from_dict.content == doc.content
+        assert doc_from_dict.meta == doc.meta
+        assert doc_from_dict.id == doc.id
+
+
 @pytest.mark.asyncio
 @pytest.mark.integration
 async def test_bm25_retriever_async_runtime_document_store_switching(

integrations/opensearch/tests/test_document_store.py

@@ -615,7 +615,6 @@ def test_update_by_filter(self, document_store: OpenSearchDocumentStore):
         assert len(draft_docs) == 1
         assert draft_docs[0].meta["category"] == "B"
 
-
     def test_count_documents_by_filter(self, document_store: OpenSearchDocumentStore):
         docs = [
             Document(content="Doc 1", meta={"category": "A", "status": "active"}),
@@ -914,4 +913,3 @@ def test_delete_with_routing(self, document_store: OpenSearchDocumentStore):
         document_store.delete_documents(["1", "2"], routing=routing_map)
 
         assert document_store.count_documents() == 1
-

integrations/opensearch/tests/test_embedding_retriever.py

@@ -404,3 +404,51 @@ async def test_embedding_retriever_runtime_document_store_switching_async(
     python_query_embedding = [0.4, 0.4, 0.4] + [0.0] * 765
     results_1_again = await retriever.run_async(query_embedding=python_query_embedding)
     assert "Python" in results_1_again["documents"][0].content
+
+
+@pytest.mark.integration
+def test_embedding_retriever_document_structure_with_metadata(document_store, test_documents_with_embeddings_1):
+    """
+    Test that documents returned by embedding retriever have correct structure:
+    - Metadata fields are in doc.meta (not at top level)
+    - Special fields (content, embedding, id, score) are at top level
+    - All original metadata is preserved
+    """
+    document_store.write_documents(test_documents_with_embeddings_1, refresh=True)
+    retriever = OpenSearchEmbeddingRetriever(document_store=document_store)
+
+    # Query embedding to match functional programming languages
+    query_embedding = [0.2, 0.3, 0.4] + [0.0] * 765
+    results = retriever.run(query_embedding=query_embedding, top_k=5)
+
+    assert len(results["documents"]) > 0
+
+    for doc in results["documents"]:
+        # Verify special fields are at top level
+        assert hasattr(doc, "content")
+        assert isinstance(doc.content, str)
+        assert hasattr(doc, "id")
+        assert isinstance(doc.id, str)
+        assert hasattr(doc, "score")
+        assert doc.score is not None
+        assert hasattr(doc, "embedding")
+        assert isinstance(doc.embedding, list)
+        assert len(doc.embedding) == 768
+
+        # Verify metadata fields are in meta dict (not at top level)
+        assert hasattr(doc, "meta")
+        assert isinstance(doc.meta, dict)
+
+        # Verify original metadata is preserved
+        assert "likes" in doc.meta
+        assert "language_type" in doc.meta
+        assert isinstance(doc.meta["likes"], int)
+        assert isinstance(doc.meta["language_type"], str)
+
+        # Verify document can be serialized/deserialized
+        doc_dict = doc.to_dict()
+        doc_from_dict = Document.from_dict(doc_dict)
+        assert doc_from_dict.content == doc.content
+        assert doc_from_dict.meta == doc.meta
+        assert doc_from_dict.id == doc.id
+        assert doc_from_dict.embedding == doc.embedding