feat: add support for system indexes [ECS-1791] (#818)

Author: gcuip

pyproject.toml

@@ -7,7 +7,7 @@ requires-python = ">=3.11"
 dependencies = [
     "uipath>=2.10.57, <2.11.0",
     "uipath-core>=0.5.13, <0.6.0",
-    "uipath-platform>=0.1.36, <0.2.0",
+    "uipath-platform>=0.1.43, <0.2.0",
     "uipath-runtime>=0.10.0, <0.11.0",
     "langgraph>=1.1.8, <2.0.0",
     "langchain-core>=1.2.11, <2.0.0",

src/uipath_langchain/agent/tools/context_tool.py

@@ -248,13 +248,16 @@ async def context_tool_fn(
             static_folder_path_prefix or _resolved_arg_folder_prefix
         )
 
+        debug_run = UiPathConfig.is_studio_project
+
         retriever = ContextGroundingRetriever(
             index_name=resource.index_name,
             folder_path=get_execution_folder_path(),
             number_of_results=result_count,
             threshold=threshold,
             scope_folder=resolved_folder_path_prefix,
             scope_extension=file_extension,
+            include_system_indexes=debug_run,
         )
 
         actual_query = prompt or query

src/uipath_langchain/retrievers/context_grounding_retriever.py

@@ -1,3 +1,5 @@
+import logging
+
 from langchain_core.callbacks import (
     AsyncCallbackManagerForRetrieverRun,
     CallbackManagerForRetrieverRun,
@@ -7,6 +9,8 @@
 from uipath.platform import UiPath
 from uipath.platform.context_grounding import SearchMode, UnifiedSearchScope
 
+logger = logging.getLogger(__name__)
+
 
 class ContextGroundingRetriever(BaseRetriever):
     index_name: str
@@ -17,6 +21,7 @@ class ContextGroundingRetriever(BaseRetriever):
     threshold: float = 0.0
     scope_folder: str | None = None
     scope_extension: str | None = None
+    include_system_indexes: bool = False
 
     def _build_scope(self) -> UnifiedSearchScope | None:
         if self.scope_folder or self.scope_extension:
@@ -32,6 +37,11 @@ def _get_relevant_documents(
         """Sync implementation calls context_grounding unified_search API."""
 
         sdk = self.uipath_sdk if self.uipath_sdk is not None else UiPath()
+        if self.include_system_indexes:
+            logger.debug(
+                "Searching index '%s' with system-index fallback enabled",
+                self.index_name,
+            )
         result = sdk.context_grounding.unified_search(
             self.index_name,
             query,
@@ -43,6 +53,7 @@ def _get_relevant_documents(
             scope=self._build_scope(),
             folder_path=self.folder_path,
             folder_key=self.folder_key,
+            include_system_indexes=self.include_system_indexes,
         )
 
         values = result.semantic_results.values if result.semantic_results else []
@@ -72,6 +83,11 @@ async def _aget_relevant_documents(
         """Async implementation calls context_grounding unified_search_async API."""
 
         sdk = self.uipath_sdk if self.uipath_sdk is not None else UiPath()
+        if self.include_system_indexes:
+            logger.debug(
+                "Searching index '%s' with system-index fallback enabled",
+                self.index_name,
+            )
         result = await sdk.context_grounding.unified_search_async(
             self.index_name,
             query,
@@ -83,6 +99,7 @@ async def _aget_relevant_documents(
             scope=self._build_scope(),
             folder_path=self.folder_path,
             folder_key=self.folder_key,
+            include_system_indexes=self.include_system_indexes,
         )
 
         values = result.semantic_results.values if result.semantic_results else []

tests/agent/tools/test_context_tool.py

@@ -525,6 +525,48 @@ async def test_semantic_search_uses_execution_folder_path(self, semantic_config)
             call_kwargs = mock_retriever_class.call_args[1]
             assert call_kwargs["folder_path"] == "/Shared/TestFolder"
 
+    @pytest.mark.asyncio
+    async def test_semantic_search_enables_system_index_fallback_in_studio_project(
+        self,
+        semantic_config,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        monkeypatch.setenv("UIPATH_PROJECT_ID", "some-project-id")
+        with patch(
+            "uipath_langchain.agent.tools.context_tool.ContextGroundingRetriever"
+        ) as mock_retriever_class:
+            mock_retriever = AsyncMock()
+            mock_retriever.ainvoke.return_value = []
+            mock_retriever_class.return_value = mock_retriever
+
+            tool = handle_semantic_search("semantic_tool", semantic_config)
+            assert tool.coroutine is not None
+            await tool.coroutine(query="test query")
+
+            call_kwargs = mock_retriever_class.call_args.kwargs
+            assert call_kwargs["include_system_indexes"] is True
+
+    @pytest.mark.asyncio
+    async def test_semantic_search_disables_system_index_fallback_outside_studio_project(
+        self,
+        semantic_config,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        monkeypatch.delenv("UIPATH_PROJECT_ID", raising=False)
+        with patch(
+            "uipath_langchain.agent.tools.context_tool.ContextGroundingRetriever"
+        ) as mock_retriever_class:
+            mock_retriever = AsyncMock()
+            mock_retriever.ainvoke.return_value = []
+            mock_retriever_class.return_value = mock_retriever
+
+            tool = handle_semantic_search("semantic_tool", semantic_config)
+            assert tool.coroutine is not None
+            await tool.coroutine(query="test query")
+
+            call_kwargs = mock_retriever_class.call_args.kwargs
+            assert call_kwargs["include_system_indexes"] is False
+
 
 class TestHandleBatchTransform:
     """Test cases for handle_batch_transform function."""
@@ -1095,3 +1137,87 @@ async def test_non_400_enriched_exception_propagates(
 
             with pytest.raises(EnrichedException):
                 await tool.coroutine(query="test query")
+
+
+class TestSemanticSearchSystemIndexFallbackIntegration:
+    """End-to-end mocked test that exercises the full SDK chain.
+
+    Verifies that when running as a Studio project (debug run), the agent's
+    semantic-search tool resolves the index via the system-indexes
+    endpoint after the across-folders listing returns empty, and
+    successfully runs the unified search against the resolved id.
+    """
+
+    @pytest.mark.asyncio
+    async def test_resolves_system_index_and_runs_unified_search(
+        self,
+        httpx_mock,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        monkeypatch.setenv("UIPATH_URL", "https://cloud.uipath.com/org/tenant")
+        monkeypatch.setenv("UIPATH_ACCESS_TOKEN", "test-token")
+        monkeypatch.setenv("UIPATH_ORGANIZATION_ID", "org-id")
+        monkeypatch.setenv("UIPATH_TENANT_ID", "tenant-id")
+        monkeypatch.setenv("UIPATH_TRACING_ENABLED", "False")
+        monkeypatch.setenv("UIPATH_PROJECT_ID", "studio-project-id")
+        monkeypatch.delenv("UIPATH_FOLDER_PATH", raising=False)
+        monkeypatch.delenv("UIPATH_FOLDER_KEY", raising=False)
+
+        base = "https://cloud.uipath.com/org/tenant"
+        httpx_mock.add_response(
+            url=f"{base}/ecs_/v2/indexes/allacrossfolders?$expand=dataSource&$filter=Name eq 'system-template-index'",
+            status_code=200,
+            json={"value": []},
+        )
+        httpx_mock.add_response(
+            url=f"{base}/ecs_/v2/indexes/allsystemindexes?$expand=dataSource&$filter=Name eq 'system-template-index'",
+            status_code=200,
+            json={
+                "value": [
+                    {
+                        "id": "sys-1",
+                        "name": "system-template-index",
+                        "lastIngestionStatus": "Completed",
+                    }
+                ]
+            },
+        )
+        httpx_mock.add_response(
+            url=f"{base}/ecs_/v1.2/search/sys-1",
+            status_code=200,
+            json={
+                "semanticResults": {
+                    "metadata": {"operation_id": "op-1", "strategy": "semantic"},
+                    "values": [
+                        {
+                            "id": "doc-1",
+                            "source": "src",
+                            "page_number": 1,
+                            "content": "hello world",
+                            "score": 0.9,
+                        }
+                    ],
+                }
+            },
+        )
+
+        resource = _make_context_resource(
+            name="semantic_tool",
+            description="Semantic search tool",
+            index_name="system-template-index",
+            retrieval_mode=AgentContextRetrievalMode.SEMANTIC,
+            query_variant="dynamic",
+        )
+
+        tool = handle_semantic_search("semantic_tool", resource)
+        assert tool.coroutine is not None
+        result = await tool.coroutine(query="hi")
+
+        assert "documents" in result
+        assert len(result["documents"]) == 1
+        assert result["documents"][0]["page_content"] == "hello world"
+
+        urls = [str(r.url) for r in httpx_mock.get_requests()]
+        assert any("/v2/indexes/allacrossfolders" in u for u in urls)
+        assert any("/v2/indexes/allsystemindexes" in u for u in urls)
+        assert any("/v1.2/search/sys-1" in u for u in urls)

tests/retrievers/test_context_grounding_retriever.py

@@ -0,0 +1,79 @@
+"""Tests for ContextGroundingRetriever's include_system_indexes plumbing."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from uipath.platform import UiPath
+
+from uipath_langchain.retrievers import ContextGroundingRetriever
+
+
+def _make_unified_search_result() -> MagicMock:
+    result = MagicMock()
+    result.semantic_results.values = []
+    result.semantic_results.metadata.operation_id = "op-1"
+    return result
+
+
+def _make_sdk_mock() -> MagicMock:
+    return MagicMock(spec=UiPath)
+
+
+def test_retriever_forwards_include_system_indexes_when_true() -> None:
+    sdk = _make_sdk_mock()
+    sdk.context_grounding.unified_search.return_value = _make_unified_search_result()
+
+    retriever = ContextGroundingRetriever(
+        index_name="my-index",
+        uipath_sdk=sdk,
+        include_system_indexes=True,
+    )
+    retriever.invoke("hello")
+
+    sdk.context_grounding.unified_search.assert_called_once()
+    kwargs = sdk.context_grounding.unified_search.call_args.kwargs
+    assert kwargs["include_system_indexes"] is True
+
+
+def test_retriever_defaults_include_system_indexes_to_false() -> None:
+    sdk = _make_sdk_mock()
+    sdk.context_grounding.unified_search.return_value = _make_unified_search_result()
+
+    retriever = ContextGroundingRetriever(index_name="my-index", uipath_sdk=sdk)
+    retriever.invoke("hello")
+
+    kwargs = sdk.context_grounding.unified_search.call_args.kwargs
+    assert kwargs["include_system_indexes"] is False
+
+
+@pytest.mark.asyncio
+async def test_retriever_async_forwards_include_system_indexes_when_true() -> None:
+    sdk = _make_sdk_mock()
+    sdk.context_grounding.unified_search_async = AsyncMock(
+        return_value=_make_unified_search_result()
+    )
+
+    retriever = ContextGroundingRetriever(
+        index_name="my-index",
+        uipath_sdk=sdk,
+        include_system_indexes=True,
+    )
+    await retriever.ainvoke("hello")
+
+    sdk.context_grounding.unified_search_async.assert_awaited_once()
+    kwargs = sdk.context_grounding.unified_search_async.call_args.kwargs
+    assert kwargs["include_system_indexes"] is True
+
+
+@pytest.mark.asyncio
+async def test_retriever_async_defaults_include_system_indexes_to_false() -> None:
+    sdk = _make_sdk_mock()
+    sdk.context_grounding.unified_search_async = AsyncMock(
+        return_value=_make_unified_search_result()
+    )
+
+    retriever = ContextGroundingRetriever(index_name="my-index", uipath_sdk=sdk)
+    await retriever.ainvoke("hello")
+
+    kwargs = sdk.context_grounding.unified_search_async.call_args.kwargs
+    assert kwargs["include_system_indexes"] is False