Add test for checking if tags exist

Author: Ickerday

.github/workflows/test.yml

@@ -46,10 +46,12 @@ jobs:
         uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
           path: .pytest_cache
-          key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{ github.sha }}
+          key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{
+            github.sha }}
           restore-keys: |
             pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-
       - name: Run unit tests
         run: make test-unit
       - name: Run entire test suite
-        run: make test-integration
+      #   run: make test-integration
+        run: uv run pytest ./tests/system/test_ai_agentic_test_app.py::TestAgenticApp

tests/system/__init__.py

@@ -107,6 +107,41 @@ def test_agentic_app_with_remote_tools(self) -> None:
         app.delete()
         self.restart_splunk()  # app removal requires a restart
 
+    # To execute this test locally, download the Splunk MCP Server App tarball from
+    # https://splunkbase.splunk.com/app/7931 and place it in a file named
+    # splunk-mcp-server.tgz at the root of this repo (i.e. ../../splunk-mcp-server.tgz).
+    #
+    # Note: that the downloaded file could have a: .spl, .tar, .tar.gz or .tgz extension,
+    # if it is not .tgz, then you must change it to .tgz.
+    #
+    # Our CI does this automatically.
+    def test_remote_tools_have_tags(self) -> None:
+        pytest.importorskip("langchain_openai")
+        self.requires_splunk_10_2()
+
+        # Skip test in case the instance does not have a /splunk-mcp-server.tgz file.
+        try:
+            resp = self.service.get("agentic_app/has_mcp_app_file")
+            assert resp.status == 200
+        except HTTPError as e:
+            if e.status == 404:
+                pytest.skip("Splunk MCP Server App file not found on Splunk instance")
+            raise
+
+        app = self.service.apps.create(name="/splunk-mcp-server.tgz", filename=True)  # pyright: ignore[reportUnknownVariableType]
+
+        resp = self.service.post(
+            "agentic_app/tool-tags",
+            body=self.test_llm_settings.model_dump_json(),
+        )
+
+        assert resp.status == 200
+        body = str(resp.body)  # pyright: ignore[reportUnknownArgumentType]
+        assert "tags" in body
+
+        app.delete()
+        self.restart_splunk()  # app removal requires a restart
+
     def requires_splunk_10_2(self) -> None:
         if self.service.splunk_version[0] < 10 or self.service.splunk_version[1] < 2:
             pytest.skip("Python 3.13 not available on splunk < 10.2")

tests/system/test_ai_agentic_test_app.py

@@ -12,13 +12,14 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import json
 import os
 import sys
 
 sys.path.insert(0, "/splunklib-deps")
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib"))
 
-from typing import override
+from typing import final, override
 
 from pydantic import BaseModel, Field
 
@@ -72,3 +73,29 @@ class Output(BaseModel):
             )
 
             self.response.write(result.structured_output.model_dump_json())
+
+
+@final
+class ToolTagsHandler(CRETestHandler):
+    @override
+    async def run(self) -> None:
+        async with Agent(
+            model=await self.model(),
+            system_prompt="You are a helpful Splunk assistant",
+            tool_settings=ToolSettings(
+                local=False,
+                remote=RemoteToolSettings(
+                    allowlist=ToolAllowlist(names=["splunk_get_indexes"])
+                ),
+            ),
+            service=self.service,
+        ) as agent:
+            assert len(agent.tools) > 0, "No remote tools loaded"
+
+            tools_with_tags = [t for t in agent.tools if len(t.tags) > 0]
+            assert len(tools_with_tags) > 0, (
+                f"No tools have tags. Tools: {[t.name for t in agent.tools]}"
+            )
+
+            result = {"tools": [{"name": t.name, "tags": t.tags} for t in agent.tools]}
+            self.response.write(json.dumps(result))

tests/system/test_apps/ai_agentic_test_app/bin/indexes.py

@@ -15,3 +15,9 @@ match           = /agentic_app/has_mcp_app_file
 scripttype      = python
 handler         = mcp_app_file_exists.Handler
 python.required = 3.13
+
+[script:tool_tags]
+match           = /agentic_app/tool-tags
+scripttype      = python
+handler         = indexes.ToolTagsHandler
+python.required = 3.13