Test

Author: Ickerday

.github/workflows/test.yml

@@ -46,10 +46,12 @@ jobs:
         uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae
         with:
           path: .pytest_cache
-          key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{ github.sha }}
+          key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{
+            github.sha }}
           restore-keys: |
             pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-
       - name: Run unit tests
         run: make test-unit
       - name: Run entire test suite
-        run: make test-integration
+        #   run: make test-integration
+        run: uv run pytest ./tests/system/test_ai_agentic_test_app.py::TestAgenticApp

pytest.ini

@@ -5,3 +5,6 @@ markers =
 
 junit_family =
     xunit2
+
+log_cli = true
+log_cli_level = INFO

tests/system/__init__.py

@@ -13,11 +13,15 @@
 # under the License.
 
 
+import logging
+
 import pytest
 
 from splunklib.binding import HTTPError
 from tests.ai_testlib import AITestCase
 
+logger = logging.getLogger(__name__)
+
 
 class TestAgenticApp(AITestCase):
     def test_agentic_app(self) -> None:
@@ -107,6 +111,34 @@ def test_agentic_app_with_remote_tools(self) -> None:
         app.delete()
         self.restart_splunk()  # app removal requires a restart
 
+    def test_remote_tools_have_tags(self) -> None:
+        pytest.importorskip("langchain_openai")
+        self.requires_splunk_10_2()
+
+        # Skip test in case the instance does not have a /splunk-mcp-server.tgz file.
+        try:
+            resp = self.service.get("agentic_app/has_mcp_app_file")
+            assert resp.status == 200
+        except HTTPError as e:
+            if e.status == 404:
+                pytest.skip("Splunk MCP Server App file not found on Splunk instance")
+            raise
+
+        app = self.service.apps.create(name="/splunk-mcp-server.tgz", filename=True)  # pyright: ignore[reportUnknownVariableType]
+
+        resp = self.service.post(
+            "agentic_app/tags",
+            body=self.test_llm_settings.model_dump_json(),
+        )
+
+        assert resp.status == 200
+        body = str(resp.body)  # pyright: ignore[reportUnknownArgumentType]
+        logger.info("tool-tags response: %s", body)
+        assert "tags" in body
+
+        app.delete()
+        self.restart_splunk()  # app removal requires a restart
+
     def requires_splunk_10_2(self) -> None:
         if self.service.splunk_version[0] < 10 or self.service.splunk_version[1] < 2:
             pytest.skip("Python 3.13 not available on splunk < 10.2")

tests/system/test_ai_agentic_test_app.py

@@ -14,12 +14,11 @@
 
 import os
 import sys
+from typing import override
 
 sys.path.insert(0, "/splunklib-deps")
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib"))
 
-from typing import override
-
 from pydantic import BaseModel, Field
 
 from splunklib.ai.agent import Agent

tests/system/test_apps/ai_agentic_test_app/bin/indexes.py

@@ -0,0 +1,65 @@
+# Copyright © 2011-2026 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import os
+import sys
+from typing import override
+
+sys.path.insert(0, "/splunklib-deps")
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib"))
+
+
+from splunklib.ai.agent import Agent
+from splunklib.ai.tool_settings import RemoteToolSettings, ToolAllowlist, ToolSettings
+from tests.cre_testlib import CRETestHandler
+
+# BUG: For some reason the CRE process is started with a overridden trust store path, that
+# does not exist on the filesystem. As a workaround in such case if it does not exist,
+# remove the env, this causes the default CAs to be used instead.
+CA_TRUST_STORE = "/opt/splunk/openssl/cert.pem"
+if os.environ.get("SSL_CERT_FILE") == CA_TRUST_STORE and not os.path.exists(
+    CA_TRUST_STORE
+):
+    os.environ["SSL_CERT_FILE"] = ""
+
+# This app uses the splunk_get_indexes remote tool (from Splunk MCP Server App).
+# Requires that the MCP Server App is installed.
+
+
+class ToolTagsHandler(CRETestHandler):
+    @override
+    async def run(self) -> None:
+        async with Agent(
+            model=await self.model(),
+            system_prompt="You are a helpful Splunk assistant",
+            tool_settings=ToolSettings(
+                local=False,
+                remote=RemoteToolSettings(
+                    allowlist=ToolAllowlist(names=["splunk_get_indexes"])
+                ),
+            ),
+            service=self.service,
+        ) as agent:
+            assert len(agent.tools) > 0, "No remote tools loaded"
+
+            assert len([t for t in agent.tools if len(t.tags) > 0]) > 0, (
+                f"No tools have tags. Tools: {[t.name for t in agent.tools]}"
+            )
+
+            self.response.write(
+                json.dumps(
+                    {"tools": [{"name": t.name, "tags": t.tags} for t in agent.tools]}
+                )
+            )

tests/system/test_apps/ai_agentic_test_app/bin/tags.py

@@ -0,0 +1,75 @@
+# Copyright © 2011-2026 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"): you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import os
+import sys
+from uuid import uuid4
+
+sys.path.insert(0, "/splunklib-deps")
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib"))
+
+from typing import override
+
+from splunklib.ai.agent import (
+    _get_splunk_username,  # pyright: ignore[reportPrivateUsage]
+)
+from splunklib.ai.tools import (
+    _list_all_tools,  # pyright: ignore[reportPrivateUsage]
+    connect_remote_mcp,
+)
+from tests.cre_testlib import CRETestHandler
+
+# BUG: For some reason the CRE process is started with a overridden trust store path, that
+# does not exist on the filesystem. As a workaround in such case if it does not exist,
+# remove the env, this causes the default CAs to be used instead.
+CA_TRUST_STORE = "/opt/splunk/openssl/cert.pem"
+if os.environ.get("SSL_CERT_FILE") == CA_TRUST_STORE and not os.path.exists(
+    CA_TRUST_STORE
+):
+    os.environ["SSL_CERT_FILE"] = ""
+
+# This handler connects directly to the Splunk MCP Server App and logs the raw
+# MCP tool list response, verifying that tool metadata includes tags.
+
+
+class ToolTagsHandler(CRETestHandler):
+    @override
+    async def run(self) -> None:
+        import asyncio
+
+        splunk_username = await asyncio.to_thread(
+            lambda: _get_splunk_username(self.service)
+        )
+
+        async with connect_remote_mcp(
+            service=self.service,
+            app_id="ai_agentic_test_app",
+            trace_id=str(uuid4()),
+            splunk_username=splunk_username,
+        ) as session:
+            assert session is not None, "MCP Server App not available"
+
+            raw_tools = await _list_all_tools(session)
+
+        assert len(raw_tools) > 0, "No tools returned by MCP Server App"
+
+        tools_with_tags = [
+            t for t in raw_tools if ((t.meta or {}).get("splunk") or {}).get("tags")
+        ]
+        assert len(tools_with_tags) > 0, (
+            f"No tools have tags. Raw tools: {json.dumps([t.model_dump() for t in raw_tools], indent=2)}"
+        )
+
+        self.response.write(json.dumps([t.model_dump() for t in raw_tools], indent=2))

tests/system/test_apps/ai_agentic_test_app/bin/tool_tags.py

@@ -15,3 +15,9 @@ match           = /agentic_app/has_mcp_app_file
 scripttype      = python
 handler         = mcp_app_file_exists.Handler
 python.required = 3.13
+
+[script:tool_tags]
+match           = /agentic_app/tool-tags
+scripttype      = python
+handler         = tool_tags.ToolTagsHandler
+python.required = 3.13