diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 567c4954..2661e335 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,10 +46,13 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae with: path: .pytest_cache - key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{ github.sha }} + key: pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}-${{ + github.sha }} restore-keys: | pytest-cache-${{ runner.os }}-py${{ matrix.python-version }}-${{ github.ref_name }}- - name: Run unit tests run: make test-unit - name: Run entire test suite - run: make test-integration + # run: make test-integration + run: uv run pytest + ./tests/system/test_ai_agentic_test_app.py::TestAgenticApp::test_remote_tools_have_tags diff --git a/tests/system/__init__.py b/tests/system/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/system/test_ai_agentic_test_app.py b/tests/system/test_ai_agentic_test_app.py index f45dae26..f55a804b 100644 --- a/tests/system/test_ai_agentic_test_app.py +++ b/tests/system/test_ai_agentic_test_app.py @@ -107,6 +107,33 @@ def test_agentic_app_with_remote_tools(self) -> None: app.delete() self.restart_splunk() # app removal requires a restart + def test_remote_tools_have_tags(self) -> None: + pytest.importorskip("langchain_openai") + self.requires_splunk_10_2() + + # Skip test in case the instance does not have a /splunk-mcp-server.tgz file. + try: + resp = self.service.get("agentic_app/has_mcp_app_file") + assert resp.status == 200 + except HTTPError as e: + if e.status == 404: + pytest.skip("Splunk MCP Server App file not found on Splunk instance") + raise + + app = self.service.apps.create(name="/splunk-mcp-server.tgz", filename=True) # pyright: ignore[reportUnknownVariableType] + + resp = self.service.post( + "agentic_app/tags", + body=self.test_llm_settings.model_dump_json(), + ) + + assert resp.status == 200 + body = str(resp.body) # pyright: ignore[reportUnknownArgumentType] + assert "tags" in body + + app.delete() + self.restart_splunk() # app removal requires a restart + def requires_splunk_10_2(self) -> None: if self.service.splunk_version[0] < 10 or self.service.splunk_version[1] < 2: pytest.skip("Python 3.13 not available on splunk < 10.2") diff --git a/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py b/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py index 5dbc8650..2f488557 100644 --- a/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py +++ b/tests/system/test_apps/ai_agentic_test_app/bin/indexes.py @@ -14,12 +14,11 @@ import os import sys +from typing import override sys.path.insert(0, "/splunklib-deps") sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib")) -from typing import override - from pydantic import BaseModel, Field from splunklib.ai.agent import Agent diff --git a/tests/system/test_apps/ai_agentic_test_app/bin/tags.py b/tests/system/test_apps/ai_agentic_test_app/bin/tags.py new file mode 100644 index 00000000..4de13ef2 --- /dev/null +++ b/tests/system/test_apps/ai_agentic_test_app/bin/tags.py @@ -0,0 +1,65 @@ +# Copyright © 2011-2026 Splunk, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"): you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os +import sys +from typing import override + +sys.path.insert(0, "/splunklib-deps") +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "lib")) + + +from splunklib.ai.agent import Agent +from splunklib.ai.tool_settings import RemoteToolSettings, ToolAllowlist, ToolSettings +from tests.cre_testlib import CRETestHandler + +# BUG: For some reason the CRE process is started with a overridden trust store path, that +# does not exist on the filesystem. As a workaround in such case if it does not exist, +# remove the env, this causes the default CAs to be used instead. +CA_TRUST_STORE = "/opt/splunk/openssl/cert.pem" +if os.environ.get("SSL_CERT_FILE") == CA_TRUST_STORE and not os.path.exists( + CA_TRUST_STORE +): + os.environ["SSL_CERT_FILE"] = "" + +# This app uses the splunk_get_indexes remote tool (from Splunk MCP Server App). +# Requires that the MCP Server App is installed. + + +class ToolTagsHandler(CRETestHandler): + @override + async def run(self) -> None: + async with Agent( + model=await self.model(), + system_prompt="You are a helpful Splunk assistant", + tool_settings=ToolSettings( + local=False, + remote=RemoteToolSettings( + allowlist=ToolAllowlist(names=["splunk_get_indexes"]) + ), + ), + service=self.service, + ) as agent: + assert len(agent.tools) > 0, "No remote tools loaded" + + assert len([t for t in agent.tools if len(t.tags) > 0]) > 0, ( + f"No tools have tags. Tools: {[t.name for t in agent.tools]}" + ) + + self.response.write( + json.dumps( + {"tools": [{"name": t.name, "tags": t.tags} for t in agent.tools]} + ) + ) diff --git a/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf b/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf index 5e16d9eb..c8d66b46 100644 --- a/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf +++ b/tests/system/test_apps/ai_agentic_test_app/default/restmap.conf @@ -15,3 +15,9 @@ match = /agentic_app/has_mcp_app_file scripttype = python handler = mcp_app_file_exists.Handler python.required = 3.13 + +[script:tool_tags] +match = /agentic_app/tags +scripttype = python +handler = tags.ToolTagsHandler +python.required = 3.13