From be8693df782ec1ff1593a52cd301e70bad1283cd Mon Sep 17 00:00:00 2001 From: gaurav0107 Date: Tue, 30 Jun 2026 02:31:31 +0530 Subject: [PATCH 1/2] feat: honor tool_not_found_behavior for missing custom tool calls When the model emitted a custom tool call for a tool the agent does not have, process_model_response always raised ModelBehaviorError and aborted the run, ignoring RunConfig.tool_not_found_behavior. Only function tool calls honored the "return_error_to_model" opt-in. Surface a missing custom tool back to the model as a custom_tool_call_output error when return_error_to_model is set, mirroring the existing function-tool path, so the run can recover instead of crashing. The default raise_error behavior is unchanged. Related to #325 --- src/agents/run_config.py | 7 +-- src/agents/run_internal/run_steps.py | 9 ++++ src/agents/run_internal/turn_resolution.py | 60 +++++++++++++++++++++- tests/test_process_model_response.py | 58 +++++++++++++++++++++ 4 files changed, 130 insertions(+), 4 deletions(-) diff --git a/src/agents/run_config.py b/src/agents/run_config.py index 45dcca5b10..466365804b 100644 --- a/src/agents/run_config.py +++ b/src/agents/run_config.py @@ -330,11 +330,12 @@ class RunConfig: """Optional SDK-side execution settings for local tool calls.""" tool_not_found_behavior: ToolNotFoundBehavior = "raise_error" - """Controls unresolved function tool calls emitted by the model. + """Controls unresolved function and custom tool calls emitted by the model. - ``"raise_error"`` preserves the default behavior and raises ``ModelBehaviorError``. - - ``"return_error_to_model"`` returns a model-visible ``function_call_output`` error and lets - the run continue. + - ``"return_error_to_model"`` returns a model-visible error and lets the run continue. The + error is surfaced as a ``function_call_output`` for missing function tools and as a + ``custom_tool_call_output`` for missing custom tools. """ diff --git a/src/agents/run_internal/run_steps.py b/src/agents/run_internal/run_steps.py index ec64b760a6..71e9af46bc 100644 --- a/src/agents/run_internal/run_steps.py +++ b/src/agents/run_internal/run_steps.py @@ -40,6 +40,7 @@ "ToolRunShellCall", "ToolRunApplyPatchCall", "ToolRunFunctionNotFound", + "ToolRunCustomNotFound", "ProcessedResponse", "NextStepHandoff", "NextStepFinalOutput", @@ -76,6 +77,12 @@ class ToolRunFunctionNotFound: tool_name: str +@dataclass +class ToolRunCustomNotFound: + tool_call: Any + tool_name: str + + @dataclass class ToolRunComputerAction: tool_call: ResponseComputerToolCall @@ -128,6 +135,7 @@ class ProcessedResponse: default_factory=list ) custom_tool_calls: list[ToolRunCustom] = dataclasses.field(default_factory=list) + custom_tools_not_found: list[ToolRunCustomNotFound] = dataclasses.field(default_factory=list) def has_tools_or_approvals_to_run(self) -> bool: # Handoffs, functions and computer actions need local processing @@ -143,6 +151,7 @@ def has_tools_or_approvals_to_run(self) -> bool: self.apply_patch_calls, self.mcp_approval_requests, self.function_tools_not_found, + self.custom_tools_not_found, ] ) diff --git a/src/agents/run_internal/turn_resolution.py b/src/agents/run_internal/turn_resolution.py index 2c95cf2e13..a2c58dbf8b 100644 --- a/src/agents/run_internal/turn_resolution.py +++ b/src/agents/run_internal/turn_resolution.py @@ -115,6 +115,7 @@ ToolRunApplyPatchCall, ToolRunComputerAction, ToolRunCustom, + ToolRunCustomNotFound, ToolRunFunction, ToolRunFunctionNotFound, ToolRunHandoff, @@ -220,6 +221,7 @@ async def _resolve_tool_not_found_message( run_config: RunConfig, tool_name: str, call_id: str, + tool_type: Literal["function", "computer", "shell", "apply_patch", "custom"] = "function", ) -> str: default_message = _default_tool_not_found_message(tool_name) formatter = run_config.tool_error_formatter @@ -230,7 +232,7 @@ async def _resolve_tool_not_found_message( maybe_message = formatter( ToolErrorFormatterArgs( kind="tool_not_found", - tool_type="function", + tool_type=tool_type, tool_name=tool_name, call_id=call_id, default_message=default_message, @@ -281,6 +283,42 @@ async def _build_tool_not_found_output_items( return items +async def _build_custom_tool_not_found_output_items( + *, + agent: Agent[Any], + calls: Sequence[ToolRunCustomNotFound], + context_wrapper: RunContextWrapper[Any], + run_config: RunConfig, +) -> list[RunItem]: + items: list[RunItem] = [] + for call in calls: + message = await _resolve_tool_not_found_message( + context_wrapper=context_wrapper, + run_config=run_config, + tool_name=call.tool_name, + call_id=call.tool_call.call_id, + tool_type="custom", + ) + # A custom tool call must be answered with a ``custom_tool_call_output`` item so + # the call id is resolved with the correct output type, mirroring the successful + # and rejected custom-tool paths. + items.append( + ToolCallOutputItem( + output=message, + raw_item=cast( + Any, + { + "type": "custom_tool_call_output", + "call_id": call.tool_call.call_id, + "output": message, + }, + ), + agent=agent, + ) + ) + return items + + async def run_final_output_hooks( agent: Agent[TContext], hooks: RunHooks[TContext], @@ -695,6 +733,14 @@ async def execute_tools_and_side_effects( run_config=run_config, ) ) + new_step_items.extend( + await _build_custom_tool_not_found_output_items( + agent=public_agent, + calls=processed_response.custom_tools_not_found, + context_wrapper=context_wrapper, + run_config=run_config, + ) + ) interruptions = _collect_tool_interruptions( function_results=function_results, @@ -1569,6 +1615,7 @@ def process_model_response( apply_patch_calls = [] mcp_approval_requests = [] function_tools_not_found = [] + custom_tools_not_found = [] tools_used: list[str] = [] handoff_map = {handoff.tool_name: handoff for handoff in handoffs} function_map = build_function_tool_lookup_map( @@ -1880,6 +1927,16 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: data={"tool_name": output.name}, ) ) + if run_config is not None and ( + run_config.tool_not_found_behavior == "return_error_to_model" + ): + # Mirror the function-tool path: instead of aborting the run, surface the + # missing custom tool back to the model as a custom_tool_call_output error + # so it can recover on the next turn. + custom_tools_not_found.append( + ToolRunCustomNotFound(tool_call=output, tool_name=output.name) + ) + continue raise ModelBehaviorError(f"Tool {output.name} not found in agent {agent.name}") elif ( isinstance(output, ResponseFunctionToolCall) @@ -1998,6 +2055,7 @@ def _dump_output_item(raw_item: Any) -> dict[str, Any]: mcp_approval_requests=mcp_approval_requests, interruptions=[], function_tools_not_found=function_tools_not_found, + custom_tools_not_found=custom_tools_not_found, ) diff --git a/tests/test_process_model_response.py b/tests/test_process_model_response.py index f21d65911f..26d3c56c1e 100644 --- a/tests/test_process_model_response.py +++ b/tests/test_process_model_response.py @@ -889,3 +889,61 @@ def test_process_model_response_collects_missing_function_tool_when_opted_in() - assert processed.function_tools_not_found[0].tool_call is missing_call assert processed.function_tools_not_found[0].tool_name == "missing_tool" assert processed.has_tools_or_approvals_to_run() + + +def test_process_model_response_raises_for_missing_custom_tool_by_default() -> None: + custom_tool = CustomTool( + name="raw_editor", + description="Edit raw text.", + on_invoke_tool=lambda _ctx, raw_input: raw_input, + format={"type": "text"}, + ) + agent = Agent(name="custom-agent", model=FakeModel(), tools=[custom_tool]) + missing_call = ResponseCustomToolCall( + type="custom_tool_call", + name="ghost_custom_tool", + call_id="custom-missing-1", + input="payload", + ) + + with pytest.raises(ModelBehaviorError, match="ghost_custom_tool"): + run_loop.process_model_response( + agent=agent, + all_tools=[custom_tool], + response=_response([missing_call]), + output_schema=None, + handoffs=[], + ) + + +def test_process_model_response_collects_missing_custom_tool_when_opted_in() -> None: + custom_tool = CustomTool( + name="raw_editor", + description="Edit raw text.", + on_invoke_tool=lambda _ctx, raw_input: raw_input, + format={"type": "text"}, + ) + agent = Agent(name="custom-agent", model=FakeModel(), tools=[custom_tool]) + missing_call = ResponseCustomToolCall( + type="custom_tool_call", + name="ghost_custom_tool", + call_id="custom-missing-1", + input="payload", + ) + + processed = run_loop.process_model_response( + agent=agent, + all_tools=[custom_tool], + response=_response([missing_call]), + output_schema=None, + handoffs=[], + run_config=RunConfig(tool_not_found_behavior="return_error_to_model"), + ) + + assert len(processed.new_items) == 1 + assert isinstance(processed.new_items[0], ToolCallItem) + assert processed.custom_tool_calls == [] + assert len(processed.custom_tools_not_found) == 1 + assert cast(object, processed.custom_tools_not_found[0].tool_call) is missing_call + assert processed.custom_tools_not_found[0].tool_name == "ghost_custom_tool" + assert processed.has_tools_or_approvals_to_run() From d7362ec851a2cf181ff061d7ac8dc01d93d79719 Mon Sep 17 00:00:00 2001 From: gaurav0107 Date: Tue, 30 Jun 2026 02:38:36 +0530 Subject: [PATCH 2/2] test: cover end-to-end custom tool not-found recovery Add Runner-level tests that drive a missing custom tool call through return_error_to_model and assert the run recovers and the next turn input contains a custom_tool_call_output with the missing call id and the not-found message, including a tool_error_formatter variant that confirms tool_type is "custom". --- tests/test_agent_runner.py | 93 +++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/tests/test_agent_runner.py b/tests/test_agent_runner.py index 4b5ea867ce..421066a4fa 100644 --- a/tests/test_agent_runner.py +++ b/tests/test_agent_runner.py @@ -12,7 +12,7 @@ import httpx import pytest from openai import APIConnectionError, BadRequestError -from openai.types.responses import ResponseFunctionToolCall +from openai.types.responses import ResponseCustomToolCall, ResponseFunctionToolCall from openai.types.responses.response_output_text import AnnotationFileCitation, ResponseOutputText from openai.types.responses.response_reasoning_item import ResponseReasoningItem, Summary from typing_extensions import TypedDict @@ -4534,6 +4534,97 @@ async def formatter(args: Any) -> str | None: ] +@pytest.mark.asyncio +async def test_tool_not_found_behavior_returns_error_to_model_for_custom_tool() -> None: + model = FakeModel() + agent = Agent(name="test", model=model, tool_use_behavior="run_llm_again") + missing_call = ResponseCustomToolCall( + type="custom_tool_call", + name="missing_custom_tool", + call_id="call_missing_custom", + input="payload", + ) + model.add_multiple_turn_outputs( + [ + [missing_call], + [get_text_message("recovered")], + ] + ) + + result = await Runner.run( + agent, + input="start", + run_config=RunConfig(tool_not_found_behavior="return_error_to_model"), + ) + + assert result.final_output == "recovered" + second_turn_input = model.last_turn_args["input"] + assert isinstance(second_turn_input, list) + tool_outputs = [ + item + for item in second_turn_input + if isinstance(item, dict) and item.get("type") == "custom_tool_call_output" + ] + assert tool_outputs == [ + { + "call_id": "call_missing_custom", + "output": "Tool 'missing_custom_tool' not found.", + "type": "custom_tool_call_output", + } + ] + + +@pytest.mark.asyncio +async def test_tool_not_found_behavior_uses_tool_error_formatter_for_custom_tool() -> None: + model = FakeModel() + agent = Agent(name="test", model=model, tool_use_behavior="run_llm_again") + missing_call = ResponseCustomToolCall( + type="custom_tool_call", + name="missing_custom_tool", + call_id="call_missing_custom", + input="payload", + ) + model.add_multiple_turn_outputs( + [ + [missing_call], + [get_text_message("recovered")], + ] + ) + seen_tool_types: list[str] = [] + + async def formatter(args: Any) -> str | None: + if args.kind != "tool_not_found": + return None + seen_tool_types.append(args.tool_type) + return f"{args.tool_name} unavailable for {args.call_id}" + + result = await Runner.run( + agent, + input="start", + run_config=RunConfig( + tool_not_found_behavior="return_error_to_model", + tool_error_formatter=formatter, + ), + ) + + assert result.final_output == "recovered" + assert seen_tool_types == ["custom"] + second_turn_input = model.last_turn_args["input"] + assert isinstance(second_turn_input, list) + tool_outputs = [ + item + for item in second_turn_input + if isinstance(item, dict) and item.get("type") == "custom_tool_call_output" + ] + assert tool_outputs == [ + { + "call_id": "call_missing_custom", + "output": "missing_custom_tool unavailable for call_missing_custom", + "type": "custom_tool_call_output", + } + ] + + @pytest.mark.asyncio async def test_tool_not_found_behavior_handles_mixed_function_tool_calls() -> None: model = FakeModel()