Procházet zdrojové kódy

fix: harden event agent tool arguments

zhenyu.hu před 3 týdny
rodič
revize
23f91b1d9f

+ 33 - 0
docs/plans/todo-27-event-agent-tool-arguments.md

@@ -0,0 +1,33 @@
+# Todo 27: EventAgent Tool Arguments
+
+## Status
+
+done
+
+## Goal
+
+Stop EventAgent turns from ending with `event agent did not return arguments for mock_search` when the compatible provider fails to return a tool call in the exact shape the parser expects.
+
+## Root Cause
+
+EventAgent currently sends one tool schema to the LLM and expects the stream parser to emit a provider tool-call event. There are two brittle points:
+
+- The OpenAI-compatible request includes `tools` but does not force the single tool through `tool_choice`.
+- The stream parser only drains accumulated provider tool calls when `finish_reason == "tool_calls"`, so providers that end with another terminal reason can drop the accumulated tool call.
+
+If neither path emits an event, EventAgent raises a ValueError and the tool result becomes a handler error.
+
+## Scope
+
+- Force the single supplied tool in OpenAI-compatible requests.
+- Drain accumulated provider tool calls on any terminal finish reason.
+- Add an EventAgent fallback to registry/context argument resolution when the LLM returns no tool call.
+- Keep ChatAgent tool-free.
+
+## Verification
+
+- Failing parser test for terminal `finish_reason="stop"` with accumulated tool call.
+- Failing OpenAI client payload test for `tool_choice`.
+- Failing EventAgent test for no-tool-call fallback.
+- `uv run pytest tests/test_openai_stream_parser.py tests/test_event_agent.py tests/test_websocket_api.py::test_openai_chat_client_streams_sse_chunks_through_parser -q` passes: 9 tests.
+- `uv run pytest` passes: 53 tests.

+ 1 - 0
docs/plans/todos.md

@@ -53,3 +53,4 @@
 | 24 | done | `docs/plans/todo-24-chat-agent-prompt-list.md` | Put all ChatAgent prompts, including generated EventAgent event rules, into one ordered prompt list with add/delete/drag controls. | `uv run pytest` passes (`51 passed`, one existing Starlette deprecation warning), plus local browser prompt-list check. |
 | 25 | done | `docs/plans/todo-25-prompt-list-message-types.md` | Make the ChatAgent prompt list visually compact and show the message type for each prompt item. | `uv run pytest` passes (`51 passed`, one existing Starlette deprecation warning); local service returned `/health` and versioned JS asset. |
 | 26 | done | `docs/plans/todo-26-workspace-snapshot-save.md` | Move save/load/delete out of ChatAgent config and save one workspace snapshot covering prompts, Agent config, and selected tools. | `uv run pytest tests/test_websocket_api.py -q` passes (`27 passed`, one existing Starlette deprecation warning). |
+| 27 | done | `docs/plans/todo-27-event-agent-tool-arguments.md` | Make EventAgent tool argument generation robust for compatible providers that do not emit the expected tool-call finish reason or ignore tool calls. | `uv run pytest` passes (`53 passed`, one existing Starlette deprecation warning). |

+ 13 - 3
src/agent_lab/application/event_agent.py

@@ -78,7 +78,17 @@ class EventAgent:
                     system_prompt,
                     extra_body,
                 )
-                payload = self.registry.execute(resolved_event)
+                if resolved_event is None:
+                    payload = self.registry.handle(
+                        event,
+                        context=ToolExecutionContext(
+                            history=history,
+                            system_prompt=system_prompt,
+                            extra_body=extra_body or {},
+                        ),
+                    )
+                else:
+                    payload = self.registry.execute(resolved_event)
         except Exception as exc:
             payload = {
                 "tool": event.name,
@@ -111,7 +121,7 @@ class EventAgent:
         history: Sequence[ChatMessage],
         system_prompt: str,
         extra_body: dict[str, Any] | None,
-    ) -> ToolCallEvent:
+    ) -> ToolCallEvent | None:
         assert self.chat_client is not None
         tool = self.registry.tool_schema(event.name)
         if tool is None:
@@ -138,7 +148,7 @@ class EventAgent:
                     }
                 )
 
-        raise ValueError(f"event agent did not return arguments for {event.name}")
+        return None
 
     def _build_argument_messages(
         self,

+ 5 - 0
src/agent_lab/infrastructure/chat_client.py

@@ -45,6 +45,11 @@ class OpenAICompatibleChatClient:
             payload["stream_options"] = {"include_usage": True}
         if tools:
             payload["tools"] = tools
+            if len(tools) == 1:
+                payload["tool_choice"] = {
+                    "type": "function",
+                    "function": {"name": tools[0]["function"]["name"]},
+                }
         if params.extra_body:
             payload.update(params.extra_body)
 

+ 1 - 3
src/agent_lab/infrastructure/openai_compatible.py

@@ -23,10 +23,8 @@ class ChatCompletionStreamParser:
             for tool_call in delta.get("tool_calls") or []:
                 self._accumulate_tool_call(tool_call)
 
-            if choice.get("finish_reason") == "tool_calls":
-                items.extend(self._drain_tool_events())
-
             if choice.get("finish_reason") is not None:
+                items.extend(self._drain_tool_events())
                 items.extend(self.flush())
 
         usage = chunk.get("usage")

+ 48 - 0
tests/test_event_agent.py

@@ -39,6 +39,26 @@ class ToolCallingChatClient:
         )
 
 
+class NoToolCallChatClient:
+    def __init__(self) -> None:
+        self.calls: list[dict] = []
+
+    async def stream_chat(
+        self,
+        messages: list[ChatMessage],
+        tools: list[dict],
+        params: AgentParams,
+    ) -> AsyncIterator[StreamItem]:
+        self.calls.append(
+            {
+                "messages": list(messages),
+                "tools": list(tools),
+                "params": params,
+            }
+        )
+        yield StreamItem.message_delta("I should have called the tool.")
+
+
 @pytest.mark.asyncio
 async def test_event_agent_resolves_tool_arguments_with_llm_tool_call():
     chat_client = ToolCallingChatClient({"message": "LLM generated handoff"})
@@ -86,6 +106,34 @@ async def test_event_agent_resolves_tool_arguments_with_llm_tool_call():
     assert chat_client.calls[0]["params"].model == "event-model"
 
 
+@pytest.mark.asyncio
+async def test_event_agent_falls_back_to_context_arguments_when_llm_returns_no_tool_call():
+    chat_client = NoToolCallChatClient()
+    agent = EventAgent(
+        enabled_tools=["mock_search"],
+        chat_client=chat_client,
+        params=AgentParams(model="event-model", temperature=0, max_tokens=80),
+    )
+    event = ToolCallEvent(
+        id="call_1",
+        name="mock_search",
+        arguments={},
+        raw_arguments="{}",
+    )
+    history = [
+        ChatMessage(role="user", content="Find latency docs"),
+        ChatMessage(role="assistant", content="Need a search for latency docs"),
+    ]
+
+    reply = await agent.handle(event, history=history)
+
+    payload = json.loads(reply.content)
+    assert payload["tool"] == "mock_search"
+    assert payload["query"] == "Need a search for latency docs"
+    assert "event agent did not return arguments" not in reply.content
+    assert chat_client.calls[0]["tools"][0]["function"]["name"] == "mock_search"
+
+
 @pytest.mark.asyncio
 async def test_event_agent_returns_registry_errors_for_disabled_and_unknown_tools():
     registry = ToolRegistry(

+ 39 - 0
tests/test_openai_stream_parser.py

@@ -131,3 +131,42 @@ def test_parser_emits_provider_tool_call_events_for_event_agent():
             raw_arguments='{"query":"latency docs"}',
         )
     ]
+
+
+def test_parser_drains_provider_tool_call_events_on_any_terminal_finish_reason():
+    parser = ChatCompletionStreamParser()
+
+    items = []
+    items.extend(
+        parser.feed(
+            {
+                "choices": [
+                    {
+                        "delta": {
+                            "tool_calls": [
+                                {
+                                    "index": 0,
+                                    "id": "call_1",
+                                    "type": "function",
+                                    "function": {
+                                        "name": "mock_search",
+                                        "arguments": '{"query":"latency docs"}',
+                                    },
+                                }
+                            ]
+                        },
+                        "finish_reason": "stop",
+                    }
+                ]
+            }
+        )
+    )
+
+    assert [item.event for item in items if item.kind == "event"] == [
+        ToolCallEvent(
+            id="call_1",
+            name="mock_search",
+            arguments={"query": "latency docs"},
+            raw_arguments='{"query":"latency docs"}',
+        )
+    ]

+ 4 - 0
tests/test_websocket_api.py

@@ -323,6 +323,10 @@ async def test_openai_chat_client_streams_sse_chunks_through_parser():
         assert payload["model"] == "model-x"
         assert payload["messages"] == [{"role": "user", "content": "hi"}]
         assert payload["tools"][0]["function"]["name"] == "handoff_note"
+        assert payload["tool_choice"] == {
+            "type": "function",
+            "function": {"name": "handoff_note"},
+        }
         assert payload["temperature"] == 0.3
         assert payload["max_tokens"] == 50
         assert request.headers["authorization"] == "Bearer test-key"