|
@@ -7,6 +7,7 @@ from typing import Any
|
|
|
import pytest
|
|
import pytest
|
|
|
|
|
|
|
|
from agent_lab.application.contracts import AgentParams, DebugRunRequest, EventAgentParams
|
|
from agent_lab.application.contracts import AgentParams, DebugRunRequest, EventAgentParams
|
|
|
|
|
+from agent_lab.application.event_agent import EventAgentRequest
|
|
|
from agent_lab.application.runtime import DebugRuntime
|
|
from agent_lab.application.runtime import DebugRuntime
|
|
|
from agent_lab.application.tools import ToolDefinition, ToolRegistry
|
|
from agent_lab.application.tools import ToolDefinition, ToolRegistry
|
|
|
from agent_lab.domain.events import ToolCallEvent
|
|
from agent_lab.domain.events import ToolCallEvent
|
|
@@ -44,6 +45,9 @@ class RecordingQueue(asyncio.Queue):
|
|
|
return item.role
|
|
return item.role
|
|
|
if isinstance(item, ToolCallEvent):
|
|
if isinstance(item, ToolCallEvent):
|
|
|
return f"event:{item.name}:{item.id}"
|
|
return f"event:{item.name}:{item.id}"
|
|
|
|
|
+ if isinstance(item, EventAgentRequest):
|
|
|
|
|
+ events = ",".join(f"{event.name}:{event.id}" for event in item.events)
|
|
|
|
|
+ return f"event_request:{events}"
|
|
|
if isinstance(item, dict):
|
|
if isinstance(item, dict):
|
|
|
return f"output:{item.get('type')}"
|
|
return f"output:{item.get('type')}"
|
|
|
return type(item).__name__
|
|
return type(item).__name__
|
|
@@ -65,8 +69,8 @@ class FakeChatClient:
|
|
|
ToolCallEvent(
|
|
ToolCallEvent(
|
|
|
id="call_1",
|
|
id="call_1",
|
|
|
name="handoff_note",
|
|
name="handoff_note",
|
|
|
- arguments={"message": "need event agent"},
|
|
|
|
|
- raw_arguments='{"message":"need event agent"}',
|
|
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
)
|
|
)
|
|
|
)
|
|
)
|
|
|
return
|
|
return
|
|
@@ -92,8 +96,8 @@ class StrictHistoryChatClient:
|
|
|
ToolCallEvent(
|
|
ToolCallEvent(
|
|
|
id="call_1",
|
|
id="call_1",
|
|
|
name="handoff_note",
|
|
name="handoff_note",
|
|
|
- arguments={"message": "need event agent"},
|
|
|
|
|
- raw_arguments='{"message":"need event agent"}',
|
|
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
)
|
|
)
|
|
|
)
|
|
)
|
|
|
return
|
|
return
|
|
@@ -102,6 +106,42 @@ class StrictHistoryChatClient:
|
|
|
yield StreamItem.message_delta("final answer")
|
|
yield StreamItem.message_delta("final answer")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+class MultiEventChatClient:
|
|
|
|
|
+ def __init__(self) -> None:
|
|
|
|
|
+ self.calls = 0
|
|
|
|
|
+ self.second_call_messages: list[ChatMessage] = []
|
|
|
|
|
+
|
|
|
|
|
+ async def stream_chat(
|
|
|
|
|
+ self,
|
|
|
|
|
+ messages: list[ChatMessage],
|
|
|
|
|
+ tools: list[dict],
|
|
|
|
|
+ params: AgentParams,
|
|
|
|
|
+ ) -> AsyncIterator[StreamItem]:
|
|
|
|
|
+ self.calls += 1
|
|
|
|
|
+ if self.calls == 1:
|
|
|
|
|
+ yield StreamItem.message_delta("Checking events.")
|
|
|
|
|
+ yield StreamItem.event(
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_1",
|
|
|
|
|
+ name="handoff_note",
|
|
|
|
|
+ arguments={"message": "ignored chat argument"},
|
|
|
|
|
+ raw_arguments='{"message":"ignored chat argument"}',
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ yield StreamItem.event(
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_2",
|
|
|
|
|
+ name="audit_note",
|
|
|
|
|
+ arguments={"message": "ignored chat argument"},
|
|
|
|
|
+ raw_arguments='{"message":"ignored chat argument"}',
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+ self.second_call_messages = list(messages)
|
|
|
|
|
+ yield StreamItem.message_delta("Final answer.")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
class ToolCapturingChatClient:
|
|
class ToolCapturingChatClient:
|
|
|
def __init__(self) -> None:
|
|
def __init__(self) -> None:
|
|
|
self.tools: list[dict[str, Any]] = []
|
|
self.tools: list[dict[str, Any]] = []
|
|
@@ -116,6 +156,33 @@ class ToolCapturingChatClient:
|
|
|
yield StreamItem.message_delta("final answer")
|
|
yield StreamItem.message_delta("final answer")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+class EventLoopLimitChatClient:
|
|
|
|
|
+ def __init__(self) -> None:
|
|
|
|
|
+ self.calls = 0
|
|
|
|
|
+ self.tools_by_call: list[list[dict[str, Any]]] = []
|
|
|
|
|
+
|
|
|
|
|
+ async def stream_chat(
|
|
|
|
|
+ self,
|
|
|
|
|
+ messages: list[ChatMessage],
|
|
|
|
|
+ tools: list[dict],
|
|
|
|
|
+ params: AgentParams,
|
|
|
|
|
+ ) -> AsyncIterator[StreamItem]:
|
|
|
|
|
+ self.calls += 1
|
|
|
|
|
+ self.tools_by_call.append(list(tools))
|
|
|
|
|
+ if self.calls == 1:
|
|
|
|
|
+ yield StreamItem.event(
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_1",
|
|
|
|
|
+ name="handoff_note",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ return
|
|
|
|
|
+
|
|
|
|
|
+ yield StreamItem.message_delta("final after event limit")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
class RoundStatsChatClient:
|
|
class RoundStatsChatClient:
|
|
|
async def stream_chat(
|
|
async def stream_chat(
|
|
|
self,
|
|
self,
|
|
@@ -150,8 +217,8 @@ class EventRoundStatsChatClient:
|
|
|
ToolCallEvent(
|
|
ToolCallEvent(
|
|
|
id="call_1",
|
|
id="call_1",
|
|
|
name="handoff_note",
|
|
name="handoff_note",
|
|
|
- arguments={"message": "need event agent"},
|
|
|
|
|
- raw_arguments='{"message":"need event agent"}',
|
|
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
)
|
|
)
|
|
|
)
|
|
)
|
|
|
yield StreamItem.usage_item(
|
|
yield StreamItem.usage_item(
|
|
@@ -206,6 +273,104 @@ async def test_runtime_routes_chat_events_through_event_agent_then_continues_cha
|
|
|
assert outputs[4]["content"] == "final answer"
|
|
assert outputs[4]["content"] == "final answer"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_runtime_batches_round_events_before_continuing_chat_agent():
|
|
|
|
|
+ def resolve_from_history(
|
|
|
|
|
+ event: ToolCallEvent,
|
|
|
|
|
+ history: list[ChatMessage],
|
|
|
|
|
+ ) -> dict[str, Any]:
|
|
|
|
|
+ return {"message": history[-1].content, "event": event.name}
|
|
|
|
|
+
|
|
|
|
|
+ registry = ToolRegistry(
|
|
|
|
|
+ [
|
|
|
|
|
+ ToolDefinition(
|
|
|
|
|
+ name="handoff_note",
|
|
|
|
|
+ description="Send a handoff note.",
|
|
|
|
|
+ parameters={
|
|
|
|
|
+ "type": "object",
|
|
|
|
|
+ "properties": {"message": {"type": "string"}},
|
|
|
|
|
+ "required": ["message"],
|
|
|
|
|
+ },
|
|
|
|
|
+ handler=lambda event: {
|
|
|
|
|
+ "tool": event.name,
|
|
|
|
|
+ "message": event.arguments["message"],
|
|
|
|
|
+ },
|
|
|
|
|
+ argument_resolver=resolve_from_history,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ToolDefinition(
|
|
|
|
|
+ name="audit_note",
|
|
|
|
|
+ description="Send an audit note.",
|
|
|
|
|
+ parameters={
|
|
|
|
|
+ "type": "object",
|
|
|
|
|
+ "properties": {"message": {"type": "string"}},
|
|
|
|
|
+ "required": ["message"],
|
|
|
|
|
+ },
|
|
|
|
|
+ handler=lambda event: {
|
|
|
|
|
+ "tool": event.name,
|
|
|
|
|
+ "message": event.arguments["message"],
|
|
|
|
|
+ },
|
|
|
|
|
+ argument_resolver=resolve_from_history,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ]
|
|
|
|
|
+ )
|
|
|
|
|
+ request = DebugRunRequest(
|
|
|
|
|
+ user_message="debug this",
|
|
|
|
|
+ system_prompts=[],
|
|
|
|
|
+ pre_messages=[],
|
|
|
|
|
+ chat_agent=AgentParams(model="fake-model", temperature=0.1, max_tokens=200),
|
|
|
|
|
+ event_agent=EventAgentParams(
|
|
|
|
|
+ enabled_tools=["handoff_note", "audit_note"],
|
|
|
|
|
+ max_event_loops=2,
|
|
|
|
|
+ ),
|
|
|
|
|
+ )
|
|
|
|
|
+ client = MultiEventChatClient()
|
|
|
|
|
+ runtime = DebugRuntime(client, registry=registry)
|
|
|
|
|
+
|
|
|
|
|
+ outputs = [message async for message in runtime.run(request)]
|
|
|
|
|
+
|
|
|
|
|
+ assert [message["type"] for message in outputs] == [
|
|
|
|
|
+ "session_started",
|
|
|
|
|
+ "message_delta",
|
|
|
|
|
+ "event",
|
|
|
|
|
+ "event",
|
|
|
|
|
+ "tool_result",
|
|
|
|
|
+ "tool_result",
|
|
|
|
|
+ "round_stats",
|
|
|
|
|
+ "message_delta",
|
|
|
|
|
+ "round_stats",
|
|
|
|
|
+ "done",
|
|
|
|
|
+ ]
|
|
|
|
|
+ assert client.calls == 2
|
|
|
|
|
+ assert [message.role for message in client.second_call_messages] == [
|
|
|
|
|
+ "user",
|
|
|
|
|
+ "assistant",
|
|
|
|
|
+ "tool",
|
|
|
|
|
+ "tool",
|
|
|
|
|
+ ]
|
|
|
|
|
+ assistant_message = client.second_call_messages[1]
|
|
|
|
|
+ assert assistant_message.content == "Checking events."
|
|
|
|
|
+ assert assistant_message.tool_calls == [
|
|
|
|
|
+ {
|
|
|
|
|
+ "id": "call_1",
|
|
|
|
|
+ "type": "function",
|
|
|
|
|
+ "function": {"name": "handoff_note", "arguments": "{}"},
|
|
|
|
|
+ },
|
|
|
|
|
+ {
|
|
|
|
|
+ "id": "call_2",
|
|
|
|
|
+ "type": "function",
|
|
|
|
|
+ "function": {"name": "audit_note", "arguments": "{}"},
|
|
|
|
|
+ },
|
|
|
|
|
+ ]
|
|
|
|
|
+ assert [
|
|
|
|
|
+ json.loads(message.content)
|
|
|
|
|
+ for message in client.second_call_messages
|
|
|
|
|
+ if message.role == "tool"
|
|
|
|
|
+ ] == [
|
|
|
|
|
+ {"tool": "handoff_note", "message": "Checking events."},
|
|
|
|
|
+ {"tool": "audit_note", "message": "Checking events."},
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
async def test_runtime_start_returns_queues_for_downstream_output_consumer():
|
|
async def test_runtime_start_returns_queues_for_downstream_output_consumer():
|
|
|
request = DebugRunRequest(
|
|
request = DebugRunRequest(
|
|
@@ -234,6 +399,35 @@ async def test_runtime_start_returns_queues_for_downstream_output_consumer():
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_runtime_finalizes_chat_after_reaching_event_loop_limit():
|
|
|
|
|
+ request = DebugRunRequest(
|
|
|
|
|
+ user_message="debug this",
|
|
|
|
|
+ system_prompts=[],
|
|
|
|
|
+ pre_messages=[],
|
|
|
|
|
+ chat_agent=AgentParams(model="fake-model", temperature=0.1, max_tokens=200),
|
|
|
|
|
+ event_agent=EventAgentParams(enabled_tools=["handoff_note"], max_event_loops=1),
|
|
|
|
|
+ )
|
|
|
|
|
+ client = EventLoopLimitChatClient()
|
|
|
|
|
+ runtime = DebugRuntime(client)
|
|
|
|
|
+
|
|
|
|
|
+ outputs = [message async for message in runtime.run(request)]
|
|
|
|
|
+
|
|
|
|
|
+ assert client.calls == 2
|
|
|
|
|
+ assert client.tools_by_call[0][0]["function"]["name"] == "handoff_note"
|
|
|
|
|
+ assert client.tools_by_call[1] == []
|
|
|
|
|
+ assert [message["type"] for message in outputs] == [
|
|
|
|
|
+ "session_started",
|
|
|
|
|
+ "event",
|
|
|
|
|
+ "tool_result",
|
|
|
|
|
+ "round_stats",
|
|
|
|
|
+ "message_delta",
|
|
|
|
|
+ "round_stats",
|
|
|
|
|
+ "done",
|
|
|
|
|
+ ]
|
|
|
|
|
+ assert outputs[4]["content"] == "final after event limit"
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
async def test_runtime_buffers_upstream_user_input_until_after_matching_tool_reply():
|
|
async def test_runtime_buffers_upstream_user_input_until_after_matching_tool_reply():
|
|
|
RuntimeQueues = _runtime_queues_class()
|
|
RuntimeQueues = _runtime_queues_class()
|
|
@@ -342,12 +536,12 @@ async def test_runtime_uses_event_and_input_queues_for_event_agent_handoff():
|
|
|
assert queue_log.index(("input", "put", "user")) < queue_log.index(
|
|
assert queue_log.index(("input", "put", "user")) < queue_log.index(
|
|
|
("input", "get", "user")
|
|
("input", "get", "user")
|
|
|
)
|
|
)
|
|
|
- assert queue_log.index(("events", "put", "event:handoff_note:call_1")) < queue_log.index(
|
|
|
|
|
- ("events", "get", "event:handoff_note:call_1")
|
|
|
|
|
- )
|
|
|
|
|
- assert queue_log.index(("events", "get", "event:handoff_note:call_1")) < queue_log.index(
|
|
|
|
|
- ("input", "put", "tool:call_1")
|
|
|
|
|
- )
|
|
|
|
|
|
|
+ assert queue_log.index(
|
|
|
|
|
+ ("events", "put", "event_request:handoff_note:call_1")
|
|
|
|
|
+ ) < queue_log.index(("events", "get", "event_request:handoff_note:call_1"))
|
|
|
|
|
+ assert queue_log.index(
|
|
|
|
|
+ ("events", "get", "event_request:handoff_note:call_1")
|
|
|
|
|
+ ) < queue_log.index(("input", "put", "tool:call_1"))
|
|
|
assert queue_log.index(("input", "put", "tool:call_1")) < queue_log.index(
|
|
assert queue_log.index(("input", "put", "tool:call_1")) < queue_log.index(
|
|
|
("input", "get", "tool:call_1")
|
|
("input", "get", "tool:call_1")
|
|
|
)
|
|
)
|
|
@@ -430,7 +624,7 @@ async def test_runtime_preserves_assistant_tool_calls_before_tool_reply():
|
|
|
"type": "function",
|
|
"type": "function",
|
|
|
"function": {
|
|
"function": {
|
|
|
"name": "handoff_note",
|
|
"name": "handoff_note",
|
|
|
- "arguments": '{"message":"need event agent"}',
|
|
|
|
|
|
|
+ "arguments": "{}",
|
|
|
},
|
|
},
|
|
|
}
|
|
}
|
|
|
]
|
|
]
|
|
@@ -439,7 +633,7 @@ async def test_runtime_preserves_assistant_tool_calls_before_tool_reply():
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
-async def test_runtime_passes_selected_tool_schema_from_registry_to_chat_agent():
|
|
|
|
|
|
|
+async def test_runtime_passes_event_names_without_tool_parameters_to_chat_agent():
|
|
|
registry = ToolRegistry(
|
|
registry = ToolRegistry(
|
|
|
[
|
|
[
|
|
|
ToolDefinition(
|
|
ToolDefinition(
|
|
@@ -478,11 +672,8 @@ async def test_runtime_passes_selected_tool_schema_from_registry_to_chat_agent()
|
|
|
"description": "Registry-owned handoff tool.",
|
|
"description": "Registry-owned handoff tool.",
|
|
|
"parameters": {
|
|
"parameters": {
|
|
|
"type": "object",
|
|
"type": "object",
|
|
|
- "properties": {
|
|
|
|
|
- "message": {"type": "string"},
|
|
|
|
|
- "priority": {"type": "number"},
|
|
|
|
|
- },
|
|
|
|
|
- "required": ["message"],
|
|
|
|
|
|
|
+ "properties": {},
|
|
|
|
|
+ "additionalProperties": False,
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
}
|
|
}
|