|
@@ -413,6 +413,30 @@ def test_debug_run_request_rejects_tool_pre_messages():
|
|
|
DebugRunRequest.model_validate(payload)
|
|
DebugRunRequest.model_validate(payload)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+def test_debug_run_request_rejects_assistant_tool_calls_in_pre_messages():
|
|
|
|
|
+ payload = _request_payload()
|
|
|
|
|
+ payload["pre_messages"] = [
|
|
|
|
|
+ {
|
|
|
|
|
+ "role": "assistant",
|
|
|
|
|
+ "content": "I will inspect this.",
|
|
|
|
|
+ "tool_calls": [
|
|
|
|
|
+ {
|
|
|
|
|
+ "id": "call_1",
|
|
|
|
|
+ "name": "mock_search",
|
|
|
|
|
+ "arguments": {"query": "latency"},
|
|
|
|
|
+ "raw_arguments": '{"query":"latency"}',
|
|
|
|
|
+ }
|
|
|
|
|
+ ],
|
|
|
|
|
+ }
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ with pytest.raises(
|
|
|
|
|
+ ValidationError,
|
|
|
|
|
+ match="pre_messages cannot include assistant tool calls",
|
|
|
|
|
+ ):
|
|
|
|
|
+ DebugRunRequest.model_validate(payload)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def test_chat_message_allows_internal_tool_replies():
|
|
def test_chat_message_allows_internal_tool_replies():
|
|
|
message = ChatMessage(
|
|
message = ChatMessage(
|
|
|
role="tool",
|
|
role="tool",
|
|
@@ -482,6 +506,34 @@ def test_chat_message_rejects_tool_role_without_tool_call_id():
|
|
|
ChatMessage(role="tool", content="orphan result")
|
|
ChatMessage(role="tool", content="orphan result")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.parametrize("role", ["system", "user", "assistant"])
|
|
|
|
|
+def test_chat_message_rejects_tool_call_id_on_non_tool_roles(role: str):
|
|
|
|
|
+ with pytest.raises(ValidationError, match="tool_call_id requires tool role"):
|
|
|
|
|
+ ChatMessage(role=role, content="invalid", tool_call_id="call_1")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+def test_chat_message_rejects_duplicate_assistant_tool_call_ids():
|
|
|
|
|
+ with pytest.raises(ValidationError, match="duplicate assistant tool-call ID: call_1"):
|
|
|
|
|
+ ChatMessage(
|
|
|
|
|
+ role="assistant",
|
|
|
|
|
+ content="checking twice",
|
|
|
|
|
+ tool_calls=[
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_1",
|
|
|
|
|
+ name="mock_search",
|
|
|
|
|
+ arguments={"query": "first"},
|
|
|
|
|
+ raw_arguments='{"query":"first"}',
|
|
|
|
|
+ ),
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_1",
|
|
|
|
|
+ name="handoff_note",
|
|
|
|
|
+ arguments={"message": "second"},
|
|
|
|
|
+ raw_arguments='{"message":"second"}',
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
class HistoryCapturingChatClient:
|
|
class HistoryCapturingChatClient:
|
|
|
def __init__(self) -> None:
|
|
def __init__(self) -> None:
|
|
|
self.calls = 0
|
|
self.calls = 0
|
|
@@ -492,6 +544,7 @@ class HistoryCapturingChatClient:
|
|
|
messages: list[ChatMessage],
|
|
messages: list[ChatMessage],
|
|
|
tools: list[dict],
|
|
tools: list[dict],
|
|
|
params: AgentParams,
|
|
params: AgentParams,
|
|
|
|
|
+ tool_choice: dict | None = None,
|
|
|
) -> AsyncIterator[StreamItem]:
|
|
) -> AsyncIterator[StreamItem]:
|
|
|
if tools:
|
|
if tools:
|
|
|
yield _event_tool_call_from_tools(tools, messages)
|
|
yield _event_tool_call_from_tools(tools, messages)
|
|
@@ -978,6 +1031,106 @@ async def test_openai_chat_client_rejects_unfinished_tool_calls_at_end():
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_openai_chat_client_rejects_reused_tool_call_id_across_transcript():
|
|
|
|
|
+ await _assert_tool_transcript_rejected(
|
|
|
|
|
+ [
|
|
|
|
|
+ ChatMessage(
|
|
|
|
|
+ role="assistant",
|
|
|
|
|
+ content="first round",
|
|
|
|
|
+ tool_calls=[
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_1",
|
|
|
|
|
+ name="mock_search",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
|
|
+ )
|
|
|
|
|
+ ],
|
|
|
|
|
+ ),
|
|
|
|
|
+ ChatMessage(role="tool", content="done", tool_call_id="call_1"),
|
|
|
|
|
+ ChatMessage(
|
|
|
|
|
+ role="assistant",
|
|
|
|
|
+ content="second round",
|
|
|
|
|
+ tool_calls=[
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id="call_1",
|
|
|
|
|
+ name="handoff_note",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
|
|
+ )
|
|
|
|
|
+ ],
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ "duplicate assistant tool-call ID across transcript: call_1",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+async def _assert_tool_choice_rejected(
|
|
|
|
|
+ tools: list[dict],
|
|
|
|
|
+ tool_choice: dict,
|
|
|
|
|
+ error_match: str,
|
|
|
|
|
+) -> None:
|
|
|
|
|
+ network_called = False
|
|
|
|
|
+
|
|
|
|
|
+ def handler(request: httpx.Request) -> httpx.Response:
|
|
|
|
|
+ nonlocal network_called
|
|
|
|
|
+ network_called = True
|
|
|
|
|
+ return httpx.Response(200)
|
|
|
|
|
+
|
|
|
|
|
+ async with httpx.AsyncClient(
|
|
|
|
|
+ transport=httpx.MockTransport(handler),
|
|
|
|
|
+ base_url="https://llm.test/v1",
|
|
|
|
|
+ ) as http_client:
|
|
|
|
|
+ client = OpenAICompatibleChatClient(
|
|
|
|
|
+ api_key="",
|
|
|
|
|
+ base_url="https://llm.test/v1",
|
|
|
|
|
+ default_model="provider-default",
|
|
|
|
|
+ request_timeout_seconds=5,
|
|
|
|
|
+ http_client=http_client,
|
|
|
|
|
+ )
|
|
|
|
|
+ with pytest.raises(ValueError, match=error_match):
|
|
|
|
|
+ [
|
|
|
|
|
+ item
|
|
|
|
|
+ async for item in client.stream_chat(
|
|
|
|
|
+ messages=[ChatMessage(role="user", content="use a tool")],
|
|
|
|
|
+ tools=tools,
|
|
|
|
|
+ params=AgentParams(model="provider-default"),
|
|
|
|
|
+ tool_choice=tool_choice,
|
|
|
|
|
+ )
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ assert network_called is False
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_openai_chat_client_rejects_forced_choice_without_tools():
|
|
|
|
|
+ await _assert_tool_choice_rejected(
|
|
|
|
|
+ tools=[],
|
|
|
|
|
+ tool_choice={
|
|
|
|
|
+ "type": "function",
|
|
|
|
|
+ "function": {"name": "handoff_note"},
|
|
|
|
|
+ },
|
|
|
|
|
+ error_match="forced tool choice requires tools",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_openai_chat_client_rejects_forced_choice_for_unknown_tool():
|
|
|
|
|
+ await _assert_tool_choice_rejected(
|
|
|
|
|
+ tools=[
|
|
|
|
|
+ {
|
|
|
|
|
+ "type": "function",
|
|
|
|
|
+ "function": {"name": "mock_search", "parameters": {}},
|
|
|
|
|
+ }
|
|
|
|
|
+ ],
|
|
|
|
|
+ tool_choice={
|
|
|
|
|
+ "type": "function",
|
|
|
|
|
+ "function": {"name": "handoff_note"},
|
|
|
|
|
+ },
|
|
|
|
|
+ error_match="forced tool choice references unknown tool: handoff_note",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def test_static_pre_message_role_selector_does_not_offer_tool():
|
|
def test_static_pre_message_role_selector_does_not_offer_tool():
|
|
|
html = Path("src/agent_lab/presentation/static/index.html").read_text()
|
|
html = Path("src/agent_lab/presentation/static/index.html").read_text()
|
|
|
|
|
|
|
@@ -1387,3 +1540,58 @@ async def test_openai_chat_client_merges_agent_extra_body_into_payload():
|
|
|
assert captured_payloads[0]["thinking"] == {"type": "disabled"}
|
|
assert captured_payloads[0]["thinking"] == {"type": "disabled"}
|
|
|
assert captured_payloads[0]["enable_search"] is False
|
|
assert captured_payloads[0]["enable_search"] is False
|
|
|
assert captured_payloads[0]["search_options"] == {"forced_search": False}
|
|
assert captured_payloads[0]["search_options"] == {"forced_search": False}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+@pytest.mark.parametrize(
|
|
|
|
|
+ ("reserved_key", "override_value"),
|
|
|
|
|
+ [
|
|
|
|
|
+ ("model", "overridden-model"),
|
|
|
|
|
+ ("messages", []),
|
|
|
|
|
+ ("tools", []),
|
|
|
|
|
+ (
|
|
|
|
|
+ "tool_choice",
|
|
|
|
|
+ {"type": "function", "function": {"name": "other_tool"}},
|
|
|
|
|
+ ),
|
|
|
|
|
+ ("stream", False),
|
|
|
|
|
+ ("stream_options", {"include_usage": False}),
|
|
|
|
|
+ ("temperature", 1.0),
|
|
|
|
|
+ ("max_tokens", 1),
|
|
|
|
|
+ ],
|
|
|
|
|
+)
|
|
|
|
|
+async def test_openai_chat_client_rejects_reserved_extra_body_fields_before_network(
|
|
|
|
|
+ reserved_key: str,
|
|
|
|
|
+ override_value: object,
|
|
|
|
|
+):
|
|
|
|
|
+ network_called = False
|
|
|
|
|
+
|
|
|
|
|
+ def handler(request: httpx.Request) -> httpx.Response:
|
|
|
|
|
+ nonlocal network_called
|
|
|
|
|
+ network_called = True
|
|
|
|
|
+ return httpx.Response(200)
|
|
|
|
|
+
|
|
|
|
|
+ async with httpx.AsyncClient(
|
|
|
|
|
+ transport=httpx.MockTransport(handler),
|
|
|
|
|
+ base_url="https://llm.test/v1",
|
|
|
|
|
+ ) as http_client:
|
|
|
|
|
+ client = OpenAICompatibleChatClient(
|
|
|
|
|
+ api_key="",
|
|
|
|
|
+ base_url="https://llm.test/v1",
|
|
|
|
|
+ default_model="provider-default",
|
|
|
|
|
+ request_timeout_seconds=5,
|
|
|
|
|
+ http_client=http_client,
|
|
|
|
|
+ )
|
|
|
|
|
+ with pytest.raises(
|
|
|
|
|
+ ValueError,
|
|
|
|
|
+ match=f"extra_body contains reserved field: {reserved_key}",
|
|
|
|
|
+ ):
|
|
|
|
|
+ [
|
|
|
|
|
+ item
|
|
|
|
|
+ async for item in client.stream_chat(
|
|
|
|
|
+ messages=[ChatMessage(role="user", content="hi")],
|
|
|
|
|
+ tools=[],
|
|
|
|
|
+ params=AgentParams(extra_body={reserved_key: override_value}),
|
|
|
|
|
+ )
|
|
|
|
|
+ ]
|
|
|
|
|
+
|
|
|
|
|
+ assert network_called is False
|