test_websocket_api.py 102 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081
  1. import json
  2. import asyncio
  3. import logging
  4. import subprocess
  5. from collections.abc import AsyncIterator
  6. from html.parser import HTMLParser
  7. from pathlib import Path
  8. import httpx
  9. import pytest
  10. from fastapi.testclient import TestClient
  11. from pydantic import ValidationError
  12. from agent_lab.application.contracts import AgentParams, DebugRunRequest, EventAgentParams
  13. from agent_lab.application.queues import RuntimeQueues
  14. from agent_lab.application.runtime import DebugRuntime
  15. from agent_lab.domain.events import ToolCallEvent
  16. from agent_lab.domain.messages import ChatMessage, StreamItem, TokenUsage
  17. from agent_lab.infrastructure.chat_client import OpenAICompatibleChatClient
  18. from agent_lab.infrastructure.sqlite_store import SQLiteSessionStore
  19. from agent_lab.presentation.web import create_app
  20. from agent_lab.settings import Settings
  21. def _css_rule(css: str, selector: str) -> str:
  22. start = css.index(f"{selector} {{")
  23. end = css.index("}", start)
  24. return css[start:end]
  25. class _HtmlNode:
  26. def __init__(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
  27. self.tag = tag
  28. self.attrs = dict(attrs)
  29. self.children: list["_HtmlNode"] = []
  30. self.text_parts: list[str] = []
  31. @property
  32. def classes(self) -> set[str]:
  33. return set(self.attrs.get("class", "").split())
  34. def descendants(self) -> list["_HtmlNode"]:
  35. nodes: list["_HtmlNode"] = []
  36. for child in self.children:
  37. nodes.append(child)
  38. nodes.extend(child.descendants())
  39. return nodes
  40. def text_content(self) -> str:
  41. return "".join(
  42. [*self.text_parts, *(child.text_content() for child in self.children)]
  43. )
  44. class _HtmlTreeParser(HTMLParser):
  45. _void_tags = {
  46. "area",
  47. "base",
  48. "br",
  49. "col",
  50. "embed",
  51. "hr",
  52. "img",
  53. "input",
  54. "link",
  55. "meta",
  56. "param",
  57. "source",
  58. "track",
  59. "wbr",
  60. }
  61. def __init__(self) -> None:
  62. super().__init__()
  63. self.root = _HtmlNode("document", [])
  64. self.stack = [self.root]
  65. def handle_starttag(
  66. self,
  67. tag: str,
  68. attrs: list[tuple[str, str | None]],
  69. ) -> None:
  70. node = _HtmlNode(tag, attrs)
  71. self.stack[-1].children.append(node)
  72. if tag not in self._void_tags:
  73. self.stack.append(node)
  74. def handle_startendtag(
  75. self,
  76. tag: str,
  77. attrs: list[tuple[str, str | None]],
  78. ) -> None:
  79. self.stack[-1].children.append(_HtmlNode(tag, attrs))
  80. def handle_endtag(self, tag: str) -> None:
  81. for index in range(len(self.stack) - 1, 0, -1):
  82. if self.stack[index].tag == tag:
  83. del self.stack[index:]
  84. return
  85. def handle_data(self, data: str) -> None:
  86. self.stack[-1].text_parts.append(data)
  87. def _parse_html(html: str) -> _HtmlNode:
  88. parser = _HtmlTreeParser()
  89. parser.feed(html)
  90. return parser.root
  91. def _html_node_by_id(root: _HtmlNode, element_id: str) -> _HtmlNode:
  92. node = next(
  93. (
  94. node
  95. for node in [root, *root.descendants()]
  96. if node.attrs.get("id") == element_id
  97. ),
  98. None,
  99. )
  100. assert node is not None, f"missing HTML element #{element_id}"
  101. return node
  102. def _run_node_json(source: str) -> object:
  103. completed = subprocess.run(
  104. ["node", "--input-type=module", "-e", source],
  105. capture_output=True,
  106. text=True,
  107. check=False,
  108. )
  109. assert completed.returncode == 0, completed.stderr
  110. return json.loads(completed.stdout)
  111. def _console_ui_source(js: str) -> str:
  112. assert 'const CONSOLE_UI_STORAGE_KEY = "agent-lab-console-ui-v1";' in js
  113. for function_name in [
  114. "readConsoleUiState",
  115. "writeConsoleUiState",
  116. "setControlSectionExpanded",
  117. "setDebugDrawerOpen",
  118. "initializeConsoleUi",
  119. "updateTopbarStatus",
  120. ]:
  121. assert f"function {function_name}(" in js
  122. return js[
  123. js.index("function readConsoleUiState(") :
  124. js.index("function openChatConfig(")
  125. ]
  126. def _run_load_session_replay_scenario(scenario: str) -> object:
  127. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  128. load_replay = js[
  129. js.index("async function loadSessionReplay(") :
  130. js.index("function setSessionStatus(")
  131. ]
  132. harness = """
  133. let currentSessionId = null;
  134. let sessionReplayLoadToken = 0;
  135. const sessionTitle = { value: "" };
  136. const sessionList = { value: "" };
  137. const mutations = {
  138. workspace: [],
  139. transcript: [],
  140. audit: [],
  141. usage: [],
  142. status: [],
  143. };
  144. const pending = new Map();
  145. function fetchJson(url) {
  146. let resolve;
  147. let reject;
  148. const promise = new Promise((resolvePromise, rejectPromise) => {
  149. resolve = resolvePromise;
  150. reject = rejectPromise;
  151. });
  152. pending.set(url, { resolve, reject });
  153. return promise;
  154. }
  155. function resolveReplay(sessionId) {
  156. pending.get(`/api/sessions/${sessionId}`).resolve({
  157. title: `${sessionId} title`,
  158. config: { marker: sessionId },
  159. });
  160. pending.get(`/api/sessions/${sessionId}/messages`).resolve([
  161. { content: sessionId },
  162. ]);
  163. pending.get(`/api/sessions/${sessionId}/audit`).resolve([
  164. { event: sessionId },
  165. ]);
  166. pending.get(`/api/sessions/${sessionId}/usage`).resolve({
  167. session: { marker: sessionId },
  168. });
  169. }
  170. function restoreWorkspaceSnapshot(config) {
  171. mutations.workspace.push(config.marker);
  172. }
  173. function renderReplayMessages(messages) {
  174. mutations.transcript.push(messages[0].content);
  175. }
  176. function renderAuditReplay(audit) {
  177. mutations.audit.push(audit[0].event);
  178. }
  179. function renderSessionUsage(usage) {
  180. mutations.usage.push(usage.session.marker);
  181. }
  182. function setSessionStatus(status) {
  183. mutations.status.push(status);
  184. }
  185. function isSocketOpen() {
  186. return false;
  187. }
  188. """
  189. return _run_node_json("\n".join([harness, load_replay, scenario]))
  190. class FakeRuntime:
  191. def __init__(self) -> None:
  192. self.requests: list[DebugRunRequest] = []
  193. async def run(self, request: DebugRunRequest) -> AsyncIterator[dict]:
  194. self.requests.append(request)
  195. yield {"type": "session_started"}
  196. yield {"type": "message_delta", "content": "hello"}
  197. yield {"type": "done"}
  198. class QueueAwareRuntime:
  199. def __init__(self) -> None:
  200. self.requests: list[DebugRunRequest] = []
  201. self.queues: RuntimeQueues | None = None
  202. self.task: asyncio.Task | None = None
  203. def start(self, request: DebugRunRequest) -> RuntimeQueues:
  204. self.requests.append(request)
  205. self.queues = RuntimeQueues()
  206. self.task = asyncio.create_task(self._run())
  207. return self.queues
  208. async def _run(self) -> None:
  209. assert self.queues is not None
  210. await self.queues.output.put({"type": "session_started"})
  211. message = await self.queues.input.get()
  212. await self.queues.output.put(
  213. {"type": "message_delta", "content": message.content}
  214. )
  215. await self.queues.output.put({"type": "done"})
  216. async def aclose(self) -> None:
  217. if self.task is not None and not self.task.done():
  218. self.task.cancel()
  219. await asyncio.gather(self.task, return_exceptions=True)
  220. class PersistentSessionRuntime:
  221. def __init__(self) -> None:
  222. self.requests: list[DebugRunRequest] = []
  223. self.queues: RuntimeQueues | None = None
  224. self.task: asyncio.Task | None = None
  225. def start_session(self, request: DebugRunRequest) -> RuntimeQueues:
  226. self.requests.append(request)
  227. self.queues = RuntimeQueues()
  228. self.task = asyncio.create_task(self._run(request))
  229. return self.queues
  230. async def _run(self, request: DebugRunRequest) -> None:
  231. assert self.queues is not None
  232. await self.queues.output.put({"type": "session_started"})
  233. await self._emit_turn(1, request.user_message)
  234. turn_index = 1
  235. while True:
  236. message = await self.queues.input.get()
  237. turn_index += 1
  238. await self._emit_turn(turn_index, message.content)
  239. async def _emit_turn(self, turn_index: int, content: str) -> None:
  240. assert self.queues is not None
  241. await self.queues.output.put({"type": "turn_started", "turn_index": turn_index})
  242. await self.queues.output.put({"type": "message_delta", "content": content})
  243. await self.queues.output.put(
  244. {"type": "turn_completed", "turn_index": turn_index}
  245. )
  246. async def aclose(self) -> None:
  247. if self.task is not None and not self.task.done():
  248. self.task.cancel()
  249. await asyncio.gather(self.task, return_exceptions=True)
  250. def _request_payload() -> dict:
  251. return {
  252. "user_message": "debug this",
  253. "system_prompts": ["You are a debugger."],
  254. "pre_messages": [{"role": "user", "content": "previous turn"}],
  255. "chat_agent": {
  256. "model": "fake-model",
  257. "temperature": 0.1,
  258. "max_tokens": 200,
  259. },
  260. "event_agent": {
  261. "enabled_tools": ["handoff_note"],
  262. "max_event_loops": 2,
  263. },
  264. }
  265. def _event_tool_call_from_tools(
  266. tools: list[dict],
  267. messages: list[ChatMessage],
  268. ) -> StreamItem:
  269. tool_name = tools[0]["function"]["name"]
  270. content = ""
  271. for role in ("assistant", "user"):
  272. content = next(
  273. (
  274. message.content
  275. for message in reversed(messages)
  276. if message.role == role and message.content.strip()
  277. ),
  278. "",
  279. )
  280. if content:
  281. break
  282. arguments = {
  283. "message": content,
  284. "query": content,
  285. "title": content,
  286. }
  287. return StreamItem.provider_tool_call(
  288. ToolCallEvent(
  289. id="event_agent_call_1",
  290. name=tool_name,
  291. arguments=arguments,
  292. raw_arguments=json.dumps(arguments),
  293. )
  294. )
  295. def test_agent_params_defaults_include_extra_body_and_event_agent_defaults_to_one_round():
  296. chat_params = AgentParams()
  297. event_params = EventAgentParams()
  298. assert chat_params.extra_body == {
  299. "thinking": {"type": "disabled"},
  300. "enable_search": False,
  301. "search_options": {"forced_search": False},
  302. }
  303. assert event_params.extra_body == chat_params.extra_body
  304. assert event_params.max_event_loops == 1
  305. assert event_params.max_parallel_events == 4
  306. assert event_params.batch_timeout_seconds == 15.0
  307. assert event_params.system_prompt == ""
  308. @pytest.mark.parametrize("value", [0, 17])
  309. def test_event_agent_params_rejects_out_of_range_parallelism(value: int):
  310. with pytest.raises(ValidationError, match="max_parallel_events"):
  311. EventAgentParams(max_parallel_events=value)
  312. @pytest.mark.parametrize("value", [0, -1, 121])
  313. def test_event_agent_params_rejects_invalid_batch_timeout(value: float):
  314. with pytest.raises(ValidationError, match="batch_timeout_seconds"):
  315. EventAgentParams(batch_timeout_seconds=value)
  316. def test_debug_run_request_defaults_to_dual_agent_tool_invocation_mode():
  317. request = DebugRunRequest.model_validate(_request_payload())
  318. assert request.tool_invocation_mode == "dual_agent"
  319. @pytest.mark.parametrize("mode", ["dual_agent", "chat_agent_tools"])
  320. def test_debug_run_request_accepts_supported_tool_invocation_modes(mode: str):
  321. payload = _request_payload()
  322. payload["tool_invocation_mode"] = mode
  323. request = DebugRunRequest.model_validate(payload)
  324. assert request.tool_invocation_mode == mode
  325. def test_debug_run_request_rejects_unknown_tool_invocation_mode():
  326. payload = _request_payload()
  327. payload["tool_invocation_mode"] = "unknown"
  328. with pytest.raises(ValidationError, match="tool_invocation_mode"):
  329. DebugRunRequest.model_validate(payload)
  330. def test_health_returns_ok():
  331. app = create_app(runtime_factory=FakeRuntime)
  332. client = TestClient(app)
  333. response = client.get("/health")
  334. assert response.status_code == 200
  335. assert response.json() == {"status": "ok"}
  336. def test_api_tools_returns_default_tool_catalog():
  337. app = create_app(runtime_factory=FakeRuntime)
  338. client = TestClient(app)
  339. response = client.get("/api/tools")
  340. assert response.status_code == 200
  341. tools = response.json()
  342. assert [tool["name"] for tool in tools] == [
  343. "handoff_note",
  344. "mock_search",
  345. "mock_ticket",
  346. "session.terminate",
  347. "device.volume.adjust",
  348. "calendar.schedule.create",
  349. "knowledge.web.search",
  350. ]
  351. assert tools[0]["parameters"]["required"] == ["message"]
  352. def test_session_api_creates_lists_and_returns_empty_replay_data(tmp_path):
  353. settings = Settings(database_path=str(tmp_path / "agent_lab.sqlite3"))
  354. app = create_app(settings=settings, runtime_factory=FakeRuntime)
  355. client = TestClient(app)
  356. created = client.post(
  357. "/api/sessions",
  358. json={
  359. "title": "Debug session",
  360. "config": {"chat_agent": {"model": "chat-model"}},
  361. },
  362. )
  363. assert created.status_code == 200
  364. session_id = created.json()["id"]
  365. assert created.json()["title"] == "Debug session"
  366. sessions = client.get("/api/sessions")
  367. assert sessions.status_code == 200
  368. assert sessions.json()[0]["id"] == session_id
  369. assert sessions.json()[0]["turn_count"] == 0
  370. detail = client.get(f"/api/sessions/{session_id}")
  371. assert detail.status_code == 200
  372. assert detail.json()["config"]["chat_agent"]["model"] == "chat-model"
  373. assert client.get(f"/api/sessions/{session_id}/messages").json() == []
  374. assert client.get(f"/api/sessions/{session_id}/audit").json() == []
  375. usage = client.get(f"/api/sessions/{session_id}/usage").json()
  376. assert usage == {
  377. "calls": [],
  378. "turns": [],
  379. "session": {
  380. "prompt_tokens": 0,
  381. "completion_tokens": 0,
  382. "total_tokens": 0,
  383. "cached_tokens": 0,
  384. "elapsed_ms": 0,
  385. "call_count": 0,
  386. "fallback_count": 0,
  387. "tool_count": 0,
  388. "turn_wall_time_ms": None,
  389. },
  390. }
  391. def test_session_api_returns_persisted_replay_data(tmp_path):
  392. database_path = tmp_path / "agent_lab.sqlite3"
  393. settings = Settings(database_path=str(database_path))
  394. app = create_app(settings=settings, runtime_factory=FakeRuntime)
  395. client = TestClient(app)
  396. session_id = client.post("/api/sessions", json={"title": "Replay"}).json()["id"]
  397. store = SQLiteSessionStore(database_path)
  398. store.start_turn(session_id, turn_index=1, user_message="debug this")
  399. store.append_message(
  400. session_id,
  401. turn_index=1,
  402. message=ChatMessage(role="user", content="debug this"),
  403. )
  404. store.append_message(
  405. session_id,
  406. turn_index=1,
  407. message=ChatMessage(
  408. role="assistant",
  409. content="answer",
  410. tool_calls=[
  411. ToolCallEvent(
  412. id="call-1",
  413. name="mock_search",
  414. arguments={"query": "docs"},
  415. raw_arguments='{"query":"docs"}',
  416. )
  417. ],
  418. ),
  419. )
  420. store.append_message(
  421. session_id,
  422. turn_index=1,
  423. message=ChatMessage(
  424. role="tool",
  425. content='{"results":[]}',
  426. name="mock_search",
  427. tool_call_id="call-1",
  428. ),
  429. )
  430. store.append_audit(
  431. session_id,
  432. event="chat_agent_request",
  433. details={"model": "chat-model"},
  434. turn_index=1,
  435. round_index=1,
  436. )
  437. store.append_usage(
  438. session_id,
  439. turn_index=1,
  440. round_index=1,
  441. usage=TokenUsage(prompt_tokens=1, completion_tokens=2, total_tokens=3),
  442. ttft_ms=10,
  443. elapsed_ms=20,
  444. )
  445. messages = client.get(f"/api/sessions/{session_id}/messages").json()
  446. assert [message["content"] for message in messages] == [
  447. "debug this",
  448. "answer",
  449. '{"results":[]}',
  450. ]
  451. assert messages[0]["tool_calls"] == []
  452. assert messages[1]["tool_calls"] == [
  453. {
  454. "id": "call-1",
  455. "name": "mock_search",
  456. "arguments": {"query": "docs"},
  457. "raw_arguments": '{"query":"docs"}',
  458. }
  459. ]
  460. assert messages[2]["tool_call_id"] == "call-1"
  461. assert messages[2]["tool_calls"] == []
  462. assert client.get(f"/api/sessions/{session_id}/audit").json()[0]["details"] == {
  463. "model": "chat-model"
  464. }
  465. usage = client.get(f"/api/sessions/{session_id}/usage").json()
  466. assert usage["calls"][0]["total_tokens"] == 3
  467. assert usage["turns"][0]["total_tokens"] == 3
  468. assert usage["session"]["total_tokens"] == 3
  469. def test_default_websocket_runtime_factory_accepts_session_store(tmp_path):
  470. settings = Settings(database_path=str(tmp_path / "agent_lab.sqlite3"))
  471. app = create_app(settings=settings)
  472. client = TestClient(app)
  473. with client.websocket_connect("/ws/debug") as websocket:
  474. websocket.send_json({"chat_agent": {"model": "fake-model"}})
  475. message = websocket.receive_json()
  476. assert message["type"] == "error"
  477. assert "user_message" in message["message"]
  478. def test_websocket_debug_streams_runtime_messages():
  479. runtime = FakeRuntime()
  480. app = create_app(runtime_factory=lambda: runtime)
  481. client = TestClient(app)
  482. with client.websocket_connect("/ws/debug") as websocket:
  483. websocket.send_json(_request_payload())
  484. assert websocket.receive_json() == {"type": "session_started"}
  485. assert websocket.receive_json() == {
  486. "type": "message_delta",
  487. "content": "hello",
  488. }
  489. assert websocket.receive_json() == {"type": "done"}
  490. assert runtime.requests[0].user_message == "debug this"
  491. assert runtime.requests[0].pre_messages[0].content == "previous turn"
  492. def test_websocket_debug_logs_session_lifecycle(caplog):
  493. runtime = FakeRuntime()
  494. app = create_app(runtime_factory=lambda: runtime)
  495. client = TestClient(app)
  496. with caplog.at_level(logging.INFO, logger="agent_lab.presentation.web"):
  497. with client.websocket_connect("/ws/debug") as websocket:
  498. websocket.send_json(_request_payload())
  499. assert websocket.receive_json() == {"type": "session_started"}
  500. assert websocket.receive_json() == {
  501. "type": "message_delta",
  502. "content": "hello",
  503. }
  504. assert websocket.receive_json() == {"type": "done"}
  505. assert "websocket session accepted" in caplog.text
  506. assert "websocket request accepted" in caplog.text
  507. assert "websocket session closed" in caplog.text
  508. def test_websocket_debug_enqueues_user_messages_during_running_session():
  509. runtime = QueueAwareRuntime()
  510. app = create_app(runtime_factory=lambda: runtime)
  511. client = TestClient(app)
  512. with client.websocket_connect("/ws/debug") as websocket:
  513. websocket.send_json(_request_payload())
  514. assert websocket.receive_json() == {"type": "session_started"}
  515. websocket.send_json({"type": "user_message", "content": "follow-up"})
  516. assert websocket.receive_json() == {
  517. "type": "message_delta",
  518. "content": "follow-up",
  519. }
  520. assert websocket.receive_json() == {"type": "done"}
  521. assert runtime.requests[0].user_message == "debug this"
  522. def test_websocket_debug_keeps_session_open_for_multiple_turns():
  523. runtime = PersistentSessionRuntime()
  524. app = create_app(runtime_factory=lambda: runtime)
  525. client = TestClient(app)
  526. with client.websocket_connect("/ws/debug") as websocket:
  527. websocket.send_json(_request_payload())
  528. assert websocket.receive_json() == {"type": "session_started"}
  529. assert websocket.receive_json() == {"type": "turn_started", "turn_index": 1}
  530. assert websocket.receive_json() == {
  531. "type": "message_delta",
  532. "content": "debug this",
  533. }
  534. assert websocket.receive_json() == {"type": "turn_completed", "turn_index": 1}
  535. websocket.send_json({"type": "user_message", "content": "follow-up"})
  536. assert websocket.receive_json() == {"type": "turn_started", "turn_index": 2}
  537. assert websocket.receive_json() == {
  538. "type": "message_delta",
  539. "content": "follow-up",
  540. }
  541. assert websocket.receive_json() == {"type": "turn_completed", "turn_index": 2}
  542. websocket.close()
  543. assert runtime.requests[0].user_message == "debug this"
  544. def test_websocket_debug_sends_error_for_invalid_request():
  545. app = create_app(runtime_factory=FakeRuntime)
  546. client = TestClient(app)
  547. with client.websocket_connect("/ws/debug") as websocket:
  548. websocket.send_json({"chat_agent": {"model": "fake-model"}})
  549. message = websocket.receive_json()
  550. assert message["type"] == "error"
  551. assert "user_message" in message["message"]
  552. def test_debug_run_request_rejects_invalid_pre_message_role():
  553. payload = _request_payload()
  554. payload["pre_messages"] = [{"role": "developer", "content": "invalid"}]
  555. with pytest.raises(ValidationError) as exc_info:
  556. DebugRunRequest.model_validate(payload)
  557. assert "role" in str(exc_info.value)
  558. def test_debug_run_request_rejects_tool_pre_messages():
  559. payload = _request_payload()
  560. payload["pre_messages"] = [
  561. {
  562. "role": "tool",
  563. "content": "orphan result",
  564. "tool_call_id": "call_1",
  565. }
  566. ]
  567. with pytest.raises(ValidationError, match="pre_messages cannot include tool messages"):
  568. DebugRunRequest.model_validate(payload)
  569. def test_debug_run_request_rejects_assistant_tool_calls_in_pre_messages():
  570. payload = _request_payload()
  571. payload["pre_messages"] = [
  572. {
  573. "role": "assistant",
  574. "content": "I will inspect this.",
  575. "tool_calls": [
  576. {
  577. "id": "call_1",
  578. "name": "mock_search",
  579. "arguments": {"query": "latency"},
  580. "raw_arguments": '{"query":"latency"}',
  581. }
  582. ],
  583. }
  584. ]
  585. with pytest.raises(
  586. ValidationError,
  587. match="pre_messages cannot include assistant tool calls",
  588. ):
  589. DebugRunRequest.model_validate(payload)
  590. def test_chat_message_allows_internal_tool_replies():
  591. message = ChatMessage(
  592. role="tool",
  593. content='{"message":"handled"}',
  594. name="handoff_note",
  595. tool_call_id="call_1",
  596. )
  597. assert message.role == "tool"
  598. assert message.tool_call_id == "call_1"
  599. def test_chat_message_preserves_assistant_tool_calls():
  600. message = ChatMessage(
  601. role="assistant",
  602. content="I will inspect both sources.",
  603. tool_calls=[
  604. ToolCallEvent(
  605. id="call_1",
  606. name="mock_search",
  607. arguments={"query": "latency docs"},
  608. raw_arguments='{"query":"latency docs"}',
  609. ),
  610. ToolCallEvent(
  611. id="call_2",
  612. name="handoff_note",
  613. arguments={"message": "inspect provider behavior"},
  614. raw_arguments='{"message":"inspect provider behavior"}',
  615. ),
  616. ],
  617. )
  618. assert message.model_dump()["tool_calls"] == [
  619. {
  620. "id": "call_1",
  621. "name": "mock_search",
  622. "arguments": {"query": "latency docs"},
  623. "raw_arguments": '{"query":"latency docs"}',
  624. },
  625. {
  626. "id": "call_2",
  627. "name": "handoff_note",
  628. "arguments": {"message": "inspect provider behavior"},
  629. "raw_arguments": '{"message":"inspect provider behavior"}',
  630. },
  631. ]
  632. def test_chat_message_rejects_tool_calls_on_non_assistant_role():
  633. with pytest.raises(ValidationError, match="tool_calls require assistant role"):
  634. ChatMessage(
  635. role="user",
  636. content="invalid",
  637. tool_calls=[
  638. ToolCallEvent(
  639. id="call_1",
  640. name="mock_search",
  641. arguments={},
  642. raw_arguments="{}",
  643. )
  644. ],
  645. )
  646. def test_chat_message_rejects_tool_role_without_tool_call_id():
  647. with pytest.raises(ValidationError, match="tool messages require tool_call_id"):
  648. ChatMessage(role="tool", content="orphan result")
  649. @pytest.mark.parametrize("role", ["system", "user", "assistant"])
  650. def test_chat_message_rejects_tool_call_id_on_non_tool_roles(role: str):
  651. with pytest.raises(ValidationError, match="tool_call_id requires tool role"):
  652. ChatMessage(role=role, content="invalid", tool_call_id="call_1")
  653. def test_chat_message_rejects_duplicate_assistant_tool_call_ids():
  654. with pytest.raises(ValidationError, match="duplicate assistant tool-call ID: call_1"):
  655. ChatMessage(
  656. role="assistant",
  657. content="checking twice",
  658. tool_calls=[
  659. ToolCallEvent(
  660. id="call_1",
  661. name="mock_search",
  662. arguments={"query": "first"},
  663. raw_arguments='{"query":"first"}',
  664. ),
  665. ToolCallEvent(
  666. id="call_1",
  667. name="handoff_note",
  668. arguments={"message": "second"},
  669. raw_arguments='{"message":"second"}',
  670. ),
  671. ],
  672. )
  673. class HistoryCapturingChatClient:
  674. def __init__(self) -> None:
  675. self.calls = 0
  676. self.second_call_messages: list[ChatMessage] = []
  677. async def stream_chat(
  678. self,
  679. messages: list[ChatMessage],
  680. tools: list[dict],
  681. params: AgentParams,
  682. tool_choice: dict | None = None,
  683. ) -> AsyncIterator[StreamItem]:
  684. if tools:
  685. yield _event_tool_call_from_tools(tools, messages)
  686. return
  687. self.calls += 1
  688. if self.calls == 1:
  689. yield StreamItem.message_delta("Need event help.")
  690. yield StreamItem.text_event(
  691. ToolCallEvent(
  692. id="call_1",
  693. name="handoff_note",
  694. arguments={},
  695. raw_arguments="{}",
  696. )
  697. )
  698. return
  699. self.second_call_messages = list(messages)
  700. yield StreamItem.message_delta("Final answer.")
  701. @pytest.mark.asyncio
  702. async def test_runtime_appends_event_summary_without_tool_reply_history():
  703. request = DebugRunRequest(
  704. user_message="debug this",
  705. system_prompts=["You are a debugger."],
  706. pre_messages=[],
  707. chat_agent=AgentParams(model="fake-model", temperature=0.1, max_tokens=200),
  708. event_agent=EventAgentParams(enabled_tools=["handoff_note"], max_event_loops=2),
  709. )
  710. client = HistoryCapturingChatClient()
  711. runtime = DebugRuntime(client)
  712. outputs = [message async for message in runtime.run(request)]
  713. assert client.calls == 2
  714. assert [message.role for message in client.second_call_messages] == [
  715. "system",
  716. "system",
  717. "user",
  718. "assistant",
  719. "user",
  720. ]
  721. assert "Available events:" in client.second_call_messages[1].content
  722. assert client.second_call_messages[3].content == "Need event help."
  723. assert not any(message.role == "tool" for message in client.second_call_messages)
  724. assert client.second_call_messages[4].content.startswith("EventAgent results:\n")
  725. assert client.second_call_messages[4].name == "event_agent"
  726. assert outputs[-1] == {"type": "done"}
  727. @pytest.mark.asyncio
  728. async def test_openai_chat_client_streams_sse_chunks_through_parser():
  729. requests: list[httpx.Request] = []
  730. def handler(request: httpx.Request) -> httpx.Response:
  731. requests.append(request)
  732. payload = json.loads(request.content)
  733. assert payload["stream"] is True
  734. assert payload["model"] == "model-x"
  735. assert payload["messages"] == [{"role": "user", "content": "hi"}]
  736. assert payload["tools"][0]["function"]["name"] == "handoff_note"
  737. assert "tool_choice" not in payload
  738. assert payload["temperature"] == 0.3
  739. assert payload["max_tokens"] == 50
  740. assert request.headers["authorization"] == "Bearer test-key"
  741. return httpx.Response(
  742. 200,
  743. content=(
  744. b'data: {"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}\n\n'
  745. b"data: [DONE]\n\n"
  746. ),
  747. )
  748. async with httpx.AsyncClient(
  749. transport=httpx.MockTransport(handler),
  750. base_url="https://llm.test/v1",
  751. ) as http_client:
  752. client = OpenAICompatibleChatClient(
  753. api_key="test-key",
  754. base_url="https://llm.test/v1",
  755. default_model="default-model",
  756. request_timeout_seconds=5,
  757. http_client=http_client,
  758. )
  759. items = [
  760. item
  761. async for item in client.stream_chat(
  762. messages=[ChatMessage(role="user", content="hi")],
  763. tools=[
  764. {
  765. "type": "function",
  766. "function": {"name": "handoff_note", "parameters": {}},
  767. }
  768. ],
  769. params=AgentParams(model="model-x", temperature=0.3, max_tokens=50),
  770. )
  771. ]
  772. assert requests[0].url.path == "/v1/chat/completions"
  773. assert [item.content for item in items if item.kind == "message_delta"] == ["hi"]
  774. assert [item.raw_chunk for item in items if item.kind == "raw_chunk"] == [
  775. {"choices": [{"delta": {"content": "hi"}, "finish_reason": None}]}
  776. ]
  777. @pytest.mark.asyncio
  778. async def test_openai_chat_client_serializes_explicit_tool_choice():
  779. captured_payloads: list[dict] = []
  780. forced_choice = {
  781. "type": "function",
  782. "function": {"name": "handoff_note"},
  783. }
  784. def handler(request: httpx.Request) -> httpx.Response:
  785. captured_payloads.append(json.loads(request.content))
  786. return httpx.Response(200, content=b"data: [DONE]\n\n")
  787. async with httpx.AsyncClient(
  788. transport=httpx.MockTransport(handler),
  789. base_url="https://llm.test/v1",
  790. ) as http_client:
  791. client = OpenAICompatibleChatClient(
  792. api_key="",
  793. base_url="https://llm.test/v1",
  794. default_model="provider-default",
  795. request_timeout_seconds=5,
  796. http_client=http_client,
  797. )
  798. [
  799. item
  800. async for item in client.stream_chat(
  801. messages=[ChatMessage(role="user", content="prepare handoff")],
  802. tools=[
  803. {
  804. "type": "function",
  805. "function": {"name": "handoff_note", "parameters": {}},
  806. }
  807. ],
  808. params=AgentParams(model="provider-default"),
  809. tool_choice=forced_choice,
  810. )
  811. ]
  812. assert captured_payloads[0]["tool_choice"] == forced_choice
  813. @pytest.mark.asyncio
  814. async def test_openai_chat_client_flushes_one_provider_tool_call_at_done():
  815. def handler(request: httpx.Request) -> httpx.Response:
  816. return httpx.Response(
  817. 200,
  818. content=(
  819. b'data: {"choices":[{"delta":{"tool_calls":[{"index":0,'
  820. b'"id":"call_1","function":{"name":"mock_search",'
  821. b'"arguments":"{\\"query\\":\\""}}]},"finish_reason":null}]}\n\n'
  822. b'data: {"choices":[{"delta":{"tool_calls":[{"index":0,'
  823. b'"function":{"arguments":"latency docs\\"}"}}]},'
  824. b'"finish_reason":null}]}\n\n'
  825. b"data: [DONE]\n\n"
  826. ),
  827. )
  828. async with httpx.AsyncClient(
  829. transport=httpx.MockTransport(handler),
  830. base_url="https://llm.test/v1",
  831. ) as http_client:
  832. client = OpenAICompatibleChatClient(
  833. api_key="",
  834. base_url="https://llm.test/v1",
  835. default_model="provider-default",
  836. request_timeout_seconds=5,
  837. http_client=http_client,
  838. )
  839. items = [
  840. item
  841. async for item in client.stream_chat(
  842. messages=[ChatMessage(role="user", content="find docs")],
  843. tools=[
  844. {
  845. "type": "function",
  846. "function": {"name": "mock_search", "parameters": {}},
  847. }
  848. ],
  849. params=AgentParams(model="provider-default"),
  850. )
  851. ]
  852. provider_calls = [
  853. item.event for item in items if item.kind == "provider_tool_call"
  854. ]
  855. assert provider_calls == [
  856. ToolCallEvent(
  857. id="call_1",
  858. name="mock_search",
  859. arguments={"query": "latency docs"},
  860. raw_arguments='{"query":"latency docs"}',
  861. )
  862. ]
  863. @pytest.mark.asyncio
  864. async def test_openai_chat_client_uses_default_model_when_request_model_is_blank():
  865. captured_payloads: list[dict] = []
  866. def handler(request: httpx.Request) -> httpx.Response:
  867. captured_payloads.append(json.loads(request.content))
  868. return httpx.Response(
  869. 200,
  870. content=b'data: {"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}\n\n',
  871. )
  872. async with httpx.AsyncClient(
  873. transport=httpx.MockTransport(handler),
  874. base_url="https://llm.test/v1",
  875. ) as http_client:
  876. client = OpenAICompatibleChatClient(
  877. api_key="",
  878. base_url="https://llm.test/v1",
  879. default_model="provider-default",
  880. request_timeout_seconds=5,
  881. http_client=http_client,
  882. )
  883. [
  884. item
  885. async for item in client.stream_chat(
  886. messages=[ChatMessage(role="user", content="hi")],
  887. tools=[],
  888. params=AgentParams(model=" ", temperature=0.3, max_tokens=50),
  889. )
  890. ]
  891. assert captured_payloads[0]["model"] == "provider-default"
  892. @pytest.mark.asyncio
  893. async def test_openai_chat_client_serializes_provider_tool_transcript():
  894. captured_payloads: list[dict] = []
  895. def handler(request: httpx.Request) -> httpx.Response:
  896. captured_payloads.append(json.loads(request.content))
  897. return httpx.Response(
  898. 200,
  899. content=b'data: {"choices":[{"delta":{"content":"done"},"finish_reason":null}]}\n\n',
  900. )
  901. async with httpx.AsyncClient(
  902. transport=httpx.MockTransport(handler),
  903. base_url="https://llm.test/v1",
  904. ) as http_client:
  905. client = OpenAICompatibleChatClient(
  906. api_key="",
  907. base_url="https://llm.test/v1",
  908. default_model="provider-default",
  909. request_timeout_seconds=5,
  910. http_client=http_client,
  911. )
  912. [
  913. item
  914. async for item in client.stream_chat(
  915. messages=[
  916. ChatMessage(role="user", content="inspect both"),
  917. ChatMessage(
  918. role="assistant",
  919. content="I will inspect both sources.",
  920. tool_calls=[
  921. ToolCallEvent(
  922. id="call_1",
  923. name="mock_search",
  924. arguments={"query": "latency docs"},
  925. raw_arguments='{"query":"latency docs"}',
  926. ),
  927. ToolCallEvent(
  928. id="call_2",
  929. name="handoff_note",
  930. arguments={"message": "inspect provider behavior"},
  931. raw_arguments='{"message":"inspect provider behavior"}',
  932. ),
  933. ],
  934. ),
  935. ChatMessage(
  936. role="tool",
  937. content='{"results":[]}',
  938. name="mock_search",
  939. tool_call_id="call_1",
  940. ),
  941. ChatMessage(
  942. role="tool",
  943. content='{"message":"handled"}',
  944. name="handoff_note",
  945. tool_call_id="call_2",
  946. ),
  947. ChatMessage(role="user", content="continue"),
  948. ],
  949. tools=[],
  950. params=AgentParams(
  951. model="provider-default",
  952. temperature=0.3,
  953. max_tokens=50,
  954. ),
  955. )
  956. ]
  957. assert captured_payloads[0]["messages"] == [
  958. {"role": "user", "content": "inspect both"},
  959. {
  960. "role": "assistant",
  961. "content": "I will inspect both sources.",
  962. "tool_calls": [
  963. {
  964. "id": "call_1",
  965. "type": "function",
  966. "function": {
  967. "name": "mock_search",
  968. "arguments": '{"query":"latency docs"}',
  969. },
  970. },
  971. {
  972. "id": "call_2",
  973. "type": "function",
  974. "function": {
  975. "name": "handoff_note",
  976. "arguments": '{"message":"inspect provider behavior"}',
  977. },
  978. },
  979. ],
  980. },
  981. {
  982. "role": "tool",
  983. "content": '{"results":[]}',
  984. "tool_call_id": "call_1",
  985. },
  986. {
  987. "role": "tool",
  988. "content": '{"message":"handled"}',
  989. "tool_call_id": "call_2",
  990. },
  991. {"role": "user", "content": "continue"},
  992. ]
  993. async def _assert_tool_transcript_rejected(
  994. messages: list[ChatMessage],
  995. error_match: str,
  996. ) -> None:
  997. network_called = False
  998. def handler(request: httpx.Request) -> httpx.Response:
  999. nonlocal network_called
  1000. network_called = True
  1001. return httpx.Response(200)
  1002. async with httpx.AsyncClient(
  1003. transport=httpx.MockTransport(handler),
  1004. base_url="https://llm.test/v1",
  1005. ) as http_client:
  1006. client = OpenAICompatibleChatClient(
  1007. api_key="",
  1008. base_url="https://llm.test/v1",
  1009. default_model="provider-default",
  1010. request_timeout_seconds=5,
  1011. http_client=http_client,
  1012. )
  1013. with pytest.raises(ValueError, match=error_match):
  1014. [
  1015. item
  1016. async for item in client.stream_chat(
  1017. messages=messages,
  1018. tools=[],
  1019. params=AgentParams(model="provider-default"),
  1020. )
  1021. ]
  1022. assert network_called is False
  1023. @pytest.mark.asyncio
  1024. async def test_openai_chat_client_rejects_orphan_tool_reply_before_network():
  1025. await _assert_tool_transcript_rejected(
  1026. [
  1027. ChatMessage(
  1028. role="tool",
  1029. content="orphan",
  1030. tool_call_id="call_1",
  1031. )
  1032. ],
  1033. "orphan tool reply: call_1",
  1034. )
  1035. @pytest.mark.asyncio
  1036. async def test_openai_chat_client_rejects_mismatched_tool_call_id_before_network():
  1037. await _assert_tool_transcript_rejected(
  1038. [
  1039. ChatMessage(
  1040. role="assistant",
  1041. content="",
  1042. tool_calls=[
  1043. ToolCallEvent(
  1044. id="call_1",
  1045. name="mock_search",
  1046. arguments={},
  1047. raw_arguments="{}",
  1048. )
  1049. ],
  1050. ),
  1051. ChatMessage(role="tool", content="wrong", tool_call_id="call_2"),
  1052. ],
  1053. "mismatched tool_call_id: call_2",
  1054. )
  1055. @pytest.mark.asyncio
  1056. async def test_openai_chat_client_rejects_duplicate_tool_reply_before_network():
  1057. await _assert_tool_transcript_rejected(
  1058. [
  1059. ChatMessage(
  1060. role="assistant",
  1061. content="",
  1062. tool_calls=[
  1063. ToolCallEvent(
  1064. id="call_1",
  1065. name="mock_search",
  1066. arguments={},
  1067. raw_arguments="{}",
  1068. )
  1069. ],
  1070. ),
  1071. ChatMessage(role="tool", content="first", tool_call_id="call_1"),
  1072. ChatMessage(role="tool", content="duplicate", tool_call_id="call_1"),
  1073. ],
  1074. "duplicate tool reply: call_1",
  1075. )
  1076. @pytest.mark.asyncio
  1077. async def test_openai_chat_client_rejects_non_tool_message_with_unresolved_calls():
  1078. await _assert_tool_transcript_rejected(
  1079. [
  1080. ChatMessage(
  1081. role="assistant",
  1082. content="checking",
  1083. tool_calls=[
  1084. ToolCallEvent(
  1085. id="call_1",
  1086. name="mock_search",
  1087. arguments={},
  1088. raw_arguments="{}",
  1089. ),
  1090. ToolCallEvent(
  1091. id="call_2",
  1092. name="handoff_note",
  1093. arguments={},
  1094. raw_arguments="{}",
  1095. ),
  1096. ],
  1097. ),
  1098. ChatMessage(role="tool", content="done", tool_call_id="call_1"),
  1099. ChatMessage(role="user", content="continue too early"),
  1100. ],
  1101. "unresolved tool calls before user message: call_2",
  1102. )
  1103. @pytest.mark.asyncio
  1104. async def test_openai_chat_client_rejects_unfinished_tool_calls_at_end():
  1105. await _assert_tool_transcript_rejected(
  1106. [
  1107. ChatMessage(
  1108. role="assistant",
  1109. content="checking",
  1110. tool_calls=[
  1111. ToolCallEvent(
  1112. id="call_1",
  1113. name="mock_search",
  1114. arguments={},
  1115. raw_arguments="{}",
  1116. )
  1117. ],
  1118. )
  1119. ],
  1120. "unresolved tool calls at end: call_1",
  1121. )
  1122. @pytest.mark.asyncio
  1123. async def test_openai_chat_client_rejects_reused_tool_call_id_across_transcript():
  1124. await _assert_tool_transcript_rejected(
  1125. [
  1126. ChatMessage(
  1127. role="assistant",
  1128. content="first round",
  1129. tool_calls=[
  1130. ToolCallEvent(
  1131. id="call_1",
  1132. name="mock_search",
  1133. arguments={},
  1134. raw_arguments="{}",
  1135. )
  1136. ],
  1137. ),
  1138. ChatMessage(role="tool", content="done", tool_call_id="call_1"),
  1139. ChatMessage(
  1140. role="assistant",
  1141. content="second round",
  1142. tool_calls=[
  1143. ToolCallEvent(
  1144. id="call_1",
  1145. name="handoff_note",
  1146. arguments={},
  1147. raw_arguments="{}",
  1148. )
  1149. ],
  1150. ),
  1151. ],
  1152. "duplicate assistant tool-call ID across transcript: call_1",
  1153. )
  1154. async def _assert_tool_choice_rejected(
  1155. tools: list[dict],
  1156. tool_choice: dict,
  1157. error_match: str,
  1158. ) -> None:
  1159. network_called = False
  1160. def handler(request: httpx.Request) -> httpx.Response:
  1161. nonlocal network_called
  1162. network_called = True
  1163. return httpx.Response(200)
  1164. async with httpx.AsyncClient(
  1165. transport=httpx.MockTransport(handler),
  1166. base_url="https://llm.test/v1",
  1167. ) as http_client:
  1168. client = OpenAICompatibleChatClient(
  1169. api_key="",
  1170. base_url="https://llm.test/v1",
  1171. default_model="provider-default",
  1172. request_timeout_seconds=5,
  1173. http_client=http_client,
  1174. )
  1175. with pytest.raises(ValueError, match=error_match):
  1176. [
  1177. item
  1178. async for item in client.stream_chat(
  1179. messages=[ChatMessage(role="user", content="use a tool")],
  1180. tools=tools,
  1181. params=AgentParams(model="provider-default"),
  1182. tool_choice=tool_choice,
  1183. )
  1184. ]
  1185. assert network_called is False
  1186. @pytest.mark.asyncio
  1187. async def test_openai_chat_client_rejects_forced_choice_without_tools():
  1188. await _assert_tool_choice_rejected(
  1189. tools=[],
  1190. tool_choice={
  1191. "type": "function",
  1192. "function": {"name": "handoff_note"},
  1193. },
  1194. error_match="forced tool choice requires tools",
  1195. )
  1196. @pytest.mark.asyncio
  1197. async def test_openai_chat_client_rejects_forced_choice_for_unknown_tool():
  1198. await _assert_tool_choice_rejected(
  1199. tools=[
  1200. {
  1201. "type": "function",
  1202. "function": {"name": "mock_search", "parameters": {}},
  1203. }
  1204. ],
  1205. tool_choice={
  1206. "type": "function",
  1207. "function": {"name": "handoff_note"},
  1208. },
  1209. error_match="forced tool choice references unknown tool: handoff_note",
  1210. )
  1211. def test_static_pre_message_role_selector_does_not_offer_tool():
  1212. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1213. assert '<option value="tool">tool</option>' not in html
  1214. def test_static_tools_ui_is_dynamic_and_not_hardcoded_to_handoff_note_checkbox():
  1215. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1216. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1217. assert 'id="tool-handoff-note"' not in html
  1218. assert "#tool-handoff-note" not in js
  1219. assert 'id="tool-list"' in html
  1220. assert '"/api/tools"' in js
  1221. assert "fetch" in js
  1222. def test_static_workspace_snapshot_controls_are_available_outside_agent_config():
  1223. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1224. chat_dialog_start = html.index('<dialog id="chat-config-dialog"')
  1225. event_dialog_start = html.index('<dialog id="event-config-dialog"')
  1226. chat_dialog_html = html[chat_dialog_start:event_dialog_start]
  1227. sidebar_html = html[:chat_dialog_start]
  1228. assert 'id="open-prompt-config"' not in html
  1229. assert '<dialog id="prompt-dialog"' not in html
  1230. assert 'id="workspace-snapshot-name"' in sidebar_html
  1231. assert 'id="saved-workspace-snapshots"' in sidebar_html
  1232. assert 'id="save-workspace-snapshot"' in sidebar_html
  1233. assert 'id="load-workspace-snapshot"' in sidebar_html
  1234. assert 'id="delete-workspace-snapshot"' in sidebar_html
  1235. assert 'id="workspace-snapshot-name"' not in chat_dialog_html
  1236. assert 'id="saved-workspace-snapshots"' not in chat_dialog_html
  1237. assert 'id="save-workspace-snapshot"' not in chat_dialog_html
  1238. assert 'id="system-prompts"' in chat_dialog_html
  1239. assert 'id="pre-messages"' in chat_dialog_html
  1240. def test_static_session_ui_controls_and_replay_panels_are_available():
  1241. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1242. chat_dialog_start = html.index('<dialog id="chat-config-dialog"')
  1243. event_dialog_start = html.index('<dialog id="event-config-dialog"')
  1244. chat_dialog_html = html[chat_dialog_start:event_dialog_start]
  1245. event_dialog_html = html[event_dialog_start:]
  1246. sidebar_html = html[:chat_dialog_start]
  1247. for control_id in [
  1248. "session-title",
  1249. "session-list",
  1250. "new-session",
  1251. "refresh-sessions",
  1252. "load-session",
  1253. "session-status",
  1254. ]:
  1255. assert f'id="{control_id}"' in sidebar_html
  1256. assert f'id="{control_id}"' not in chat_dialog_html
  1257. assert f'id="{control_id}"' not in event_dialog_html
  1258. for replay_id in [
  1259. "audit-replay",
  1260. "audit-count",
  1261. "session-usage-summary",
  1262. "session-usage-turns",
  1263. "session-usage-calls",
  1264. ]:
  1265. assert f'id="{replay_id}"' in html
  1266. def test_static_professional_console_has_semantic_three_region_shell():
  1267. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1268. root = _parse_html(html)
  1269. layout = _html_node_by_id(root, "console-layout")
  1270. direct_regions = layout.children
  1271. assert layout.tag == "main"
  1272. assert "layout" in layout.classes
  1273. assert [(node.tag, node.attrs.get("aria-label")) for node in direct_regions] == [
  1274. ("aside", "Runtime controls"),
  1275. ("section", "Conversation workspace"),
  1276. ("aside", "Audit and usage"),
  1277. ]
  1278. assert "controls" in direct_regions[0].classes
  1279. assert "workspace" in direct_regions[1].classes
  1280. assert "debug-drawer" in direct_regions[2].classes
  1281. assert direct_regions[2].attrs["id"] == "debug-drawer"
  1282. assert "Agent Lab" in html
  1283. assert "Chat and event runtime console" in html
  1284. for status_id in [
  1285. "current-session-label",
  1286. "current-mode-label",
  1287. "connection-status",
  1288. ]:
  1289. assert "status-badge" in _html_node_by_id(root, status_id).classes
  1290. drawer_toggle = _html_node_by_id(root, "toggle-debug-drawer")
  1291. assert drawer_toggle.attrs["aria-expanded"] == "true"
  1292. assert drawer_toggle.attrs["aria-controls"] == "debug-drawer"
  1293. workspace = direct_regions[1]
  1294. assert [child.attrs.get("class") for child in workspace.children] == [
  1295. "workspace-body",
  1296. "stats",
  1297. "composer",
  1298. ]
  1299. assert _html_node_by_id(workspace, "messages").attrs["class"] == "messages"
  1300. assert _html_node_by_id(workspace, "chat-form").tag == "form"
  1301. drawer_ids = {
  1302. node.attrs["id"]
  1303. for node in [direct_regions[2], *direct_regions[2].descendants()]
  1304. if "id" in node.attrs
  1305. }
  1306. workspace_ids = {
  1307. node.attrs["id"]
  1308. for node in [workspace, *workspace.descendants()]
  1309. if "id" in node.attrs
  1310. }
  1311. for replay_id in [
  1312. "audit-count",
  1313. "audit-summary",
  1314. "open-audit-dialog",
  1315. "session-usage-summary",
  1316. "session-usage-turns",
  1317. "session-usage-calls",
  1318. ]:
  1319. assert replay_id in drawer_ids
  1320. assert replay_id not in workspace_ids
  1321. def test_static_professional_console_control_sections_have_accessible_defaults():
  1322. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1323. root = _parse_html(html)
  1324. controls = next(
  1325. node
  1326. for node in _html_node_by_id(root, "console-layout").children
  1327. if "controls" in node.classes
  1328. )
  1329. sections = [child for child in controls.children if "control-section" in child.classes]
  1330. toggles = [
  1331. node
  1332. for node in root.descendants()
  1333. if "control-section-toggle" in node.classes
  1334. ]
  1335. assert len(sections) == 4
  1336. assert len(toggles) == 4
  1337. expected_defaults = {
  1338. "session-controls": ("Session", True),
  1339. "tool-controls": ("Tools", True),
  1340. "agent-config-controls": ("Agent Config", False),
  1341. "snapshot-controls": ("Snapshot", False),
  1342. }
  1343. for section, (target_id, (title, expanded)) in zip(
  1344. sections,
  1345. expected_defaults.items(),
  1346. strict=True,
  1347. ):
  1348. section_toggles = [
  1349. node
  1350. for node in section.children
  1351. if "control-section-toggle" in node.classes
  1352. ]
  1353. assert len(section_toggles) == 1
  1354. toggle = section_toggles[0]
  1355. body = _html_node_by_id(section, target_id)
  1356. assert toggle.tag == "button"
  1357. assert toggle.attrs["type"] == "button"
  1358. assert toggle.attrs["data-collapse-target"] == target_id
  1359. assert toggle.attrs["aria-controls"] == target_id
  1360. assert toggle.attrs["aria-expanded"] == str(expanded).lower()
  1361. assert title in toggle.text_content()
  1362. assert ("hidden" in body.attrs) is not expanded
  1363. def test_static_console_ui_section_click_opens_hidden_content_and_persists():
  1364. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1365. console_ui = _console_ui_source(js)
  1366. result = _run_node_json(
  1367. "\n".join(
  1368. [
  1369. 'const CONSOLE_UI_STORAGE_KEY = "agent-lab-console-ui-v1";',
  1370. """
  1371. function makeElement({ dataset = {}, expanded = "false", hidden = false } = {}) {
  1372. const listeners = {};
  1373. return {
  1374. dataset,
  1375. hidden,
  1376. textContent: "",
  1377. attributes: { "aria-expanded": expanded },
  1378. classList: { toggle() {} },
  1379. addEventListener(type, listener) { listeners[type] = listener; },
  1380. click() { listeners.click(); },
  1381. getAttribute(name) { return this.attributes[name]; },
  1382. setAttribute(name, value) { this.attributes[name] = String(value); },
  1383. };
  1384. }
  1385. const sectionButton = makeElement({
  1386. dataset: { collapseTarget: "agent-config-controls" },
  1387. expanded: "false",
  1388. });
  1389. const sectionBody = makeElement({ hidden: true });
  1390. const drawerButton = makeElement({ expanded: "true" });
  1391. drawerButton.textContent = "Hide Debug";
  1392. const drawer = makeElement();
  1393. const layout = makeElement();
  1394. const elements = {
  1395. "#agent-config-controls": sectionBody,
  1396. "#toggle-debug-drawer": drawerButton,
  1397. "#debug-drawer": drawer,
  1398. "#console-layout": layout,
  1399. };
  1400. const document = {
  1401. querySelector(selector) { return elements[selector] || null; },
  1402. querySelectorAll(selector) {
  1403. return selector === ".control-section-toggle" ? [sectionButton] : [];
  1404. },
  1405. };
  1406. let stored = JSON.stringify({
  1407. debug_drawer_open: false,
  1408. sections: { "agent-config-controls": false },
  1409. });
  1410. globalThis.localStorage = {
  1411. getItem() { return stored; },
  1412. setItem(key, value) { stored = value; },
  1413. };
  1414. """,
  1415. console_ui,
  1416. """
  1417. initializeConsoleUi();
  1418. sectionButton.click();
  1419. console.log(JSON.stringify({
  1420. expanded: sectionButton.getAttribute("aria-expanded"),
  1421. hidden: sectionBody.hidden,
  1422. stored: JSON.parse(stored),
  1423. }));
  1424. """,
  1425. ]
  1426. )
  1427. )
  1428. assert result == {
  1429. "expanded": "true",
  1430. "hidden": False,
  1431. "stored": {
  1432. "debug_drawer_open": False,
  1433. "sections": {"agent-config-controls": True},
  1434. },
  1435. }
  1436. def test_static_console_ui_drawer_click_updates_layout_visibility_and_control():
  1437. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1438. console_ui = _console_ui_source(js)
  1439. result = _run_node_json(
  1440. "\n".join(
  1441. [
  1442. 'const CONSOLE_UI_STORAGE_KEY = "agent-lab-console-ui-v1";',
  1443. """
  1444. function makeElement({ expanded = "true" } = {}) {
  1445. const listeners = {};
  1446. const classes = new Set();
  1447. return {
  1448. hidden: false,
  1449. textContent: "Hide Debug",
  1450. attributes: { "aria-expanded": expanded },
  1451. classList: {
  1452. toggle(name, force) {
  1453. if (force) classes.add(name); else classes.delete(name);
  1454. },
  1455. contains(name) { return classes.has(name); },
  1456. },
  1457. addEventListener(type, listener) { listeners[type] = listener; },
  1458. click() { listeners.click(); },
  1459. getAttribute(name) { return this.attributes[name]; },
  1460. setAttribute(name, value) { this.attributes[name] = String(value); },
  1461. };
  1462. }
  1463. const drawerButton = makeElement();
  1464. const drawer = makeElement();
  1465. const layout = makeElement();
  1466. const elements = {
  1467. "#toggle-debug-drawer": drawerButton,
  1468. "#debug-drawer": drawer,
  1469. "#console-layout": layout,
  1470. };
  1471. const document = {
  1472. querySelector(selector) { return elements[selector] || null; },
  1473. querySelectorAll() { return []; },
  1474. };
  1475. let stored = JSON.stringify({ sections: { "snapshot-controls": true } });
  1476. globalThis.localStorage = {
  1477. getItem() { return stored; },
  1478. setItem(key, value) { stored = value; },
  1479. };
  1480. function snapshot() {
  1481. return {
  1482. closed: layout.classList.contains("debug-drawer-closed"),
  1483. hidden: drawer.hidden,
  1484. text: drawerButton.textContent,
  1485. expanded: drawerButton.getAttribute("aria-expanded"),
  1486. stored: JSON.parse(stored),
  1487. };
  1488. }
  1489. """,
  1490. console_ui,
  1491. """
  1492. initializeConsoleUi();
  1493. drawerButton.click();
  1494. const closed = snapshot();
  1495. drawerButton.click();
  1496. const opened = snapshot();
  1497. console.log(JSON.stringify({ closed, opened }));
  1498. """,
  1499. ]
  1500. )
  1501. )
  1502. assert result["closed"] == {
  1503. "closed": True,
  1504. "hidden": True,
  1505. "text": "Show Debug",
  1506. "expanded": "false",
  1507. "stored": {
  1508. "sections": {"snapshot-controls": True},
  1509. "debug_drawer_open": False,
  1510. },
  1511. }
  1512. assert result["opened"] == {
  1513. "closed": False,
  1514. "hidden": False,
  1515. "text": "Hide Debug",
  1516. "expanded": "true",
  1517. "stored": {
  1518. "sections": {"snapshot-controls": True},
  1519. "debug_drawer_open": True,
  1520. },
  1521. }
  1522. def test_static_console_ui_restores_valid_state_without_writing_on_initialize():
  1523. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1524. console_ui = _console_ui_source(js)
  1525. result = _run_node_json(
  1526. "\n".join(
  1527. [
  1528. 'const CONSOLE_UI_STORAGE_KEY = "agent-lab-console-ui-v1";',
  1529. """
  1530. function makeElement({ dataset = {}, expanded = "false", hidden = false } = {}) {
  1531. const listeners = {};
  1532. const classes = new Set();
  1533. return {
  1534. dataset,
  1535. hidden,
  1536. textContent: "",
  1537. attributes: { "aria-expanded": expanded },
  1538. classList: {
  1539. toggle(name, force) {
  1540. if (force) classes.add(name); else classes.delete(name);
  1541. },
  1542. contains(name) { return classes.has(name); },
  1543. },
  1544. addEventListener(type, listener) { listeners[type] = listener; },
  1545. getAttribute(name) { return this.attributes[name]; },
  1546. setAttribute(name, value) { this.attributes[name] = String(value); },
  1547. };
  1548. }
  1549. const agentButton = makeElement({
  1550. dataset: { collapseTarget: "agent-config-controls" },
  1551. expanded: "false",
  1552. });
  1553. const snapshotButton = makeElement({
  1554. dataset: { collapseTarget: "snapshot-controls" },
  1555. expanded: "false",
  1556. });
  1557. const agentBody = makeElement({ hidden: true });
  1558. const snapshotBody = makeElement({ hidden: true });
  1559. const drawerButton = makeElement({ expanded: "true" });
  1560. drawerButton.textContent = "Hide Debug";
  1561. const drawer = makeElement();
  1562. const layout = makeElement();
  1563. const elements = {
  1564. "#agent-config-controls": agentBody,
  1565. "#snapshot-controls": snapshotBody,
  1566. "#toggle-debug-drawer": drawerButton,
  1567. "#debug-drawer": drawer,
  1568. "#console-layout": layout,
  1569. };
  1570. const document = {
  1571. querySelector(selector) { return elements[selector] || null; },
  1572. querySelectorAll(selector) {
  1573. return selector === ".control-section-toggle"
  1574. ? [agentButton, snapshotButton]
  1575. : [];
  1576. },
  1577. };
  1578. let writes = 0;
  1579. globalThis.localStorage = {
  1580. getItem() {
  1581. return JSON.stringify({
  1582. debug_drawer_open: false,
  1583. sections: { "agent-config-controls": true },
  1584. });
  1585. },
  1586. setItem() { writes += 1; },
  1587. };
  1588. """,
  1589. console_ui,
  1590. """
  1591. initializeConsoleUi();
  1592. console.log(JSON.stringify({
  1593. agentExpanded: agentButton.getAttribute("aria-expanded"),
  1594. agentHidden: agentBody.hidden,
  1595. snapshotExpanded: snapshotButton.getAttribute("aria-expanded"),
  1596. snapshotHidden: snapshotBody.hidden,
  1597. drawerExpanded: drawerButton.getAttribute("aria-expanded"),
  1598. drawerHidden: drawer.hidden,
  1599. drawerClosed: layout.classList.contains("debug-drawer-closed"),
  1600. writes,
  1601. }));
  1602. """,
  1603. ]
  1604. )
  1605. )
  1606. assert result == {
  1607. "agentExpanded": "true",
  1608. "agentHidden": False,
  1609. "snapshotExpanded": "false",
  1610. "snapshotHidden": True,
  1611. "drawerExpanded": "false",
  1612. "drawerHidden": True,
  1613. "drawerClosed": True,
  1614. "writes": 0,
  1615. }
  1616. def test_static_console_ui_storage_failures_fall_back_without_throwing():
  1617. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1618. console_ui = _console_ui_source(js)
  1619. result = _run_node_json(
  1620. "\n".join(
  1621. [
  1622. 'const CONSOLE_UI_STORAGE_KEY = "agent-lab-console-ui-v1";',
  1623. """
  1624. function makeElement({ dataset = {}, expanded = "false", hidden = false } = {}) {
  1625. return {
  1626. dataset,
  1627. hidden,
  1628. textContent: "",
  1629. attributes: { "aria-expanded": expanded },
  1630. classList: { toggle() {} },
  1631. addEventListener() {},
  1632. getAttribute(name) { return this.attributes[name]; },
  1633. setAttribute(name, value) { this.attributes[name] = String(value); },
  1634. };
  1635. }
  1636. const sectionButton = makeElement({
  1637. dataset: { collapseTarget: "snapshot-controls" },
  1638. expanded: "false",
  1639. });
  1640. const sectionBody = makeElement({ hidden: true });
  1641. const drawerButton = makeElement({ expanded: "true" });
  1642. const drawer = makeElement();
  1643. const layout = makeElement();
  1644. const elements = {
  1645. "#snapshot-controls": sectionBody,
  1646. "#toggle-debug-drawer": drawerButton,
  1647. "#debug-drawer": drawer,
  1648. "#console-layout": layout,
  1649. };
  1650. const document = {
  1651. querySelector(selector) { return elements[selector] || null; },
  1652. querySelectorAll(selector) {
  1653. return selector === ".control-section-toggle" ? [sectionButton] : [];
  1654. },
  1655. };
  1656. globalThis.localStorage = {
  1657. getItem() { return "{malformed"; },
  1658. setItem() { throw new Error("storage unavailable"); },
  1659. };
  1660. """,
  1661. console_ui,
  1662. """
  1663. const malformed = readConsoleUiState();
  1664. writeConsoleUiState({ debug_drawer_open: true });
  1665. initializeConsoleUi();
  1666. const malformedDefaults = {
  1667. sectionExpanded: sectionButton.getAttribute("aria-expanded"),
  1668. sectionHidden: sectionBody.hidden,
  1669. drawerExpanded: drawerButton.getAttribute("aria-expanded"),
  1670. drawerHidden: drawer.hidden,
  1671. };
  1672. Object.defineProperty(globalThis, "localStorage", {
  1673. configurable: true,
  1674. get() { throw new Error("storage unavailable"); },
  1675. });
  1676. const unavailable = readConsoleUiState();
  1677. writeConsoleUiState({ sections: {} });
  1678. initializeConsoleUi();
  1679. console.log(JSON.stringify({ malformed, malformedDefaults, unavailable }));
  1680. """,
  1681. ]
  1682. )
  1683. )
  1684. assert result == {
  1685. "malformed": {},
  1686. "malformedDefaults": {
  1687. "sectionExpanded": "false",
  1688. "sectionHidden": True,
  1689. "drawerExpanded": "true",
  1690. "drawerHidden": False,
  1691. },
  1692. "unavailable": {},
  1693. }
  1694. def test_static_topbar_status_updates_for_session_and_tool_mode():
  1695. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1696. console_ui = _console_ui_source(js)
  1697. result = _run_node_json(
  1698. "\n".join(
  1699. [
  1700. 'const CONSOLE_UI_STORAGE_KEY = "agent-lab-console-ui-v1";',
  1701. """
  1702. const sessionLabel = { textContent: "" };
  1703. const modeLabel = { textContent: "" };
  1704. const toolInvocationMode = { value: "dual_agent" };
  1705. const document = {
  1706. querySelector(selector) {
  1707. return {
  1708. "#current-session-label": sessionLabel,
  1709. "#current-mode-label": modeLabel,
  1710. }[selector] || null;
  1711. },
  1712. };
  1713. let currentSessionId = null;
  1714. """,
  1715. console_ui,
  1716. """
  1717. updateTopbarStatus();
  1718. const initial = { session: sessionLabel.textContent, mode: modeLabel.textContent };
  1719. currentSessionId = "1234567890abcdef";
  1720. toolInvocationMode.value = "chat_agent_tools";
  1721. updateTopbarStatus();
  1722. console.log(JSON.stringify({
  1723. initial,
  1724. updated: { session: sessionLabel.textContent, mode: modeLabel.textContent },
  1725. }));
  1726. """,
  1727. ]
  1728. )
  1729. )
  1730. assert result == {
  1731. "initial": {"session": "No session", "mode": "Dual Agent"},
  1732. "updated": {"session": "Session 12345678", "mode": "ChatAgent Tools"},
  1733. }
  1734. def test_static_console_ui_bootstrap_and_session_paths_refresh_topbar_status():
  1735. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1736. refresh_tool_mode = js[
  1737. js.index("function refreshToolModeState(") :
  1738. js.index("function updateToolModeHelp(")
  1739. ]
  1740. assert "initializeConsoleUi();" in js[: js.index("function openChatConfig(")]
  1741. assert "updateTopbarStatus();" in refresh_tool_mode
  1742. assert "currentSessionId = session.id;\n updateTopbarStatus();" in js
  1743. assert "currentSessionId = sessionId;\n updateTopbarStatus();" in js
  1744. assert "currentSessionId = message.session_id;\n updateTopbarStatus();" in js
  1745. def test_static_presentation_state_is_excluded_from_request_and_snapshot():
  1746. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1747. build_snapshot = js[
  1748. js.index("function buildWorkspaceSnapshot()") :
  1749. js.index("function restoreWorkspaceSnapshot(")
  1750. ]
  1751. build_request = js[
  1752. js.index("function buildRequest(") :
  1753. js.index("function selectedTools(")
  1754. ]
  1755. for source in [build_snapshot, build_request]:
  1756. assert "CONSOLE_UI_STORAGE_KEY" not in source
  1757. assert "readConsoleUiState" not in source
  1758. assert "writeConsoleUiState" not in source
  1759. assert "debug_drawer_open" not in source
  1760. def test_static_professional_console_defines_visual_and_responsive_contracts():
  1761. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1762. css = Path("src/agent_lab/presentation/static/styles.css").read_text()
  1763. assert '/static/styles.css?v=20260714-professional-console' in html
  1764. assert '/static/app.js?v=20260714-professional-console' in html
  1765. for token in [
  1766. "--canvas:",
  1767. "--surface:",
  1768. "--surface-subtle:",
  1769. "--border:",
  1770. "--text:",
  1771. "--muted:",
  1772. "--accent:",
  1773. "--accent-strong:",
  1774. "--danger:",
  1775. "--radius:",
  1776. "--shadow:",
  1777. ]:
  1778. assert token in _css_rule(css, ":root")
  1779. assert "button-primary" in html
  1780. assert "button-secondary" in html
  1781. assert "button-danger" in html
  1782. assert ".button-primary" in css
  1783. assert ".button-secondary" in css
  1784. assert ".button-danger" in css
  1785. assert ".status-badge" in css
  1786. assert ":focus-visible" in css
  1787. assert ".tool-card:has(input:checked)" in css
  1788. assert "display: none" in _css_rule(css, "[hidden]")
  1789. assert "@media (max-width: 900px)" in css
  1790. assert "\nsection + section {" not in css
  1791. def test_static_chat_agent_prompt_is_a_single_sortable_list_with_event_instructions():
  1792. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  1793. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1794. css = Path("src/agent_lab/presentation/static/styles.css").read_text()
  1795. assert 'id="chat-system-prompt"' not in html
  1796. assert 'id="chat-event-prompt-preview"' not in html
  1797. assert 'class="prompt-list stack"' in html
  1798. assert 'class="prompt-item"' in html
  1799. assert 'data-prompt-role="system"' in html
  1800. assert "prompt-type" in html
  1801. assert "function buildAvailableEventsPrompt(" in js
  1802. assert "Available events:" in js
  1803. assert "function updateEventPromptItem(" in js
  1804. assert "function createSystemPrompt(" in js
  1805. assert "function movePromptItem(" in js
  1806. assert 'draggable = true' in js
  1807. assert "dataset.promptRole" in js
  1808. assert "prompt-title" in js
  1809. assert "delete-system-prompt" in js
  1810. assert 'prompt_kind: "event_agent_system"' in js
  1811. assert "system_prompts: collectSystemPrompts()" in js
  1812. assert 'system_prompt: ""' in js
  1813. assert ".prompt-item" in css
  1814. assert ".prompt-item.dragging" in css
  1815. assert ".prompt-meta" in css
  1816. assert ".prompt-type" in css
  1817. assert ".prompt-actions button" in css
  1818. assert "#open-prompt-config" not in js
  1819. assert "promptDialog" not in js
  1820. def test_static_event_prompt_refresh_uses_unified_tool_mode_state_path():
  1821. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1822. render_tools = js[
  1823. js.index("function renderTools(") :
  1824. js.index("function formatRequiredParameters(")
  1825. ]
  1826. set_all_tools = js[
  1827. js.index("function setAllTools(") :
  1828. js.index("function updateToolCount(")
  1829. ]
  1830. restore_snapshot = js[
  1831. js.index("function restoreWorkspaceSnapshot(") :
  1832. js.index("function updateToolModeHelp(")
  1833. ]
  1834. apply_selected_tools = js[
  1835. js.index("function applySelectedTools(") :
  1836. js.index("function initializePromptList(")
  1837. ]
  1838. initialize_prompt_list = js[
  1839. js.index("function initializePromptList(") :
  1840. js.index("function buildPromptItems(")
  1841. ]
  1842. refresh_tool_mode = js[
  1843. js.index("function refreshToolModeState(") :
  1844. js.index("function updateToolModeHelp(")
  1845. ]
  1846. remove_generated_prompts = js[
  1847. js.index("function removeGeneratedEventPromptItems(") :
  1848. js.index("function ensureEventPromptItem(")
  1849. ]
  1850. assert (
  1851. 'toolInvocationMode.addEventListener("change", refreshToolModeState);'
  1852. in js
  1853. )
  1854. assert "refreshToolModeState();" in render_tools
  1855. assert "updateEventPromptItem(" not in render_tools
  1856. assert "refreshToolModeState();" in set_all_tools
  1857. assert "updateEventPromptItem(" not in set_all_tools
  1858. assert "refreshToolModeState();" in restore_snapshot
  1859. assert "refreshToolModeState();" in apply_selected_tools
  1860. assert "refreshToolModeState();" in initialize_prompt_list
  1861. assert "removeGeneratedEventPromptItems();" in refresh_tool_mode
  1862. assert "if (!isChatAgentTools)" in refresh_tool_mode
  1863. assert "updateEventPromptItem(availableTools);" in refresh_tool_mode
  1864. assert 'querySelectorAll(".prompt-item")' in remove_generated_prompts
  1865. assert "isGeneratedEventPromptItem(item)" in remove_generated_prompts
  1866. assert "item.remove();" in remove_generated_prompts
  1867. assert 'textarea.readOnly = promptItem.prompt_kind === "event_agent_system"' in js
  1868. assert "let eventPromptInsertionIndex" in js
  1869. capture_position = js[
  1870. js.index("function captureEventPromptInsertionIndex(") :
  1871. js.index("function removeGeneratedEventPromptItems(")
  1872. ]
  1873. ensure_event_prompt = js[
  1874. js.index("function ensureEventPromptItem(") :
  1875. js.index("function bindPromptItemControls(")
  1876. ]
  1877. restore_prompts = js[
  1878. js.index("function restoreSystemPrompts(") :
  1879. js.index("function restorePreMessages(")
  1880. ]
  1881. assert "return retainedPromptCount;" in capture_position
  1882. assert "captureEventPromptInsertionIndex();" in refresh_tool_mode
  1883. assert "capturedEventPromptIndex !== null" in refresh_tool_mode
  1884. assert "eventPromptInsertionIndex = capturedEventPromptIndex;" in refresh_tool_mode
  1885. assert "deriveEventPromptInsertionIndex(prompts);" in restore_prompts
  1886. assert "eventPromptInsertionIndex = restoredEventPromptIndex;" in restore_prompts
  1887. assert "const insertionTarget" in ensure_event_prompt
  1888. assert "systemPrompts.insertBefore(item, insertionTarget);" in ensure_event_prompt
  1889. assert "systemPrompts.append(item);" not in ensure_event_prompt
  1890. def test_static_event_prompt_order_survives_refresh_and_tool_mode_roundtrip():
  1891. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  1892. generated = "\n".join(
  1893. [
  1894. "You may request EventAgent work using this text protocol.",
  1895. "Available events:",
  1896. "- handoff_note: Handoff a note",
  1897. "<agent_events>",
  1898. "handoff_note",
  1899. "</agent_events>",
  1900. "Use exact event names only, one per line. Do not include parameters.",
  1901. ]
  1902. )
  1903. assert "function captureEventPromptInsertionIndex(" in js
  1904. refresh_tool_mode = js[
  1905. js.index("function refreshToolModeState(") :
  1906. js.index("function updateToolModeHelp(")
  1907. ]
  1908. generated_matcher = js[
  1909. js.index("function isGeneratedEventPromptContent(") :
  1910. js.index("function buildAvailableEventsPrompt(")
  1911. ]
  1912. event_prompt_functions = js[
  1913. js.index("function updateEventPromptItem(") :
  1914. js.index("function bindPromptItemControls(")
  1915. ]
  1916. result = _run_node_json(
  1917. "\n".join(
  1918. [
  1919. refresh_tool_mode,
  1920. generated_matcher,
  1921. event_prompt_functions,
  1922. """
  1923. const generated = [
  1924. "You may request EventAgent work using this text protocol.",
  1925. "Available events:",
  1926. "- handoff_note: Handoff a note",
  1927. "<agent_events>",
  1928. "handoff_note",
  1929. "</agent_events>",
  1930. "Use exact event names only, one per line. Do not include parameters.",
  1931. ].join("\\n");
  1932. const items = [];
  1933. function makePromptItem(promptKind, content) {
  1934. const item = {
  1935. dataset: { promptKind, promptRole: "system" },
  1936. content,
  1937. querySelector() {
  1938. return {
  1939. get value() { return item.content; },
  1940. set value(value) { item.content = value; },
  1941. };
  1942. },
  1943. remove() {
  1944. const index = items.indexOf(item);
  1945. if (index >= 0) items.splice(index, 1);
  1946. },
  1947. };
  1948. return item;
  1949. }
  1950. items.push(
  1951. makePromptItem("custom", "alpha"),
  1952. makePromptItem("event_agent_system", "stale generated prompt"),
  1953. makePromptItem("custom", "beta"),
  1954. makePromptItem("custom", generated),
  1955. makePromptItem("custom", "gamma"),
  1956. );
  1957. const systemPrompts = {
  1958. querySelectorAll() { return [...items]; },
  1959. querySelector() {
  1960. return items.find((item) => item.dataset.promptKind === "event_agent_system") || null;
  1961. },
  1962. insertBefore(item, target) {
  1963. const currentIndex = items.indexOf(item);
  1964. if (currentIndex >= 0) items.splice(currentIndex, 1);
  1965. const targetIndex = target ? items.indexOf(target) : items.length;
  1966. items.splice(targetIndex < 0 ? items.length : targetIndex, 0, item);
  1967. },
  1968. };
  1969. const toolInvocationMode = { value: "dual_agent" };
  1970. const eventAgentModelSettings = { querySelectorAll() { return []; } };
  1971. const availableTools = [{ name: "handoff_note" }];
  1972. let eventPromptInsertionIndex = 0;
  1973. function updateTopbarStatus() {}
  1974. function updateToolModeHelp() {}
  1975. function selectedTools() { return ["handoff_note"]; }
  1976. function buildAvailableEventsPrompt() { return generated; }
  1977. function createSystemPrompt() {
  1978. return makePromptItem("event_agent_system", "");
  1979. }
  1980. function snapshot() {
  1981. return items.map((item) => [item.dataset.promptKind, item.content]);
  1982. }
  1983. refreshToolModeState();
  1984. const afterRefresh = snapshot();
  1985. toolInvocationMode.value = "chat_agent_tools";
  1986. refreshToolModeState();
  1987. refreshToolModeState();
  1988. const whileHidden = snapshot();
  1989. const hiddenIndex = eventPromptInsertionIndex;
  1990. toolInvocationMode.value = "dual_agent";
  1991. refreshToolModeState();
  1992. const afterRoundtrip = snapshot();
  1993. console.log(JSON.stringify({ afterRefresh, whileHidden, hiddenIndex, afterRoundtrip }));
  1994. """,
  1995. ]
  1996. )
  1997. )
  1998. expected_visible = [
  1999. ["custom", "alpha"],
  2000. ["event_agent_system", generated],
  2001. ["custom", "beta"],
  2002. ["custom", "gamma"],
  2003. ]
  2004. assert result["afterRefresh"] == expected_visible
  2005. assert result["whileHidden"] == [
  2006. ["custom", "alpha"],
  2007. ["custom", "beta"],
  2008. ["custom", "gamma"],
  2009. ]
  2010. assert result["hiddenIndex"] == 1
  2011. assert result["afterRoundtrip"] == expected_visible
  2012. def test_static_prompt_restore_derives_first_generated_event_prompt_position():
  2013. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2014. assert "function deriveEventPromptInsertionIndex(" in js
  2015. normalization = js[
  2016. js.index("function normalizePromptItems(") :
  2017. js.index("function buildAvailableEventsPrompt(")
  2018. ]
  2019. result = _run_node_json(
  2020. "\n".join(
  2021. [
  2022. normalization,
  2023. """
  2024. const generated = [
  2025. "You may request EventAgent work using this text protocol.",
  2026. "Available events:",
  2027. "<agent_events>",
  2028. "handoff_note",
  2029. "</agent_events>",
  2030. "Use exact event names only, one per line. Do not include parameters.",
  2031. ].join("\\n");
  2032. const prompts = [
  2033. { prompt_kind: "custom", content: "alpha" },
  2034. generated,
  2035. { prompt_kind: "custom", content: "beta" },
  2036. { prompt_kind: "event_agent_system", content: generated },
  2037. { prompt_kind: "custom", content: "gamma" },
  2038. ];
  2039. console.log(JSON.stringify({
  2040. insertionIndex: deriveEventPromptInsertionIndex(prompts),
  2041. normalized: normalizePromptItems(prompts),
  2042. }));
  2043. """,
  2044. ]
  2045. )
  2046. )
  2047. assert result["insertionIndex"] == 1
  2048. assert result["normalized"] == [
  2049. {"prompt_kind": "custom", "role": "system", "content": "alpha"},
  2050. {"prompt_kind": "custom", "role": "system", "content": "beta"},
  2051. {"prompt_kind": "custom", "role": "system", "content": "gamma"},
  2052. ]
  2053. def test_static_prompt_restore_discards_generated_event_prompt_derived_state():
  2054. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2055. ordinary = "Keep this custom prompt with an <agent_events> example."
  2056. exact_name_docs = (
  2057. "Use exact event names only, one per line. Do not include parameters."
  2058. )
  2059. assert "function isGeneratedEventPromptContent(" in js
  2060. normalization = js[
  2061. js.index("function normalizePromptItems(") :
  2062. js.index("function buildAvailableEventsPrompt(")
  2063. ]
  2064. result = _run_node_json(
  2065. "\n".join(
  2066. [
  2067. normalization,
  2068. """
  2069. const generated = [
  2070. "You may request EventAgent work using this text protocol.",
  2071. "Available events:",
  2072. "- handoff_note: Handoff a note",
  2073. "First write the user-facing reply normally. If events are needed, append this block after the visible reply:",
  2074. "<agent_events>",
  2075. "handoff_note",
  2076. "</agent_events>",
  2077. "Use exact event names only, one per line. Do not include parameters.",
  2078. ].join("\\n");
  2079. const ordinary = "Keep this custom prompt with an <agent_events> example.";
  2080. const exactNameDocs = "Use exact event names only, one per line. Do not include parameters.";
  2081. console.log(JSON.stringify({
  2082. matches: [
  2083. isGeneratedEventPromptContent(generated),
  2084. isGeneratedEventPromptContent(ordinary),
  2085. isGeneratedEventPromptContent(exactNameDocs),
  2086. ],
  2087. normalized: normalizePromptItems([
  2088. { prompt_kind: "event_agent_system", role: "system", content: generated },
  2089. generated,
  2090. ordinary,
  2091. { prompt_kind: "custom", role: "system", content: exactNameDocs },
  2092. ]),
  2093. empty: normalizePromptItems([]),
  2094. }));
  2095. """,
  2096. ]
  2097. )
  2098. )
  2099. assert result["matches"] == [True, False, False]
  2100. assert result["normalized"] == [
  2101. {"prompt_kind": "custom", "role": "system", "content": ordinary},
  2102. {"prompt_kind": "custom", "role": "system", "content": exact_name_docs},
  2103. ]
  2104. assert result["empty"] == [
  2105. {"prompt_kind": "custom", "role": "system", "content": ""}
  2106. ]
  2107. def test_static_event_prompt_payload_singleton_depends_on_tool_mode():
  2108. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2109. managed_generated = "\n".join(
  2110. [
  2111. "You may request EventAgent work using this text protocol.",
  2112. "Available events:",
  2113. "- managed_event: Managed event",
  2114. "<agent_events>",
  2115. "managed_event",
  2116. "</agent_events>",
  2117. "Use exact event names only, one per line. Do not include parameters.",
  2118. ]
  2119. )
  2120. prompt_collection = js[
  2121. js.index("function buildPromptItems(") :
  2122. js.index("function normalizePromptItems(")
  2123. ]
  2124. generated_matcher = js[
  2125. js.index("function isGeneratedEventPromptContent(") :
  2126. js.index("function buildAvailableEventsPrompt(")
  2127. ]
  2128. result = _run_node_json(
  2129. "\n".join(
  2130. [
  2131. prompt_collection,
  2132. generated_matcher,
  2133. """
  2134. const impostorGenerated = [
  2135. "You may request EventAgent work using this text protocol.",
  2136. "Available events:",
  2137. "- impostor_event: Impostor custom event",
  2138. "<agent_events>",
  2139. "impostor_event",
  2140. "</agent_events>",
  2141. "Use exact event names only, one per line. Do not include parameters.",
  2142. ].join("\\n");
  2143. const managedGenerated = [
  2144. "You may request EventAgent work using this text protocol.",
  2145. "Available events:",
  2146. "- managed_event: Managed event",
  2147. "<agent_events>",
  2148. "managed_event",
  2149. "</agent_events>",
  2150. "Use exact event names only, one per line. Do not include parameters.",
  2151. ].join("\\n");
  2152. function promptItem(promptKind, content) {
  2153. return {
  2154. dataset: { promptKind, promptRole: "system" },
  2155. querySelector() {
  2156. return { value: content };
  2157. },
  2158. };
  2159. }
  2160. const systemPrompts = {
  2161. querySelectorAll() {
  2162. return [
  2163. promptItem("custom", "ordinary custom"),
  2164. promptItem("custom", impostorGenerated),
  2165. promptItem("event_agent_system", managedGenerated),
  2166. ];
  2167. },
  2168. };
  2169. const toolInvocationMode = { value: "chat_agent_tools" };
  2170. const chatAgentTools = collectSystemPrompts();
  2171. toolInvocationMode.value = "dual_agent";
  2172. const dualAgent = collectSystemPrompts();
  2173. const generatedMatches = [
  2174. isGeneratedEventPromptContent(impostorGenerated),
  2175. isGeneratedEventPromptContent(managedGenerated),
  2176. ];
  2177. console.log(JSON.stringify({ chatAgentTools, dualAgent, generatedMatches }));
  2178. """,
  2179. ]
  2180. )
  2181. )
  2182. assert result["generatedMatches"] == [True, True]
  2183. assert result["chatAgentTools"] == ["ordinary custom"]
  2184. assert result["dualAgent"] == ["ordinary custom", managed_generated]
  2185. def test_static_tool_mode_disables_only_event_agent_model_settings_group():
  2186. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2187. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2188. model_settings_start = html.index('id="event-agent-model-settings"')
  2189. shared_settings_start = html.index('id="max-event-loops"')
  2190. model_settings_html = html[model_settings_start:shared_settings_start]
  2191. refresh_tool_mode = js[
  2192. js.index("function refreshToolModeState(") :
  2193. js.index("function updateToolModeHelp(")
  2194. ]
  2195. for control_id in [
  2196. "event-model",
  2197. "event-temperature",
  2198. "event-max-tokens",
  2199. "event-system-prompt",
  2200. "event-thinking-disabled",
  2201. "event-enable-search",
  2202. "event-forced-search",
  2203. ]:
  2204. assert f'id="{control_id}"' in model_settings_html
  2205. for control_id in [
  2206. "max-event-loops",
  2207. "max-parallel-events",
  2208. "batch-timeout-seconds",
  2209. ]:
  2210. assert f'id="{control_id}"' not in model_settings_html
  2211. assert 'document.querySelector("#event-agent-model-settings")' in js
  2212. assert '.querySelectorAll("input, textarea, select, button")' in refresh_tool_mode
  2213. assert "control.disabled = isChatAgentTools;" in refresh_tool_mode
  2214. def test_static_app_script_is_versioned_and_click_binding_is_guarded():
  2215. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2216. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2217. assert '<script src="/static/app.js?v=' in html
  2218. assert "function bindClick(" in js
  2219. assert 'bindClick("#open-chat-config-panel", openChatConfig)' in js
  2220. assert 'bindClick("#open-event-config-panel", openEventConfig)' in js
  2221. def test_static_agent_config_controls_are_aligned_for_chat_and_event_agents():
  2222. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2223. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2224. assert '<dialog id="chat-config-dialog"' in html
  2225. assert '<dialog id="event-config-dialog"' in html
  2226. assert html.count('id="open-chat-config-panel"') == 1
  2227. assert html.count('id="open-event-config-panel"') == 1
  2228. assert 'id="open-chat-config"' not in html
  2229. assert 'id="open-event-config"' not in html
  2230. assert '#open-chat-config"' not in js
  2231. assert '#open-event-config"' not in js
  2232. assert '#open-chat-config-panel"' in js
  2233. assert '#open-event-config-panel"' in js
  2234. assert 'id="close-chat-config"' in html
  2235. assert 'id="close-event-config"' in html
  2236. assert "chatConfigDialog.showModal()" in js
  2237. assert "eventConfigDialog.showModal()" in js
  2238. assert "chatConfigDialog.close()" in js
  2239. assert "eventConfigDialog.close()" in js
  2240. for control_id in [
  2241. "model",
  2242. "temperature",
  2243. "max-tokens",
  2244. "chat-thinking-disabled",
  2245. "chat-enable-search",
  2246. "chat-forced-search",
  2247. "event-model",
  2248. "event-temperature",
  2249. "event-max-tokens",
  2250. "event-system-prompt",
  2251. "event-thinking-disabled",
  2252. "event-enable-search",
  2253. "event-forced-search",
  2254. ]:
  2255. assert f'id="{control_id}"' in html
  2256. assert 'extra_body: buildExtraBody("chat")' in js
  2257. assert 'extra_body: buildExtraBody("event")' in js
  2258. assert 'system_prompt: ""' in js
  2259. assert 'system_prompt: document.querySelector("#event-system-prompt")' in js
  2260. def test_static_tool_mode_and_batch_controls_explain_active_configuration():
  2261. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2262. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2263. assert 'id="tool-invocation-mode"' in html
  2264. assert '<option value="dual_agent" selected>' in html
  2265. assert '<option value="chat_agent_tools">' in html
  2266. assert 'id="tool-mode-help"' in html
  2267. assert 'id="max-parallel-events"' in html
  2268. assert 'value="4"' in html
  2269. assert 'id="batch-timeout-seconds"' in html
  2270. assert 'value="15"' in html
  2271. assert "missing-argument fallback" in js
  2272. assert "EventAgent model, temperature, max tokens, system prompt, and extra body are inactive" in js
  2273. assert "enabled tools, loop budget, batch concurrency, and timeout remain active" in js
  2274. assert "function updateToolModeHelp()" in js
  2275. def test_static_tool_mode_config_roundtrips_request_snapshot_and_session_restore():
  2276. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2277. build_snapshot = js[
  2278. js.index("function buildWorkspaceSnapshot()") :
  2279. js.index("function restoreWorkspaceSnapshot(")
  2280. ]
  2281. restore_snapshot = js[
  2282. js.index("function restoreWorkspaceSnapshot(") :
  2283. js.index("function buildExtraBody(")
  2284. ]
  2285. build_request = js[
  2286. js.index("function buildRequest(") :
  2287. js.index("function selectedTools(")
  2288. ]
  2289. load_replay = js[
  2290. js.index("async function loadSessionReplay(") :
  2291. js.index("function setSessionStatus(")
  2292. ]
  2293. create_session = js[
  2294. js.index("async function createSession()") :
  2295. js.index("async function loadSelectedSession()")
  2296. ]
  2297. for source in [build_snapshot, build_request]:
  2298. assert 'tool_invocation_mode: document.querySelector("#tool-invocation-mode").value' in source
  2299. assert 'max_parallel_events: Number(document.querySelector("#max-parallel-events").value)' in source
  2300. assert 'batch_timeout_seconds: Number(document.querySelector("#batch-timeout-seconds").value)' in source
  2301. assert 'snapshot.tool_invocation_mode || "dual_agent"' in restore_snapshot
  2302. assert 'eventAgent.max_parallel_events ?? 4' in restore_snapshot
  2303. assert 'eventAgent.batch_timeout_seconds ?? 15' in restore_snapshot
  2304. assert "refreshToolModeState();" in restore_snapshot
  2305. assert '`/api/sessions/${sessionId}`' in load_replay
  2306. assert "const [session, messages, audit, usage] = await Promise.all([" in load_replay
  2307. assert "restoreWorkspaceSnapshot(session.config || {});" in load_replay
  2308. assert "config: buildWorkspaceSnapshot()" in create_session
  2309. def test_static_tools_ui_shows_tool_metadata_and_selection_actions():
  2310. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2311. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2312. assert 'id="tool-count"' in html
  2313. assert 'id="select-all-tools"' in html
  2314. assert 'id="clear-tools"' in html
  2315. assert "tool-card" in js
  2316. assert "tool-description" in js
  2317. assert "tool-required" in js
  2318. assert "function updateToolCount()" in js
  2319. def test_static_app_handles_audit_messages():
  2320. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2321. css = Path("src/agent_lab/presentation/static/styles.css").read_text()
  2322. assert 'message.type === "audit"' in js
  2323. assert "appendAuditEntry(message)" in js
  2324. assert 'appendLog("audit"' not in js
  2325. assert "function renderAuditDetail(" in js
  2326. assert "chat_agent_request" in js
  2327. assert "event_agent_response" in js
  2328. assert "chat_message_stream_started" in js
  2329. assert "chat_message_stream_finished" in js
  2330. assert ".audit-event" in css
  2331. assert ".audit-detail" in css
  2332. def test_static_audit_replay_uses_event_stream_and_detail_panel():
  2333. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2334. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2335. css = Path("src/agent_lab/presentation/static/styles.css").read_text()
  2336. assert '<link rel="stylesheet" href="/static/styles.css?v=' in html
  2337. assert "20260714-professional-console" in html
  2338. assert 'id="audit-dialog"' in html
  2339. assert 'id="open-audit-dialog"' in html
  2340. assert 'id="close-audit-dialog"' in html
  2341. assert 'id="audit-summary"' in html
  2342. assert 'id="audit-replay" class="audit-stream"' in html
  2343. assert 'id="audit-detail"' in html
  2344. assert "auditDialog.showModal()" in js
  2345. assert "auditDialog.close()" in js
  2346. assert "auditEntryKey(entry)" in js
  2347. assert "selectAuditEntry(key)" in js
  2348. assert 'meta.className = "audit-detail-meta"' in js
  2349. assert 'appendMetaField(meta, "Since User Message"' in js
  2350. assert "auditRelativeTime(entry)" in js
  2351. assert 'const list = document.createElement("ul")' in js
  2352. assert "appendKeyValueSection(auditDetail, \"Model Parameters\"" in js
  2353. assert "appendRepliesSection(auditDetail, details.replies)" in js
  2354. assert "appendRawChunksSection(auditDetail, \"Raw LLM Chunks\"" in js
  2355. assert "appendRawModelChunksSection(auditDetail, details.raw_model_chunks)" in js
  2356. assert "Enabled This Round" in js
  2357. assert "Configured Events" in js
  2358. assert "details.configured_events" in js
  2359. assert "enabled this round" in js
  2360. assert ".audit-modal" in css
  2361. assert ".audit-modal-body" in css
  2362. assert ".audit-modal-body section + section" in css
  2363. assert ".audit-detail-title" in css
  2364. assert ".detail-tags code" in css
  2365. assert ".audit-detail-meta span" not in css
  2366. assert ".detail-tags span" not in css
  2367. assert "display: block" in _css_rule(css, ".audit-detail")
  2368. assert "display: grid" not in _css_rule(css, ".audit-detail")
  2369. assert "display: block" in _css_rule(css, ".audit-detail-header")
  2370. assert "display: grid" not in _css_rule(css, ".audit-detail-header")
  2371. assert "display: inline-flex" in _css_rule(css, ".audit-detail-meta div")
  2372. assert ".audit-marker::before" in css
  2373. assert ".audit-event.is-selected" in css
  2374. def test_static_layout_keeps_controls_and_composer_fixed_with_scrollable_panes():
  2375. css = Path("src/agent_lab/presentation/static/styles.css").read_text()
  2376. layout_rule = _css_rule(css, ".layout")
  2377. closed_rule = _css_rule(css, ".layout.debug-drawer-closed")
  2378. workspace_rule = _css_rule(css, ".workspace")
  2379. assert "height: calc(100vh - 64px)" in layout_rule
  2380. assert (
  2381. "grid-template-columns: minmax(260px, 320px) minmax(0, 1fr) "
  2382. "minmax(300px, 380px)"
  2383. ) in layout_rule
  2384. assert "overflow: hidden" in layout_rule
  2385. assert "grid-template-columns: minmax(260px, 320px) minmax(0, 1fr) 0" in closed_rule
  2386. assert "overflow: auto" in _css_rule(css, ".controls")
  2387. assert "grid-template-rows: minmax(0, 1fr) auto auto" in workspace_rule
  2388. assert "overflow: hidden" in _css_rule(css, ".workspace-body")
  2389. assert "overflow: auto" in _css_rule(css, ".messages")
  2390. assert "overflow: auto" in _css_rule(css, ".debug-drawer")
  2391. responsive_css = css[css.index("@media (max-width: 900px)") :]
  2392. assert "height: auto" in responsive_css
  2393. assert "overflow-x: hidden" in responsive_css
  2394. assert ".debug-drawer[hidden]" in responsive_css
  2395. def test_static_app_shows_wait_state_and_immediate_user_echo():
  2396. html = Path("src/agent_lab/presentation/static/index.html").read_text()
  2397. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2398. css = Path("src/agent_lab/presentation/static/styles.css").read_text()
  2399. assert 'id="run-button"' in html
  2400. assert 'const runButton = document.querySelector("#run-button")' in js
  2401. assert 'appendLog("user", submittedMessage)' in js
  2402. assert 'statusEl.textContent = "Waiting for first token"' in js
  2403. assert "function startElapsedTimer()" in js
  2404. assert "function stopElapsedTimer()" in js
  2405. assert "function setRunState(" in js
  2406. assert "if (firstTokenAt) {\n hasBackendRoundStats = true;" in js
  2407. assert "hasBackendRoundStats && firstTokenAt" in js
  2408. assert ".user" in css
  2409. def test_static_app_reuses_websocket_for_session_turns():
  2410. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2411. assert "function isSocketOpen()" in js
  2412. assert "socket.send(JSON.stringify({ type: \"user_message\"" in js
  2413. assert "session_id: currentSessionId || undefined" in js
  2414. assert "message.session_id" in js
  2415. assert 'message.type === "turn_completed"' in js
  2416. assert 'message.type === "done"' in js
  2417. def test_static_app_loads_session_replay_and_usage():
  2418. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2419. create_session = js[
  2420. js.index("async function createSession()") :
  2421. js.index("async function loadSelectedSession()")
  2422. ]
  2423. load_session = js[
  2424. js.index("async function loadSelectedSession()") :
  2425. js.index("async function loadSessionReplay(")
  2426. ]
  2427. assert "function loadSessions(" in js
  2428. assert "function createSession(" in js
  2429. assert "function setSessionStatus(" in js
  2430. assert '"Refreshing sessions"' in js
  2431. assert '"Creating session"' in js
  2432. assert '"Session created"' in js
  2433. assert 'sessionStatus.textContent = message' in js
  2434. assert "function loadSessionReplay(" in js
  2435. assert 'fetchJson("/api/sessions")' in js
  2436. assert 'method: "POST"' in js
  2437. assert '`/api/sessions/${sessionId}/messages`' in js
  2438. assert '`/api/sessions/${sessionId}/audit`' in js
  2439. assert '`/api/sessions/${sessionId}/usage`' in js
  2440. assert "function renderAuditReplay(" in js
  2441. assert "function renderSessionUsage(" in js
  2442. assert "session.turn_wall_time_ms ?? session.elapsed_ms" in js
  2443. assert "turn.turn_wall_time_ms ?? turn.elapsed_ms" in js
  2444. assert "call.mode" in js
  2445. assert "call.agent" in js
  2446. assert "call.call_kind" in js
  2447. assert "call.used_fallback" in js
  2448. assert "call.tool_latency_ms" in js
  2449. assert "sessionUsageSummary" in js
  2450. assert "Finish current turn before switching sessions" in js
  2451. assert "await loadSessionReplay(session.id, { preserveStatus: true });" in create_session
  2452. assert "await loadSessionReplay(sessionId);" in load_session
  2453. def test_static_session_replay_ignores_superseded_success():
  2454. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2455. result = _run_load_session_replay_scenario(
  2456. """
  2457. currentSessionId = "old";
  2458. const oldLoad = loadSessionReplay("old");
  2459. currentSessionId = "new";
  2460. const newLoad = loadSessionReplay("new");
  2461. resolveReplay("new");
  2462. await newLoad;
  2463. resolveReplay("old");
  2464. await oldLoad;
  2465. console.log(JSON.stringify({
  2466. ...mutations,
  2467. title: sessionTitle.value,
  2468. selected: sessionList.value,
  2469. }));
  2470. """
  2471. )
  2472. assert result == {
  2473. "workspace": ["new"],
  2474. "transcript": ["new"],
  2475. "audit": ["new"],
  2476. "usage": ["new"],
  2477. "status": ["Session loaded"],
  2478. "title": "new title",
  2479. "selected": "new",
  2480. }
  2481. assert "let sessionReplayLoadToken = 0;" in js
  2482. def test_static_session_replay_ignores_superseded_failure_status():
  2483. result = _run_load_session_replay_scenario(
  2484. """
  2485. currentSessionId = "old";
  2486. const oldLoad = loadSessionReplay("old");
  2487. currentSessionId = "new";
  2488. const newLoad = loadSessionReplay("new");
  2489. resolveReplay("new");
  2490. await newLoad;
  2491. pending.get("/api/sessions/old").reject(new Error("stale failure"));
  2492. await oldLoad;
  2493. console.log(JSON.stringify(mutations.status));
  2494. """
  2495. )
  2496. assert result == ["Session loaded"]
  2497. def test_static_session_replay_rejects_stale_invocation_before_token_advance():
  2498. result = _run_load_session_replay_scenario(
  2499. """
  2500. currentSessionId = "new";
  2501. const newLoad = loadSessionReplay("new");
  2502. const tokenAfterNew = sessionReplayLoadToken;
  2503. const oldLoad = loadSessionReplay("old");
  2504. const oldResult = await Promise.race([
  2505. oldLoad.then(() => "returned"),
  2506. new Promise((resolve) => setTimeout(() => resolve("pending"), 0)),
  2507. ]);
  2508. const tokenAfterOld = sessionReplayLoadToken;
  2509. const oldFetchStarted = pending.has("/api/sessions/old");
  2510. resolveReplay("new");
  2511. await newLoad;
  2512. console.log(JSON.stringify({
  2513. oldResult,
  2514. tokenAfterNew,
  2515. tokenAfterOld,
  2516. oldFetchStarted,
  2517. mutations,
  2518. title: sessionTitle.value,
  2519. selected: sessionList.value,
  2520. }));
  2521. """
  2522. )
  2523. assert result == {
  2524. "oldResult": "returned",
  2525. "tokenAfterNew": 1,
  2526. "tokenAfterOld": 1,
  2527. "oldFetchStarted": False,
  2528. "mutations": {
  2529. "workspace": ["new"],
  2530. "transcript": ["new"],
  2531. "audit": ["new"],
  2532. "usage": ["new"],
  2533. "status": ["Session loaded"],
  2534. },
  2535. "title": "new title",
  2536. "selected": "new",
  2537. }
  2538. def test_static_replay_hides_whitespace_assistant_tool_protocol_records():
  2539. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2540. render_replay = js[
  2541. js.index("function renderReplayMessages(") :
  2542. js.index("function appendAuditEntry(")
  2543. ]
  2544. result = _run_node_json(
  2545. "\n".join(
  2546. [
  2547. """
  2548. let activeAssistant = null;
  2549. const messagesEl = {
  2550. children: [],
  2551. scrollHeight: 0,
  2552. scrollTop: 0,
  2553. set textContent(value) {
  2554. if (value === "") {
  2555. this.children = [];
  2556. }
  2557. },
  2558. append(item) {
  2559. this.children.push(item);
  2560. },
  2561. };
  2562. const document = {
  2563. createElement() {
  2564. return { className: "", textContent: "" };
  2565. },
  2566. };
  2567. function appendLog(kind, content) {
  2568. messagesEl.append({ className: `message ${kind}`, textContent: content });
  2569. }
  2570. """,
  2571. render_replay,
  2572. """
  2573. renderReplayMessages([
  2574. { role: "assistant", content: " \\n\\t", tool_calls: [{ id: "call-1" }] },
  2575. { role: "assistant", content: "visible", tool_calls: [] },
  2576. ]);
  2577. console.log(JSON.stringify(messagesEl.children.map((item) => item.textContent)));
  2578. """,
  2579. ]
  2580. )
  2581. )
  2582. assert result == ["visible"]
  2583. def test_static_session_switch_guard_uses_active_turn_not_open_socket():
  2584. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2585. create_session = js[js.index("async function createSession()") : js.index("async function loadSelectedSession()")]
  2586. load_session = js[js.index("async function loadSelectedSession()") : js.index("async function loadSessionReplay(")]
  2587. assert "function hasActiveTurn()" in js
  2588. assert "turnActive = isRunning" in js
  2589. assert "if (hasActiveTurn())" in create_session
  2590. assert "if (hasActiveTurn())" in load_session
  2591. assert "if (isSocketOpen())" not in create_session
  2592. assert "if (isSocketOpen())" not in load_session
  2593. assert "closeSocket();" in create_session
  2594. assert "closeSocket();" in load_session
  2595. assert "const activeSocket = socket" in js
  2596. assert "if (socket !== activeSocket)" in js
  2597. def test_static_workspace_snapshot_uses_stable_storage_hooks():
  2598. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2599. assert "agent-lab.workspace-snapshots.v1" in js
  2600. assert "agent-lab.prompt-sets.v1" in js
  2601. assert "function saveWorkspaceSnapshot()" in js
  2602. assert "function loadWorkspaceSnapshot()" in js
  2603. assert "function deleteWorkspaceSnapshot()" in js
  2604. assert "function buildWorkspaceSnapshot()" in js
  2605. assert "prompt_items: buildPromptItems()" in js
  2606. assert "enabled_tools: selectedTools()" in js
  2607. assert "extra_body: buildExtraBody(\"chat\")" in js
  2608. assert "extra_body: buildExtraBody(\"event\")" in js
  2609. def test_static_app_handles_backend_round_stats_messages():
  2610. js = Path("src/agent_lab/presentation/static/app.js").read_text()
  2611. assert 'message.type === "round_stats"' in js
  2612. assert "function updateRoundStats(" in js
  2613. assert "#stat-ttft" in js
  2614. assert "#stat-elapsed" in js
  2615. @pytest.mark.asyncio
  2616. async def test_openai_chat_client_omits_stream_options_when_usage_disabled():
  2617. captured_payloads: list[dict] = []
  2618. def handler(request: httpx.Request) -> httpx.Response:
  2619. captured_payloads.append(json.loads(request.content))
  2620. return httpx.Response(
  2621. 200,
  2622. content=b'data: {"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}\n\n',
  2623. )
  2624. async with httpx.AsyncClient(
  2625. transport=httpx.MockTransport(handler),
  2626. base_url="https://llm.test/v1",
  2627. ) as http_client:
  2628. client = OpenAICompatibleChatClient(
  2629. api_key="",
  2630. base_url="https://llm.test/v1",
  2631. default_model="provider-default",
  2632. request_timeout_seconds=5,
  2633. include_usage=False,
  2634. http_client=http_client,
  2635. )
  2636. [
  2637. item
  2638. async for item in client.stream_chat(
  2639. messages=[ChatMessage(role="user", content="hi")],
  2640. tools=[],
  2641. params=AgentParams(model="provider-default", temperature=0.3, max_tokens=50),
  2642. )
  2643. ]
  2644. assert "stream_options" not in captured_payloads[0]
  2645. @pytest.mark.asyncio
  2646. async def test_openai_chat_client_merges_agent_extra_body_into_payload():
  2647. captured_payloads: list[dict] = []
  2648. def handler(request: httpx.Request) -> httpx.Response:
  2649. captured_payloads.append(json.loads(request.content))
  2650. return httpx.Response(
  2651. 200,
  2652. content=b'data: {"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}\n\n',
  2653. )
  2654. async with httpx.AsyncClient(
  2655. transport=httpx.MockTransport(handler),
  2656. base_url="https://llm.test/v1",
  2657. ) as http_client:
  2658. client = OpenAICompatibleChatClient(
  2659. api_key="",
  2660. base_url="https://llm.test/v1",
  2661. default_model="provider-default",
  2662. request_timeout_seconds=5,
  2663. http_client=http_client,
  2664. )
  2665. [
  2666. item
  2667. async for item in client.stream_chat(
  2668. messages=[ChatMessage(role="user", content="hi")],
  2669. tools=[],
  2670. params=AgentParams(
  2671. model="provider-default",
  2672. temperature=0.3,
  2673. max_tokens=50,
  2674. extra_body={
  2675. "thinking": {"type": "disabled"},
  2676. "enable_search": False,
  2677. "search_options": {"forced_search": False},
  2678. },
  2679. ),
  2680. )
  2681. ]
  2682. assert captured_payloads[0]["thinking"] == {"type": "disabled"}
  2683. assert captured_payloads[0]["enable_search"] is False
  2684. assert captured_payloads[0]["search_options"] == {"forced_search": False}
  2685. @pytest.mark.asyncio
  2686. @pytest.mark.parametrize(
  2687. ("reserved_key", "override_value"),
  2688. [
  2689. ("model", "overridden-model"),
  2690. ("messages", []),
  2691. ("tools", []),
  2692. (
  2693. "tool_choice",
  2694. {"type": "function", "function": {"name": "other_tool"}},
  2695. ),
  2696. ("stream", False),
  2697. ("stream_options", {"include_usage": False}),
  2698. ("temperature", 1.0),
  2699. ("max_tokens", 1),
  2700. ],
  2701. )
  2702. async def test_openai_chat_client_rejects_reserved_extra_body_fields_before_network(
  2703. reserved_key: str,
  2704. override_value: object,
  2705. ):
  2706. network_called = False
  2707. def handler(request: httpx.Request) -> httpx.Response:
  2708. nonlocal network_called
  2709. network_called = True
  2710. return httpx.Response(200)
  2711. async with httpx.AsyncClient(
  2712. transport=httpx.MockTransport(handler),
  2713. base_url="https://llm.test/v1",
  2714. ) as http_client:
  2715. client = OpenAICompatibleChatClient(
  2716. api_key="",
  2717. base_url="https://llm.test/v1",
  2718. default_model="provider-default",
  2719. request_timeout_seconds=5,
  2720. http_client=http_client,
  2721. )
  2722. with pytest.raises(
  2723. ValueError,
  2724. match=f"extra_body contains reserved field: {reserved_key}",
  2725. ):
  2726. [
  2727. item
  2728. async for item in client.stream_chat(
  2729. messages=[ChatMessage(role="user", content="hi")],
  2730. tools=[],
  2731. params=AgentParams(extra_body={reserved_key: override_value}),
  2732. )
  2733. ]
  2734. assert network_called is False