from __future__ import annotations import asyncio import json import threading from collections.abc import Callable from enum import IntEnum from typing import Any import pytest import agent_lab.application.events as events_module from agent_lab.application.events import ( EventDefinition, EventKernel, EventRegistry, EventRequest, EventSource, EventStatus, ResultPolicy, ) from agent_lab.application.tools import ToolDefinition, ToolRegistry from agent_lab.domain.events import ToolCallEvent def _executor_type(): executor_type = getattr(events_module, "EventBatchExecutor", None) assert executor_type is not None, "EventBatchExecutor is not implemented" return executor_type def _definition( name: str, handler: Callable[[EventRequest], Any], *, conflict_keys: tuple[str, ...] = (), timeout_seconds: float | None = None, terminal: bool = False, normalizer: Callable[[dict[str, Any]], dict[str, Any]] | None = None, ) -> EventDefinition: return EventDefinition( name=name, description=f"Execute {name}.", parameters={"type": "object"}, handler=handler, conflict_keys=conflict_keys, timeout_seconds=timeout_seconds, terminal=terminal, normalizer=normalizer, ) def _request(event_id: str, name: str, arguments: dict[str, Any] | None = None): return EventRequest( id=event_id, name=name, arguments=arguments or {}, ) def _executor( definitions: list[EventDefinition], *, max_parallel_events: int = 4, batch_timeout_seconds: float = 1.0, **options: Any, ): return _executor_type()( EventKernel(EventRegistry(definitions)), max_parallel_events=max_parallel_events, batch_timeout_seconds=batch_timeout_seconds, **options, ) @pytest.mark.asyncio async def test_batch_executor_bounds_global_parallelism(): active = 0 max_active = 0 async def handler(request: EventRequest) -> dict[str, Any]: nonlocal active, max_active active += 1 max_active = max(max_active, active) await asyncio.sleep(0.02) active -= 1 return {"event_id": request.id} executor = _executor( [_definition("example.work", handler)], max_parallel_events=2, ) batch = await executor.execute( [ _request(str(index), "example.work", {"index": index}) for index in range(4) ], scope="session-1:turn-1", ) assert max_active == 2 assert [result.status for result in batch.results] == [ EventStatus.SUCCESS, ] * 4 @pytest.mark.asyncio async def test_batch_executor_preserves_input_order_when_completion_order_differs(): release_first = asyncio.Event() second_finished = asyncio.Event() async def handler(request: EventRequest) -> dict[str, Any]: if request.id == "first": await release_first.wait() else: second_finished.set() return {"event_id": request.id} executor = _executor([_definition("example.work", handler)]) task = asyncio.create_task( executor.execute( [ _request("first", "example.work", {"order": 1}), _request("second", "example.work", {"order": 2}), ], scope="session-1:turn-1", ) ) await asyncio.wait_for(second_finished.wait(), timeout=0.2) release_first.set() batch = await asyncio.wait_for(task, timeout=0.2) assert [result.event_id for result in batch.results] == ["first", "second"] @pytest.mark.asyncio async def test_batch_executor_serializes_shared_conflict_keys(): active = 0 max_active = 0 async def handler(request: EventRequest) -> dict[str, Any]: nonlocal active, max_active active += 1 max_active = max(max_active, active) await asyncio.sleep(0.02) active -= 1 return {"event_id": request.id} executor = _executor( [ _definition( "example.write", handler, conflict_keys=("shared-resource",), ) ] ) await executor.execute( [ _request("first", "example.write", {"order": 1}), _request("second", "example.write", {"order": 2}), ], scope="session-1:turn-1", ) assert max_active == 1 @pytest.mark.asyncio async def test_batch_executor_isolates_per_event_timeout_from_successful_sibling(): async def slow_handler(request: EventRequest) -> dict[str, Any]: await asyncio.sleep(1) return {"event_id": request.id} executor = _executor( [ _definition( "example.slow", slow_handler, timeout_seconds=0.01, ), _definition( "example.fast", lambda request: {"event_id": request.id}, ), ], batch_timeout_seconds=0.2, ) batch = await executor.execute( [ _request("slow", "example.slow"), _request("fast", "example.fast"), ], scope="session-1:turn-1", ) assert [result.status for result in batch.results] == [ EventStatus.TIMEOUT, EventStatus.SUCCESS, ] assert batch.deadline_exceeded is False @pytest.mark.asyncio async def test_batch_executor_applies_overall_deadline_to_waiting_work(): async def blocked_handler(request: EventRequest) -> dict[str, Any]: await asyncio.Event().wait() return {"event_id": request.id} executor = _executor( [_definition("example.blocked", blocked_handler)], max_parallel_events=1, batch_timeout_seconds=0.02, ) batch = await executor.execute( [ _request("first", "example.blocked"), _request("second", "example.blocked"), ], scope="session-1:turn-1", ) assert [result.status for result in batch.results] == [ EventStatus.TIMEOUT, EventStatus.TIMEOUT, ] assert batch.deadline_exceeded is True @pytest.mark.asyncio async def test_batch_executor_coalesces_exact_duplicates_inside_one_batch(): calls = 0 def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"event_id": request.id, "arguments": request.arguments} executor = _executor([_definition("example.write", handler)]) batch = await executor.execute( [ _request("same", "example.write", {"a": 1, "b": 2}), _request("same", "example.write", {"b": 2, "a": 1}), ], scope="session-1:turn-1", ) assert calls == 1 assert batch.coalesced_count == 1 assert batch.results[0].event_id == batch.results[1].event_id == "same" assert batch.results[0].payload == batch.results[1].payload assert batch.results[0].deduplicated_from is None assert batch.results[1].deduplicated_from == "same" @pytest.mark.asyncio async def test_batch_executor_replay_key_includes_scope_event_id_and_request(): calls = 0 def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"call": calls, "arguments": request.arguments} executor = _executor([_definition("example.write", handler)]) original = _request("same", "example.write", {"a": 1, "b": 2}) canonical_replay = _request( "same", "example.write", {"b": 2, "a": 1}, ) first = await executor.execute([original], scope="session-1:turn-1") replay = await executor.execute( [canonical_replay], scope="session-1:turn-1", ) changed_request = await executor.execute( [_request("same", "example.write", {"a": 2})], scope="session-1:turn-1", ) changed_id = await executor.execute( [_request("other", "example.write", {"a": 1, "b": 2})], scope="session-1:turn-1", ) changed_scope = await executor.execute( [original], scope="session-1:turn-2", ) assert first.results[0] == replay.results[0] assert replay.replayed_count == 1 assert changed_request.replayed_count == 0 assert changed_id.replayed_count == 0 assert changed_scope.replayed_count == 0 assert calls == 4 @pytest.mark.asyncio async def test_batch_executor_runs_terminal_events_after_all_nonterminal_results(): observed: list[str] = [] async def nonterminal(request: EventRequest) -> dict[str, Any]: observed.append(f"start:{request.id}") await asyncio.sleep(0.01) observed.append(f"finish:{request.id}") return {"event_id": request.id} def terminal(request: EventRequest) -> dict[str, Any]: observed.append(f"terminal:{request.id}") return {"event_id": request.id} executor = _executor( [ _definition("example.work", nonterminal), _definition("example.terminate", terminal, terminal=True), ] ) batch = await executor.execute( [ _request("terminate", "example.terminate"), _request("first", "example.work", {"order": 1}), _request("second", "example.work", {"order": 2}), ], scope="session-1:turn-1", ) terminal_index = observed.index("terminal:terminate") assert observed.index("finish:first") < terminal_index assert observed.index("finish:second") < terminal_index assert [result.event_id for result in batch.results] == [ "terminate", "first", "second", ] @pytest.mark.asyncio async def test_batch_executor_does_not_swallow_external_cancellation(): started = asyncio.Event() async def handler(request: EventRequest) -> dict[str, Any]: started.set() await asyncio.Event().wait() return {"event_id": request.id} executor = _executor([_definition("example.blocked", handler)]) task = asyncio.create_task( executor.execute( [_request("blocked", "example.blocked")], scope="session-1:turn-1", ) ) await asyncio.wait_for(started.wait(), timeout=0.2) task.cancel() with pytest.raises(asyncio.CancelledError): await task @pytest.mark.asyncio async def test_batch_executor_coalesces_normalized_provider_arguments_across_ids(): calls: list[EventRequest] = [] def normalize(arguments: dict[str, Any]) -> dict[str, Any]: value = arguments["value"] return {"value": int(value) if isinstance(value, float) else value} executor = _executor( [ _definition( "example.normalized", lambda request: calls.append(request) or {"value": request.arguments["value"]}, normalizer=normalize, ) ] ) requests = [ EventRequest( id="integer-id", name="example.normalized", arguments={"value": 30}, source=EventSource.PROVIDER_RESOLVED, raw_arguments='{"value":30}', ), EventRequest( id="float-id", name="example.normalized", arguments={"value": 30.0}, source=EventSource.PROVIDER_RESOLVED, raw_arguments='{"value":30.0}', ), ] batch = await executor.execute(requests, scope="normalized-scope") assert len(calls) == 1 assert batch.coalesced_count == 1 assert [result.event_id for result in batch.results] == [ "integer-id", "float-id", ] assert batch.results[0].payload == {"value": 30} assert batch.results[1].payload == {"value": 30} assert getattr(batch.results[0], "deduplicated_from", None) is None assert getattr(batch.results[1], "deduplicated_from", None) == "integer-id" @pytest.mark.asyncio async def test_coalesced_clone_rewrites_only_correlated_payload_event_id(): correlated = _executor( [ _definition( "example.correlated", lambda request: {"event_id": request.id, "value": "same"}, ) ] ) correlated_batch = await correlated.execute( [ EventRequest( id="primary", name="example.correlated", arguments={}, raw_arguments='{"call":"primary"}', ), EventRequest( id="duplicate", name="example.correlated", arguments={}, raw_arguments='{"call":"duplicate"}', ), ], scope="correlated-payload", ) primary, duplicate = correlated_batch.results assert primary.event_id == "primary" assert duplicate.event_id == "duplicate" assert duplicate.raw_arguments == '{"call":"duplicate"}' assert duplicate.payload == {"event_id": "duplicate", "value": "same"} assert getattr(duplicate, "deduplicated_from", None) == "primary" domain = _executor( [ _definition( "example.domain", lambda request: {"event_id": "domain-object"}, ) ] ) domain_batch = await domain.execute( [ _request("primary", "example.domain"), _request("duplicate", "example.domain"), ], scope="domain-payload", ) assert domain_batch.results[1].payload == {"event_id": "domain-object"} assert getattr(domain_batch.results[1], "deduplicated_from", None) == "primary" @pytest.mark.asyncio async def test_batch_executor_tool_replies_keep_each_coalesced_call_id(): registry = ToolRegistry( [ ToolDefinition( name="example.tool", description="Execute a coalesced tool.", parameters={"type": "object"}, handler=lambda event: {"event_id": event.id}, ) ] ) executor = _executor_type()( EventKernel(registry.event_registry), batch_timeout_seconds=1, ) events = [ ToolCallEvent(id=event_id, name="example.tool", arguments={}, raw_arguments="{}") for event_id in ("first-id", "second-id") ] batch = await executor.execute( [ registry.event_request(event, source=EventSource.PROVIDER_RESOLVED) for event in events ], scope="tool-scope", ) replies = registry.tool_replies(events, batch) assert [reply.tool_call_id for reply in replies] == ["first-id", "second-id"] assert [json.loads(reply.content) for reply in replies] == [ {"event_id": "first-id"}, {"event_id": "second-id"}, ] assert [result.event_id for result in batch.results] == [ "first-id", "second-id", ] @pytest.mark.asyncio async def test_batch_policy_and_compact_summary_skip_logical_duplicates(): registry = ToolRegistry( [ ToolDefinition( name="example.template", description="Render one template.", parameters={"type": "object"}, handler=lambda event: {"event_id": event.id}, result_policy=ResultPolicy.TEMPLATE_FOLLOW_UP, result_message_factory=lambda result: "Completed once.", ) ] ) executor = _executor_type()(EventKernel(registry.event_registry)) events = [ ToolCallEvent( id=event_id, name="example.template", arguments={}, raw_arguments="{}", ) for event_id in ("primary", "duplicate") ] batch = await executor.execute( [registry.event_request(event) for event in events], scope="policy-dedupe", ) decision = registry.batch_decision(batch) summary = registry.compact_results_message(batch) assert decision.template_messages == ("Completed once.",) assert summary is not None assert len(summary.content.splitlines()) == 2 @pytest.mark.asyncio async def test_batch_key_falls_back_to_original_arguments_when_normalizer_fails(): calls = 0 def normalize(arguments: dict[str, Any]) -> dict[str, Any]: if isinstance(arguments.get("value"), str): raise ValueError("unsupported value") return {"value": int(arguments["value"])} def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"value": request.arguments["value"]} executor = _executor( [_definition("example.normalized", handler, normalizer=normalize)] ) batch = await executor.execute( [ EventRequest( id="valid", name="example.normalized", arguments={"value": 30}, source=EventSource.PROVIDER_RESOLVED, ), EventRequest( id="invalid", name="example.normalized", arguments={"value": "30"}, source=EventSource.PROVIDER_RESOLVED, ), ], scope="normalizer-failure", ) assert batch.coalesced_count == 0 assert [result.status for result in batch.results] == [ EventStatus.SUCCESS, EventStatus.INVALID_ARGUMENTS, ] assert calls == 1 @pytest.mark.asyncio @pytest.mark.parametrize( ("invalid_arguments", "raw_arguments"), [ ({"value": float("nan")}, '{"value":NaN}'), ({"value": object()}, '{"value":"object"}'), ], ) async def test_invalid_json_arguments_do_not_coalesce_with_valid_empty_object( invalid_arguments: dict[str, Any], raw_arguments: str, ): calls = 0 def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"event_id": request.id} executor = _executor([_definition("example.strict", handler)]) batch = await executor.execute( [ EventRequest( id="valid", name="example.strict", arguments={}, source=EventSource.PROVIDER_RESOLVED, raw_arguments="{}", ), EventRequest( id="invalid", name="example.strict", arguments=invalid_arguments, source=EventSource.PROVIDER_RESOLVED, raw_arguments=raw_arguments, ), ], scope="strict-json-batch", ) assert batch.coalesced_count == 0 assert [result.status for result in batch.results] == [ EventStatus.SUCCESS, EventStatus.INVALID_ARGUMENTS, ] assert calls == 1 @pytest.mark.asyncio @pytest.mark.parametrize( ("invalid_arguments", "raw_arguments"), [ ({"value": float("nan")}, '{"value":NaN}'), ({"value": object()}, '{"value":"object"}'), ], ) async def test_invalid_json_arguments_do_not_replay_valid_success( invalid_arguments: dict[str, Any], raw_arguments: str, ): executor = _executor( [_definition("example.strict", lambda request: {"event_id": request.id})] ) valid = EventRequest( id="same-id", name="example.strict", arguments={}, source=EventSource.PROVIDER_RESOLVED, raw_arguments="{}", ) invalid = EventRequest( id="same-id", name="example.strict", arguments=invalid_arguments, source=EventSource.PROVIDER_RESOLVED, raw_arguments=raw_arguments, ) first = await executor.execute([valid], scope="strict-json-replay") second = await executor.execute([invalid], scope="strict-json-replay") assert first.results[0].status is EventStatus.SUCCESS assert second.replayed_count == 0 assert second.results[0].status is EventStatus.INVALID_ARGUMENTS @pytest.mark.asyncio async def test_non_json_normalizer_output_has_distinct_canonical_key(): calls = 0 def normalize(arguments: dict[str, Any]) -> dict[str, Any]: if arguments.get("invalid"): return {"value": object()} return {} def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"event_id": request.id} executor = _executor( [_definition("example.normalized", handler, normalizer=normalize)] ) batch = await executor.execute( [ EventRequest( id="valid", name="example.normalized", arguments={}, source=EventSource.PROVIDER_RESOLVED, ), EventRequest( id="invalid", name="example.normalized", arguments={"invalid": True}, source=EventSource.PROVIDER_RESOLVED, ), ], scope="normalizer-invalid-json", ) assert batch.coalesced_count == 0 assert [result.status for result in batch.results] == [ EventStatus.SUCCESS, EventStatus.INVALID_ARGUMENTS, ] assert calls == 1 @pytest.mark.asyncio @pytest.mark.parametrize( "wrap", [ pytest.param(lambda value: {"value": value}, id="top-level"), pytest.param( lambda value: {"outer": {"value": value}}, id="nested-dict", ), pytest.param( lambda value: {"outer": [value]}, id="nested-list", ), ], ) async def test_canonical_json_equality_requires_recursive_exact_types(wrap): class NumericEnum(IntEnum): ONE = 1 calls: list[str] = [] def handler(request: EventRequest) -> dict[str, Any]: calls.append(request.id) return {"event_id": request.id} executor = _executor([_definition("example.strict-types", handler)]) values = [NumericEnum.ONE, True, 1, 1.0] requests = [ EventRequest( id=event_id, name="example.strict-types", arguments=wrap(value), source=EventSource.PROVIDER_RESOLVED, raw_arguments=json.dumps(wrap(value)), ) for event_id, value in zip( ("enum", "bool", "int", "float"), values, strict=True, ) ] batch = await executor.execute(requests, scope=f"strict-types-{id(wrap)}") assert batch.coalesced_count == 0 assert [result.status for result in batch.results] == [ EventStatus.INVALID_ARGUMENTS, EventStatus.SUCCESS, EventStatus.SUCCESS, EventStatus.SUCCESS, ] assert calls == ["bool", "int", "float"] @pytest.mark.asyncio async def test_normalizer_explicitly_coalesces_valid_numeric_types_to_int(): class NumericEnum(IntEnum): ONE = 1 calls = 0 def normalize(arguments: dict[str, Any]) -> dict[str, Any]: return {"value": int(arguments["value"])} def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"event_id": request.id, "value": request.arguments["value"]} executor = _executor( [ _definition( "example.normalized-types", handler, normalizer=normalize, ) ] ) requests = [ EventRequest( id=event_id, name="example.normalized-types", arguments={"value": value}, source=EventSource.PROVIDER_RESOLVED, raw_arguments=json.dumps({"value": value}), ) for event_id, value in zip( ("enum", "bool", "int", "float"), (NumericEnum.ONE, True, 1, 1.0), strict=True, ) ] batch = await executor.execute(requests, scope="normalized-strict-types") assert batch.coalesced_count == 2 assert [result.status for result in batch.results] == [ EventStatus.INVALID_ARGUMENTS, EventStatus.SUCCESS, EventStatus.SUCCESS, EventStatus.SUCCESS, ] assert calls == 1 @pytest.mark.asyncio async def test_concurrent_replay_waiters_share_execution_and_cancel_independently(): calls = 0 started = asyncio.Event() release = asyncio.Event() async def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 started.set() await release.wait() return {"event_id": request.id} executor = _executor([_definition("example.shared", handler)]) request = _request("same-id", "example.shared") first = asyncio.create_task(executor.execute([request], scope="shared-scope")) second = asyncio.create_task(executor.execute([request], scope="shared-scope")) await asyncio.wait_for(started.wait(), timeout=0.2) first.cancel() with pytest.raises(asyncio.CancelledError): await first release.set() second_batch = await asyncio.wait_for(second, timeout=0.2) assert calls == 1 assert second_batch.results[0].status is EventStatus.SUCCESS scope_tokens = getattr(executor, "_scope_tokens", None) assert scope_tokens is not None assert len(scope_tokens) == 1 assert scope_tokens["shared-scope"] > 0 @pytest.mark.asyncio async def test_scope_tokens_are_globally_unique_removed_and_never_reused(): async def handler(request: EventRequest) -> dict[str, Any]: return {"event_id": request.id} executor = _executor([_definition("example.token", handler)]) request = _request("same-id", "example.token") await executor.execute([request], scope="scope-a") await executor.execute([request], scope="scope-b") scope_tokens = getattr(executor, "_scope_tokens", None) assert scope_tokens is not None first_a = scope_tokens["scope-a"] first_b = scope_tokens["scope-b"] assert first_a != first_b executor.release_scope("scope-a") assert "scope-a" not in scope_tokens await executor.execute([request], scope="scope-a") second_a = scope_tokens["scope-a"] assert second_a not in {first_a, first_b} executor.release_scope("scope-a") executor.release_scope("scope-b") assert scope_tokens == {} @pytest.mark.asyncio async def test_release_scope_removes_replay_entries_for_reuse(): calls = 0 def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls calls += 1 return {"call": calls} executor = _executor([_definition("example.cached", handler)]) request = _request("same-id", "example.cached") await executor.execute([request], scope="released-scope") executor.release_scope("released-scope") batch = await executor.execute([request], scope="released-scope") assert calls == 2 assert batch.replayed_count == 0 @pytest.mark.asyncio async def test_release_scope_starts_new_generation_before_old_task_finishes(): calls = 0 started = [asyncio.Event(), asyncio.Event()] releases = [asyncio.Event(), asyncio.Event()] async def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls call_index = calls calls += 1 started[call_index].set() await releases[call_index].wait() return {"call": call_index + 1, "event_id": request.id} executor = _executor([_definition("example.generated", handler)]) request = _request("same-id", "example.generated") first = asyncio.create_task( executor.execute([request], scope="generation-scope") ) await asyncio.wait_for(started[0].wait(), timeout=0.2) executor.release_scope("generation-scope") second = asyncio.create_task( executor.execute([request], scope="generation-scope") ) await asyncio.sleep(0.02) new_generation_started = started[1].is_set() releases[0].set() releases[1].set() first_batch, second_batch = await asyncio.gather(first, second) assert new_generation_started is True assert first_batch.results[0].payload["call"] == 1 assert second_batch.results[0].payload["call"] == 2 @pytest.mark.asyncio async def test_old_generation_completion_cannot_replace_new_inflight_or_cache(): calls = 0 started = [asyncio.Event(), asyncio.Event(), asyncio.Event()] releases = [asyncio.Event(), asyncio.Event(), asyncio.Event()] async def handler(request: EventRequest) -> dict[str, Any]: nonlocal calls call_index = calls calls += 1 started[call_index].set() await releases[call_index].wait() return {"call": call_index + 1, "event_id": request.id} executor = _executor([_definition("example.generated", handler)]) request = _request("same-id", "example.generated") old = asyncio.create_task(executor.execute([request], scope="reused-scope")) await asyncio.wait_for(started[0].wait(), timeout=0.2) executor.release_scope("reused-scope") current = asyncio.create_task( executor.execute([request], scope="reused-scope") ) await asyncio.sleep(0.02) new_generation_started = started[1].is_set() if not new_generation_started: releases[0].set() await asyncio.gather(old, current) assert new_generation_started is True releases[0].set() old_batch = await asyncio.wait_for(old, timeout=0.2) waiter = asyncio.create_task( executor.execute([request], scope="reused-scope") ) await asyncio.sleep(0.02) assert calls == 2 releases[1].set() current_batch, waiter_batch = await asyncio.gather(current, waiter) replayed = await executor.execute([request], scope="reused-scope") assert old_batch.results[0].payload["call"] == 1 assert current_batch.results[0].payload["call"] == 2 assert waiter_batch.results[0].payload["call"] == 2 assert replayed.replayed_count == 1 assert replayed.results[0].payload["call"] == 2 @pytest.mark.asyncio async def test_replay_cache_stays_bounded_across_250_scopes(): executor = _executor( [_definition("example.cached", lambda request: {"id": request.id})], max_replay_entries=100, ) request = _request("same-id", "example.cached") for index in range(250): await executor.execute([request], scope=f"scope-{index}") assert len(executor._replay_results) <= 100 @pytest.mark.asyncio async def test_thousand_released_scopes_leave_no_token_or_replay_state(): async def handler(request: EventRequest) -> dict[str, Any]: return {"event_id": request.id} executor = _executor([_definition("example.cleanup", handler)]) request = _request("same-id", "example.cleanup") for index in range(1000): scope = f"released-{index}" await executor.execute([request], scope=scope) executor.release_scope(scope) scope_tokens = getattr(executor, "_scope_tokens", None) retained_generations = getattr(executor, "_scope_generations", {}) assert scope_tokens == {} assert retained_generations == {} assert executor._replay_results == {} assert executor._inflight == {} @pytest.mark.asyncio async def test_terminal_uses_independent_grace_after_sibling_batch_timeout(): terminal_calls: list[str] = [] async def blocked(request: EventRequest) -> dict[str, Any]: await asyncio.Event().wait() return {"event_id": request.id} def terminate(request: EventRequest) -> dict[str, Any]: terminal_calls.append(request.id) return {"terminated": True} executor = _executor( [ _definition("example.blocked", blocked), _definition( "example.terminate", terminate, terminal=True, timeout_seconds=0.05, ), ], batch_timeout_seconds=0.01, terminal_grace_seconds=0.1, ) batch = await executor.execute( [ _request("sibling", "example.blocked"), _request("terminal", "example.terminate"), ], scope="terminal-grace", ) assert [result.status for result in batch.results] == [ EventStatus.TIMEOUT, EventStatus.SUCCESS, ] assert batch.deadline_exceeded is True assert terminal_calls == ["terminal"] @pytest.mark.asyncio async def test_terminal_grace_is_bounded_by_terminal_event_timeout(): async def blocked_terminal(request: EventRequest) -> dict[str, Any]: await asyncio.Event().wait() return {"event_id": request.id} executor = _executor( [ _definition( "example.terminate", blocked_terminal, terminal=True, timeout_seconds=0.01, ) ], terminal_grace_seconds=0.2, ) batch = await executor.execute( [_request("terminal", "example.terminate")], scope="terminal-timeout", ) assert batch.results[0].status is EventStatus.TIMEOUT assert batch.results[0].error == "event timed out after 0.01 seconds" assert batch.deadline_exceeded is False @pytest.mark.asyncio async def test_replayed_terminal_grace_timeout_preserves_deadline_flag(): async def blocked_terminal(request: EventRequest) -> dict[str, Any]: await asyncio.Event().wait() return {"event_id": request.id} executor = _executor( [_definition("example.terminate", blocked_terminal, terminal=True)], terminal_grace_seconds=0.01, ) request = _request("terminal", "example.terminate") first = await executor.execute([request], scope="terminal-replay") replayed = await executor.execute([request], scope="terminal-replay") assert first.results[0].status is EventStatus.TIMEOUT assert first.results[0].error == "batch deadline exceeded" assert first.deadline_exceeded is False assert replayed.replayed_count == 1 assert replayed.deadline_exceeded is False @pytest.mark.asyncio async def test_replayed_sibling_batch_timeout_preserves_deadline_flag(): async def blocked(request: EventRequest) -> dict[str, Any]: await asyncio.Event().wait() return {"event_id": request.id} executor = _executor( [_definition("example.blocked", blocked)], batch_timeout_seconds=0.01, ) request = _request("sibling", "example.blocked") first = await executor.execute([request], scope="sibling-replay") replayed = await executor.execute([request], scope="sibling-replay") assert first.results[0].status is EventStatus.TIMEOUT assert first.deadline_exceeded is True assert replayed.replayed_count == 1 assert replayed.deadline_exceeded is True @pytest.mark.asyncio async def test_blocking_sync_handler_timeout_returns_before_thread_finishes(): started = threading.Event() release = threading.Event() finished = threading.Event() def blocking_handler(request: EventRequest) -> dict[str, Any]: started.set() release.wait(timeout=1) finished.set() return {"event_id": request.id} executor = _executor( [ _definition( "example.blocking", blocking_handler, timeout_seconds=0.02, ) ], batch_timeout_seconds=0.2, ) loop = asyncio.get_running_loop() started_at = loop.time() batch = await executor.execute( [_request("blocking", "example.blocking")], scope="blocking-handler", ) elapsed = loop.time() - started_at release.set() await asyncio.to_thread(finished.wait, 0.2) assert started.is_set() assert batch.results[0].status is EventStatus.TIMEOUT assert elapsed < 0.1 assert finished.is_set() @pytest.mark.asyncio async def test_timed_out_sync_handler_keeps_conflict_lock_until_thread_finishes(): worker_started = threading.Event() worker_release = threading.Event() follower_started = asyncio.Event() def blocking_handler(request: EventRequest) -> dict[str, Any]: worker_started.set() worker_release.wait(timeout=1) return {"event_id": request.id} async def follower_handler(request: EventRequest) -> dict[str, Any]: follower_started.set() return {"event_id": request.id} executor = _executor( [ _definition( "example.blocking", blocking_handler, conflict_keys=("shared",), timeout_seconds=0.02, ), _definition( "example.follower", follower_handler, conflict_keys=("shared",), timeout_seconds=0.2, ), ] ) timed_out = await executor.execute( [_request("blocking", "example.blocking")], scope="sync-conflict-blocking", ) follower = asyncio.create_task( executor.execute( [_request("follower", "example.follower")], scope="sync-conflict-follower", ) ) await asyncio.sleep(0.03) entered_before_worker_finished = follower_started.is_set() worker_release.set() follower_batch = await asyncio.wait_for(follower, timeout=0.3) assert worker_started.is_set() assert timed_out.results[0].status is EventStatus.TIMEOUT assert entered_before_worker_finished is False assert follower_batch.results[0].status is EventStatus.SUCCESS @pytest.mark.asyncio async def test_timed_out_sync_handler_keeps_parallel_slot_until_thread_finishes(): worker_started = threading.Event() worker_release = threading.Event() follower_started = asyncio.Event() def blocking_handler(request: EventRequest) -> dict[str, Any]: worker_started.set() worker_release.wait(timeout=1) return {"event_id": request.id} async def follower_handler(request: EventRequest) -> dict[str, Any]: follower_started.set() return {"event_id": request.id} executor = _executor( [ _definition( "example.blocking", blocking_handler, timeout_seconds=0.02, ), _definition( "example.follower", follower_handler, timeout_seconds=0.2, ), ], max_parallel_events=1, ) timed_out = await executor.execute( [_request("blocking", "example.blocking")], scope="sync-slot-blocking", ) follower = asyncio.create_task( executor.execute( [_request("follower", "example.follower")], scope="sync-slot-follower", ) ) await asyncio.sleep(0.03) entered_before_worker_finished = follower_started.is_set() worker_release.set() follower_batch = await asyncio.wait_for(follower, timeout=0.3) assert worker_started.is_set() assert timed_out.results[0].status is EventStatus.TIMEOUT assert entered_before_worker_finished is False assert follower_batch.results[0].status is EventStatus.SUCCESS @pytest.mark.asyncio async def test_timed_out_async_handler_releases_resources_after_cancellation(): cancelled = asyncio.Event() follower_started = asyncio.Event() async def blocked_handler(request: EventRequest) -> dict[str, Any]: try: await asyncio.Event().wait() finally: cancelled.set() return {"event_id": request.id} async def follower_handler(request: EventRequest) -> dict[str, Any]: follower_started.set() return {"event_id": request.id} executor = _executor( [ _definition( "example.blocked", blocked_handler, conflict_keys=("shared",), timeout_seconds=0.01, ), _definition( "example.follower", follower_handler, conflict_keys=("shared",), timeout_seconds=0.2, ), ], max_parallel_events=1, ) timed_out = await executor.execute( [_request("blocked", "example.blocked")], scope="async-release-blocked", ) follower = await executor.execute( [_request("follower", "example.follower")], scope="async-release-follower", ) assert timed_out.results[0].status is EventStatus.TIMEOUT assert cancelled.is_set() assert follower_started.is_set() assert follower.results[0].status is EventStatus.SUCCESS