|
@@ -1,6 +1,7 @@
|
|
|
from __future__ import annotations
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
import asyncio
|
|
import asyncio
|
|
|
|
|
+import json
|
|
|
import threading
|
|
import threading
|
|
|
from collections.abc import Callable
|
|
from collections.abc import Callable
|
|
|
from typing import Any
|
|
from typing import Any
|
|
@@ -15,6 +16,7 @@ from agent_lab.application.events import (
|
|
|
EventRequest,
|
|
EventRequest,
|
|
|
EventSource,
|
|
EventSource,
|
|
|
EventStatus,
|
|
EventStatus,
|
|
|
|
|
+ ResultPolicy,
|
|
|
)
|
|
)
|
|
|
from agent_lab.application.tools import ToolDefinition, ToolRegistry
|
|
from agent_lab.application.tools import ToolDefinition, ToolRegistry
|
|
|
from agent_lab.domain.events import ToolCallEvent
|
|
from agent_lab.domain.events import ToolCallEvent
|
|
@@ -250,7 +252,10 @@ async def test_batch_executor_coalesces_exact_duplicates_inside_one_batch():
|
|
|
|
|
|
|
|
assert calls == 1
|
|
assert calls == 1
|
|
|
assert batch.coalesced_count == 1
|
|
assert batch.coalesced_count == 1
|
|
|
- assert batch.results[0] == batch.results[1]
|
|
|
|
|
|
|
+ assert batch.results[0].event_id == batch.results[1].event_id == "same"
|
|
|
|
|
+ assert batch.results[0].payload == batch.results[1].payload
|
|
|
|
|
+ assert batch.results[0].deduplicated_from is None
|
|
|
|
|
+ assert batch.results[1].deduplicated_from == "same"
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
@@ -403,10 +408,64 @@ async def test_batch_executor_coalesces_normalized_provider_arguments_across_ids
|
|
|
"float-id",
|
|
"float-id",
|
|
|
]
|
|
]
|
|
|
assert batch.results[0].payload == {"value": 30}
|
|
assert batch.results[0].payload == {"value": 30}
|
|
|
- assert batch.results[1].payload == {
|
|
|
|
|
- "value": 30,
|
|
|
|
|
- "deduplicated_from": "integer-id",
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ assert batch.results[1].payload == {"value": 30}
|
|
|
|
|
+ assert getattr(batch.results[0], "deduplicated_from", None) is None
|
|
|
|
|
+ assert getattr(batch.results[1], "deduplicated_from", None) == "integer-id"
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_coalesced_clone_rewrites_only_correlated_payload_event_id():
|
|
|
|
|
+ correlated = _executor(
|
|
|
|
|
+ [
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.correlated",
|
|
|
|
|
+ lambda request: {"event_id": request.id, "value": "same"},
|
|
|
|
|
+ )
|
|
|
|
|
+ ]
|
|
|
|
|
+ )
|
|
|
|
|
+ correlated_batch = await correlated.execute(
|
|
|
|
|
+ [
|
|
|
|
|
+ EventRequest(
|
|
|
|
|
+ id="primary",
|
|
|
|
|
+ name="example.correlated",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments='{"call":"primary"}',
|
|
|
|
|
+ ),
|
|
|
|
|
+ EventRequest(
|
|
|
|
|
+ id="duplicate",
|
|
|
|
|
+ name="example.correlated",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments='{"call":"duplicate"}',
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ scope="correlated-payload",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ primary, duplicate = correlated_batch.results
|
|
|
|
|
+ assert primary.event_id == "primary"
|
|
|
|
|
+ assert duplicate.event_id == "duplicate"
|
|
|
|
|
+ assert duplicate.raw_arguments == '{"call":"duplicate"}'
|
|
|
|
|
+ assert duplicate.payload == {"event_id": "duplicate", "value": "same"}
|
|
|
|
|
+ assert getattr(duplicate, "deduplicated_from", None) == "primary"
|
|
|
|
|
+
|
|
|
|
|
+ domain = _executor(
|
|
|
|
|
+ [
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.domain",
|
|
|
|
|
+ lambda request: {"event_id": "domain-object"},
|
|
|
|
|
+ )
|
|
|
|
|
+ ]
|
|
|
|
|
+ )
|
|
|
|
|
+ domain_batch = await domain.execute(
|
|
|
|
|
+ [
|
|
|
|
|
+ _request("primary", "example.domain"),
|
|
|
|
|
+ _request("duplicate", "example.domain"),
|
|
|
|
|
+ ],
|
|
|
|
|
+ scope="domain-payload",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ assert domain_batch.results[1].payload == {"event_id": "domain-object"}
|
|
|
|
|
+ assert getattr(domain_batch.results[1], "deduplicated_from", None) == "primary"
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
@@ -417,7 +476,7 @@ async def test_batch_executor_tool_replies_keep_each_coalesced_call_id():
|
|
|
name="example.tool",
|
|
name="example.tool",
|
|
|
description="Execute a coalesced tool.",
|
|
description="Execute a coalesced tool.",
|
|
|
parameters={"type": "object"},
|
|
parameters={"type": "object"},
|
|
|
- handler=lambda event: {"handled_by": event.id},
|
|
|
|
|
|
|
+ handler=lambda event: {"event_id": event.id},
|
|
|
)
|
|
)
|
|
|
]
|
|
]
|
|
|
)
|
|
)
|
|
@@ -440,12 +499,53 @@ async def test_batch_executor_tool_replies_keep_each_coalesced_call_id():
|
|
|
replies = registry.tool_replies(events, batch)
|
|
replies = registry.tool_replies(events, batch)
|
|
|
|
|
|
|
|
assert [reply.tool_call_id for reply in replies] == ["first-id", "second-id"]
|
|
assert [reply.tool_call_id for reply in replies] == ["first-id", "second-id"]
|
|
|
|
|
+ assert [json.loads(reply.content) for reply in replies] == [
|
|
|
|
|
+ {"event_id": "first-id"},
|
|
|
|
|
+ {"event_id": "second-id"},
|
|
|
|
|
+ ]
|
|
|
assert [result.event_id for result in batch.results] == [
|
|
assert [result.event_id for result in batch.results] == [
|
|
|
"first-id",
|
|
"first-id",
|
|
|
"second-id",
|
|
"second-id",
|
|
|
]
|
|
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_batch_policy_and_compact_summary_skip_logical_duplicates():
|
|
|
|
|
+ registry = ToolRegistry(
|
|
|
|
|
+ [
|
|
|
|
|
+ ToolDefinition(
|
|
|
|
|
+ name="example.template",
|
|
|
|
|
+ description="Render one template.",
|
|
|
|
|
+ parameters={"type": "object"},
|
|
|
|
|
+ handler=lambda event: {"event_id": event.id},
|
|
|
|
|
+ result_policy=ResultPolicy.TEMPLATE_FOLLOW_UP,
|
|
|
|
|
+ result_message_factory=lambda result: "Completed once.",
|
|
|
|
|
+ )
|
|
|
|
|
+ ]
|
|
|
|
|
+ )
|
|
|
|
|
+ executor = _executor_type()(EventKernel(registry.event_registry))
|
|
|
|
|
+ events = [
|
|
|
|
|
+ ToolCallEvent(
|
|
|
|
|
+ id=event_id,
|
|
|
|
|
+ name="example.template",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
|
|
+ )
|
|
|
|
|
+ for event_id in ("primary", "duplicate")
|
|
|
|
|
+ ]
|
|
|
|
|
+ batch = await executor.execute(
|
|
|
|
|
+ [registry.event_request(event) for event in events],
|
|
|
|
|
+ scope="policy-dedupe",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ decision = registry.batch_decision(batch)
|
|
|
|
|
+ summary = registry.compact_results_message(batch)
|
|
|
|
|
+
|
|
|
|
|
+ assert decision.template_messages == ("Completed once.",)
|
|
|
|
|
+ assert summary is not None
|
|
|
|
|
+ assert len(summary.content.splitlines()) == 2
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
async def test_batch_key_falls_back_to_original_arguments_when_normalizer_fails():
|
|
async def test_batch_key_falls_back_to_original_arguments_when_normalizer_fails():
|
|
|
calls = 0
|
|
calls = 0
|
|
@@ -490,6 +590,135 @@ async def test_batch_key_falls_back_to_original_arguments_when_normalizer_fails(
|
|
|
assert calls == 1
|
|
assert calls == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+@pytest.mark.parametrize(
|
|
|
|
|
+ ("invalid_arguments", "raw_arguments"),
|
|
|
|
|
+ [
|
|
|
|
|
+ ({"value": float("nan")}, '{"value":NaN}'),
|
|
|
|
|
+ ({"value": object()}, '{"value":"object"}'),
|
|
|
|
|
+ ],
|
|
|
|
|
+)
|
|
|
|
|
+async def test_invalid_json_arguments_do_not_coalesce_with_valid_empty_object(
|
|
|
|
|
+ invalid_arguments: dict[str, Any],
|
|
|
|
|
+ raw_arguments: str,
|
|
|
|
|
+):
|
|
|
|
|
+ calls = 0
|
|
|
|
|
+
|
|
|
|
|
+ def handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ nonlocal calls
|
|
|
|
|
+ calls += 1
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor([_definition("example.strict", handler)])
|
|
|
|
|
+ batch = await executor.execute(
|
|
|
|
|
+ [
|
|
|
|
|
+ EventRequest(
|
|
|
|
|
+ id="valid",
|
|
|
|
|
+ name="example.strict",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ source=EventSource.PROVIDER_RESOLVED,
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
|
|
+ ),
|
|
|
|
|
+ EventRequest(
|
|
|
|
|
+ id="invalid",
|
|
|
|
|
+ name="example.strict",
|
|
|
|
|
+ arguments=invalid_arguments,
|
|
|
|
|
+ source=EventSource.PROVIDER_RESOLVED,
|
|
|
|
|
+ raw_arguments=raw_arguments,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ scope="strict-json-batch",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ assert batch.coalesced_count == 0
|
|
|
|
|
+ assert [result.status for result in batch.results] == [
|
|
|
|
|
+ EventStatus.SUCCESS,
|
|
|
|
|
+ EventStatus.INVALID_ARGUMENTS,
|
|
|
|
|
+ ]
|
|
|
|
|
+ assert calls == 1
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+@pytest.mark.parametrize(
|
|
|
|
|
+ ("invalid_arguments", "raw_arguments"),
|
|
|
|
|
+ [
|
|
|
|
|
+ ({"value": float("nan")}, '{"value":NaN}'),
|
|
|
|
|
+ ({"value": object()}, '{"value":"object"}'),
|
|
|
|
|
+ ],
|
|
|
|
|
+)
|
|
|
|
|
+async def test_invalid_json_arguments_do_not_replay_valid_success(
|
|
|
|
|
+ invalid_arguments: dict[str, Any],
|
|
|
|
|
+ raw_arguments: str,
|
|
|
|
|
+):
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [_definition("example.strict", lambda request: {"event_id": request.id})]
|
|
|
|
|
+ )
|
|
|
|
|
+ valid = EventRequest(
|
|
|
|
|
+ id="same-id",
|
|
|
|
|
+ name="example.strict",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ source=EventSource.PROVIDER_RESOLVED,
|
|
|
|
|
+ raw_arguments="{}",
|
|
|
|
|
+ )
|
|
|
|
|
+ invalid = EventRequest(
|
|
|
|
|
+ id="same-id",
|
|
|
|
|
+ name="example.strict",
|
|
|
|
|
+ arguments=invalid_arguments,
|
|
|
|
|
+ source=EventSource.PROVIDER_RESOLVED,
|
|
|
|
|
+ raw_arguments=raw_arguments,
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ first = await executor.execute([valid], scope="strict-json-replay")
|
|
|
|
|
+ second = await executor.execute([invalid], scope="strict-json-replay")
|
|
|
|
|
+
|
|
|
|
|
+ assert first.results[0].status is EventStatus.SUCCESS
|
|
|
|
|
+ assert second.replayed_count == 0
|
|
|
|
|
+ assert second.results[0].status is EventStatus.INVALID_ARGUMENTS
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_non_json_normalizer_output_has_distinct_canonical_key():
|
|
|
|
|
+ calls = 0
|
|
|
|
|
+
|
|
|
|
|
+ def normalize(arguments: dict[str, Any]) -> dict[str, Any]:
|
|
|
|
|
+ if arguments.get("invalid"):
|
|
|
|
|
+ return {"value": object()}
|
|
|
|
|
+ return {}
|
|
|
|
|
+
|
|
|
|
|
+ def handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ nonlocal calls
|
|
|
|
|
+ calls += 1
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [_definition("example.normalized", handler, normalizer=normalize)]
|
|
|
|
|
+ )
|
|
|
|
|
+ batch = await executor.execute(
|
|
|
|
|
+ [
|
|
|
|
|
+ EventRequest(
|
|
|
|
|
+ id="valid",
|
|
|
|
|
+ name="example.normalized",
|
|
|
|
|
+ arguments={},
|
|
|
|
|
+ source=EventSource.PROVIDER_RESOLVED,
|
|
|
|
|
+ ),
|
|
|
|
|
+ EventRequest(
|
|
|
|
|
+ id="invalid",
|
|
|
|
|
+ name="example.normalized",
|
|
|
|
|
+ arguments={"invalid": True},
|
|
|
|
|
+ source=EventSource.PROVIDER_RESOLVED,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ scope="normalizer-invalid-json",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ assert batch.coalesced_count == 0
|
|
|
|
|
+ assert [result.status for result in batch.results] == [
|
|
|
|
|
+ EventStatus.SUCCESS,
|
|
|
|
|
+ EventStatus.INVALID_ARGUMENTS,
|
|
|
|
|
+ ]
|
|
|
|
|
+ assert calls == 1
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
async def test_concurrent_replay_waiters_share_execution_and_cancel_independently():
|
|
async def test_concurrent_replay_waiters_share_execution_and_cancel_independently():
|
|
|
calls = 0
|
|
calls = 0
|
|
@@ -539,6 +768,92 @@ async def test_release_scope_removes_replay_entries_for_reuse():
|
|
|
assert batch.replayed_count == 0
|
|
assert batch.replayed_count == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_release_scope_starts_new_generation_before_old_task_finishes():
|
|
|
|
|
+ calls = 0
|
|
|
|
|
+ started = [asyncio.Event(), asyncio.Event()]
|
|
|
|
|
+ releases = [asyncio.Event(), asyncio.Event()]
|
|
|
|
|
+
|
|
|
|
|
+ async def handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ nonlocal calls
|
|
|
|
|
+ call_index = calls
|
|
|
|
|
+ calls += 1
|
|
|
|
|
+ started[call_index].set()
|
|
|
|
|
+ await releases[call_index].wait()
|
|
|
|
|
+ return {"call": call_index + 1, "event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor([_definition("example.generated", handler)])
|
|
|
|
|
+ request = _request("same-id", "example.generated")
|
|
|
|
|
+ first = asyncio.create_task(
|
|
|
|
|
+ executor.execute([request], scope="generation-scope")
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.wait_for(started[0].wait(), timeout=0.2)
|
|
|
|
|
+
|
|
|
|
|
+ executor.release_scope("generation-scope")
|
|
|
|
|
+ second = asyncio.create_task(
|
|
|
|
|
+ executor.execute([request], scope="generation-scope")
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.sleep(0.02)
|
|
|
|
|
+ new_generation_started = started[1].is_set()
|
|
|
|
|
+
|
|
|
|
|
+ releases[0].set()
|
|
|
|
|
+ releases[1].set()
|
|
|
|
|
+ first_batch, second_batch = await asyncio.gather(first, second)
|
|
|
|
|
+
|
|
|
|
|
+ assert new_generation_started is True
|
|
|
|
|
+ assert first_batch.results[0].payload["call"] == 1
|
|
|
|
|
+ assert second_batch.results[0].payload["call"] == 2
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_old_generation_completion_cannot_replace_new_inflight_or_cache():
|
|
|
|
|
+ calls = 0
|
|
|
|
|
+ started = [asyncio.Event(), asyncio.Event(), asyncio.Event()]
|
|
|
|
|
+ releases = [asyncio.Event(), asyncio.Event(), asyncio.Event()]
|
|
|
|
|
+
|
|
|
|
|
+ async def handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ nonlocal calls
|
|
|
|
|
+ call_index = calls
|
|
|
|
|
+ calls += 1
|
|
|
|
|
+ started[call_index].set()
|
|
|
|
|
+ await releases[call_index].wait()
|
|
|
|
|
+ return {"call": call_index + 1, "event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor([_definition("example.generated", handler)])
|
|
|
|
|
+ request = _request("same-id", "example.generated")
|
|
|
|
|
+ old = asyncio.create_task(executor.execute([request], scope="reused-scope"))
|
|
|
|
|
+ await asyncio.wait_for(started[0].wait(), timeout=0.2)
|
|
|
|
|
+
|
|
|
|
|
+ executor.release_scope("reused-scope")
|
|
|
|
|
+ current = asyncio.create_task(
|
|
|
|
|
+ executor.execute([request], scope="reused-scope")
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.sleep(0.02)
|
|
|
|
|
+ new_generation_started = started[1].is_set()
|
|
|
|
|
+ if not new_generation_started:
|
|
|
|
|
+ releases[0].set()
|
|
|
|
|
+ await asyncio.gather(old, current)
|
|
|
|
|
+ assert new_generation_started is True
|
|
|
|
|
+
|
|
|
|
|
+ releases[0].set()
|
|
|
|
|
+ old_batch = await asyncio.wait_for(old, timeout=0.2)
|
|
|
|
|
+ waiter = asyncio.create_task(
|
|
|
|
|
+ executor.execute([request], scope="reused-scope")
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.sleep(0.02)
|
|
|
|
|
+ assert calls == 2
|
|
|
|
|
+
|
|
|
|
|
+ releases[1].set()
|
|
|
|
|
+ current_batch, waiter_batch = await asyncio.gather(current, waiter)
|
|
|
|
|
+ replayed = await executor.execute([request], scope="reused-scope")
|
|
|
|
|
+
|
|
|
|
|
+ assert old_batch.results[0].payload["call"] == 1
|
|
|
|
|
+ assert current_batch.results[0].payload["call"] == 2
|
|
|
|
|
+ assert waiter_batch.results[0].payload["call"] == 2
|
|
|
|
|
+ assert replayed.replayed_count == 1
|
|
|
|
|
+ assert replayed.results[0].payload["call"] == 2
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
async def test_replay_cache_stays_bounded_across_250_scopes():
|
|
async def test_replay_cache_stays_bounded_across_250_scopes():
|
|
|
executor = _executor(
|
|
executor = _executor(
|
|
@@ -623,6 +938,49 @@ async def test_terminal_grace_is_bounded_by_terminal_event_timeout():
|
|
|
assert batch.deadline_exceeded is False
|
|
assert batch.deadline_exceeded is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_replayed_terminal_grace_timeout_preserves_deadline_flag():
|
|
|
|
|
+ async def blocked_terminal(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ await asyncio.Event().wait()
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [_definition("example.terminate", blocked_terminal, terminal=True)],
|
|
|
|
|
+ terminal_grace_seconds=0.01,
|
|
|
|
|
+ )
|
|
|
|
|
+ request = _request("terminal", "example.terminate")
|
|
|
|
|
+
|
|
|
|
|
+ first = await executor.execute([request], scope="terminal-replay")
|
|
|
|
|
+ replayed = await executor.execute([request], scope="terminal-replay")
|
|
|
|
|
+
|
|
|
|
|
+ assert first.results[0].status is EventStatus.TIMEOUT
|
|
|
|
|
+ assert first.results[0].error == "batch deadline exceeded"
|
|
|
|
|
+ assert first.deadline_exceeded is False
|
|
|
|
|
+ assert replayed.replayed_count == 1
|
|
|
|
|
+ assert replayed.deadline_exceeded is False
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_replayed_sibling_batch_timeout_preserves_deadline_flag():
|
|
|
|
|
+ async def blocked(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ await asyncio.Event().wait()
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [_definition("example.blocked", blocked)],
|
|
|
|
|
+ batch_timeout_seconds=0.01,
|
|
|
|
|
+ )
|
|
|
|
|
+ request = _request("sibling", "example.blocked")
|
|
|
|
|
+
|
|
|
|
|
+ first = await executor.execute([request], scope="sibling-replay")
|
|
|
|
|
+ replayed = await executor.execute([request], scope="sibling-replay")
|
|
|
|
|
+
|
|
|
|
|
+ assert first.results[0].status is EventStatus.TIMEOUT
|
|
|
|
|
+ assert first.deadline_exceeded is True
|
|
|
|
|
+ assert replayed.replayed_count == 1
|
|
|
|
|
+ assert replayed.deadline_exceeded is True
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.asyncio
|
|
|
async def test_blocking_sync_handler_timeout_returns_before_thread_finishes():
|
|
async def test_blocking_sync_handler_timeout_returns_before_thread_finishes():
|
|
|
started = threading.Event()
|
|
started = threading.Event()
|
|
@@ -660,3 +1018,159 @@ async def test_blocking_sync_handler_timeout_returns_before_thread_finishes():
|
|
|
assert batch.results[0].status is EventStatus.TIMEOUT
|
|
assert batch.results[0].status is EventStatus.TIMEOUT
|
|
|
assert elapsed < 0.1
|
|
assert elapsed < 0.1
|
|
|
assert finished.is_set()
|
|
assert finished.is_set()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_timed_out_sync_handler_keeps_conflict_lock_until_thread_finishes():
|
|
|
|
|
+ worker_started = threading.Event()
|
|
|
|
|
+ worker_release = threading.Event()
|
|
|
|
|
+ follower_started = asyncio.Event()
|
|
|
|
|
+
|
|
|
|
|
+ def blocking_handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ worker_started.set()
|
|
|
|
|
+ worker_release.wait(timeout=1)
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ async def follower_handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ follower_started.set()
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.blocking",
|
|
|
|
|
+ blocking_handler,
|
|
|
|
|
+ conflict_keys=("shared",),
|
|
|
|
|
+ timeout_seconds=0.02,
|
|
|
|
|
+ ),
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.follower",
|
|
|
|
|
+ follower_handler,
|
|
|
|
|
+ conflict_keys=("shared",),
|
|
|
|
|
+ timeout_seconds=0.2,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ]
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ timed_out = await executor.execute(
|
|
|
|
|
+ [_request("blocking", "example.blocking")],
|
|
|
|
|
+ scope="sync-conflict-blocking",
|
|
|
|
|
+ )
|
|
|
|
|
+ follower = asyncio.create_task(
|
|
|
|
|
+ executor.execute(
|
|
|
|
|
+ [_request("follower", "example.follower")],
|
|
|
|
|
+ scope="sync-conflict-follower",
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.sleep(0.03)
|
|
|
|
|
+ entered_before_worker_finished = follower_started.is_set()
|
|
|
|
|
+
|
|
|
|
|
+ worker_release.set()
|
|
|
|
|
+ follower_batch = await asyncio.wait_for(follower, timeout=0.3)
|
|
|
|
|
+
|
|
|
|
|
+ assert worker_started.is_set()
|
|
|
|
|
+ assert timed_out.results[0].status is EventStatus.TIMEOUT
|
|
|
|
|
+ assert entered_before_worker_finished is False
|
|
|
|
|
+ assert follower_batch.results[0].status is EventStatus.SUCCESS
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_timed_out_sync_handler_keeps_parallel_slot_until_thread_finishes():
|
|
|
|
|
+ worker_started = threading.Event()
|
|
|
|
|
+ worker_release = threading.Event()
|
|
|
|
|
+ follower_started = asyncio.Event()
|
|
|
|
|
+
|
|
|
|
|
+ def blocking_handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ worker_started.set()
|
|
|
|
|
+ worker_release.wait(timeout=1)
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ async def follower_handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ follower_started.set()
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.blocking",
|
|
|
|
|
+ blocking_handler,
|
|
|
|
|
+ timeout_seconds=0.02,
|
|
|
|
|
+ ),
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.follower",
|
|
|
|
|
+ follower_handler,
|
|
|
|
|
+ timeout_seconds=0.2,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ max_parallel_events=1,
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ timed_out = await executor.execute(
|
|
|
|
|
+ [_request("blocking", "example.blocking")],
|
|
|
|
|
+ scope="sync-slot-blocking",
|
|
|
|
|
+ )
|
|
|
|
|
+ follower = asyncio.create_task(
|
|
|
|
|
+ executor.execute(
|
|
|
|
|
+ [_request("follower", "example.follower")],
|
|
|
|
|
+ scope="sync-slot-follower",
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ await asyncio.sleep(0.03)
|
|
|
|
|
+ entered_before_worker_finished = follower_started.is_set()
|
|
|
|
|
+
|
|
|
|
|
+ worker_release.set()
|
|
|
|
|
+ follower_batch = await asyncio.wait_for(follower, timeout=0.3)
|
|
|
|
|
+
|
|
|
|
|
+ assert worker_started.is_set()
|
|
|
|
|
+ assert timed_out.results[0].status is EventStatus.TIMEOUT
|
|
|
|
|
+ assert entered_before_worker_finished is False
|
|
|
|
|
+ assert follower_batch.results[0].status is EventStatus.SUCCESS
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+@pytest.mark.asyncio
|
|
|
|
|
+async def test_timed_out_async_handler_releases_resources_after_cancellation():
|
|
|
|
|
+ cancelled = asyncio.Event()
|
|
|
|
|
+ follower_started = asyncio.Event()
|
|
|
|
|
+
|
|
|
|
|
+ async def blocked_handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ try:
|
|
|
|
|
+ await asyncio.Event().wait()
|
|
|
|
|
+ finally:
|
|
|
|
|
+ cancelled.set()
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ async def follower_handler(request: EventRequest) -> dict[str, Any]:
|
|
|
|
|
+ follower_started.set()
|
|
|
|
|
+ return {"event_id": request.id}
|
|
|
|
|
+
|
|
|
|
|
+ executor = _executor(
|
|
|
|
|
+ [
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.blocked",
|
|
|
|
|
+ blocked_handler,
|
|
|
|
|
+ conflict_keys=("shared",),
|
|
|
|
|
+ timeout_seconds=0.01,
|
|
|
|
|
+ ),
|
|
|
|
|
+ _definition(
|
|
|
|
|
+ "example.follower",
|
|
|
|
|
+ follower_handler,
|
|
|
|
|
+ conflict_keys=("shared",),
|
|
|
|
|
+ timeout_seconds=0.2,
|
|
|
|
|
+ ),
|
|
|
|
|
+ ],
|
|
|
|
|
+ max_parallel_events=1,
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ timed_out = await executor.execute(
|
|
|
|
|
+ [_request("blocked", "example.blocked")],
|
|
|
|
|
+ scope="async-release-blocked",
|
|
|
|
|
+ )
|
|
|
|
|
+ follower = await executor.execute(
|
|
|
|
|
+ [_request("follower", "example.follower")],
|
|
|
|
|
+ scope="async-release-follower",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ assert timed_out.results[0].status is EventStatus.TIMEOUT
|
|
|
|
|
+ assert cancelled.is_set()
|
|
|
|
|
+ assert follower_started.is_set()
|
|
|
|
|
+ assert follower.results[0].status is EventStatus.SUCCESS
|