Classify xAI push-to-talk held-audio send failures and clears after a request's commit in the realtime simulator (#9857)

tests/realtime/simulation/_findings.py

@@ -174,20 +174,29 @@ LOST_RESPONSE_RESERVATION = Finding(
 )
 
 
+def _xai_push_to_talk(sim: Simulation) -> bool:
+    options = getattr(sim, 'openai', None)
+    return options is not None and options.dialect == 'xai' and options.turn_detection == 'manual'
+
+
 def _receive_loop_send_failed(sim: Simulation, violation: InvariantViolation) -> bool:
     network = getattr(getattr(sim, 'server', None), 'network', None)
+    # On xAI push-to-talk, the receive loop sends more once the `response.done` that ends a reply is handled (#9070):
+    # the audio held back behind the reply, and the clear a deferred request needs to be answered.
+    xai_frames = ('input_audio_buffer.append', 'input_audio_buffer.clear') if _xai_push_to_talk(sim) else ()
+    sent = ('response.create', *xai_frames)
     return network is not None and any(
-        frame == 'response.create' and last_read in ('response.done', 'error')
-        for frame, last_read, _ in network.failed_sends
+        frame in sent and last_read in ('response.done', 'error') for frame, last_read, _ in network.failed_sends
     )
 
 
 RECEIVE_LOOP_SEND_FAILURE = Finding(
     id='SIM-4',
     title=(
-        'a deferred `response.create` that the connection sends while handling a `response.done` (or a refusal) fails '
-        "on a dying socket, and the whole frame is dropped: that response's usage and terminal never reach the "
-        'session, and the deferred request is neither re-asked nor released'
+        'a deferred `response.create` (or, on xAI push-to-talk, audio held back behind the reply, or the clear the '
+        'request needs) that the connection sends while handling a `response.done` (or a refusal) fails on a dying '
+        "socket, and the whole frame is dropped: that response's usage and terminal never reach the session, and the "
+        'deferred request is neither re-asked nor released'
     ),
     tracked_by='an ordered outbox, so the receive loop never sends on the socket itself; found by this simulator',
     evidence='simulated',
@@ -727,11 +736,8 @@ def _hand_commit_unanswered_at_the_end(sim: Simulation) -> bool:
 
 
 def _push_to_talk_audio_after_a_repeated_terminal(sim: Simulation) -> bool:
-    options = getattr(sim, 'openai', None)
     return (
-        options is not None
-        and options.dialect == 'xai'
-        and options.turn_detection == 'manual'
+        _xai_push_to_talk(sim)
         and bool(sim.truth.repeated_terminals)
         and any(operation.name == 'send_audio' for operation in sim.operations)
     )

tests/realtime/simulation/test_simulation.py

@@ -547,6 +547,61 @@ def test_known_failed_deferred_create_after_a_refusal_keeps_its_reservation() ->
     reproduce('SIM-4', OpenAISimulation(), scenario)
 
 
+@known('SIM-4')
+def test_known_xai_held_audio_failing_after_a_reply_drops_its_terminal() -> None:
+    """Audio sent during a reply is held back, and goes out from the receive loop once the reply ends."""
+
+    def scenario(sim: OpenAISimulation) -> None:
+        sim.create_response()
+        sim.fail_next_send()
+        sim.send_audio()
+        sim.settle()
+
+    reproduce(
+        'SIM-4',
+        OpenAISimulation(openai=OpenAIOptions(dialect='xai', turn_detection='manual', transcription=False)),
+        scenario,
+    )
+
+
+@known('SIM-4')
+def test_known_xai_held_audio_failing_before_a_deferred_request_loses_it() -> None:
+    """The held audio goes out ahead of the request deferred behind the reply, which the failure takes with it."""
+
+    def scenario(sim: OpenAISimulation) -> None:
+        sim.send_text()
+        sim.create_response()
+        sim.fail_next_send()
+        sim.send_audio()
+        sim.settle()
+
+    reproduce(
+        'SIM-4',
+        OpenAISimulation(openai=OpenAIOptions(dialect='xai', turn_detection='manual', transcription=False)),
+        scenario,
+    )
+
+
+@known('SIM-4')
+def test_known_xai_clear_before_a_deferred_request_failing_loses_it() -> None:
+    """The clear xAI needs before a deferred request goes out from the receive loop, once the reply ends."""
+
+    def scenario(sim: OpenAISimulation) -> None:
+        sim.create_response()
+        sim.clear_audio()
+        sim.create_response()
+        sim.fail_next_send()
+        sim.send_audio()
+        sim.clear_audio()
+        sim.settle()
+
+    reproduce(
+        'SIM-4',
+        OpenAISimulation(openai=OpenAIOptions(dialect='xai', turn_detection='manual', transcription=False)),
+        scenario,
+    )
+
+
 def test_barge_in_keeps_a_request_vad_will_not_answer() -> None:
     """SIM-32, fixed by the session core."""
 
@@ -676,6 +731,21 @@ def test_clear_after_an_unread_vad_commit_keeps_the_turn() -> None:
     run_clean(OpenAISimulation(openai=OpenAIOptions(transcription=False)), scenario)
 
 
+def test_xai_clear_after_a_request_committed_speech_keeps_the_turn() -> None:
+    """On xAI push-to-talk, the request a text turn sends commits the speech in the buffer before the clear (SIM-35)."""
+
+    def scenario(sim: OpenAISimulation) -> None:
+        sim.send_audio()
+        sim.settle()
+        sim.send_text()
+        sim.clear_audio()
+        sim.settle()
+
+    run_clean(
+        OpenAISimulation(openai=OpenAIOptions(dialect='xai', turn_detection='manual', transcription=False)), scenario
+    )
+
+
 @known('SIM-36')
 def test_known_xai_push_to_talk_audio_after_a_repeated_terminal_is_held() -> None:
     def scenario(sim: OpenAISimulation) -> None: