Realtime: send audio item/content index

Author: rm-openai

src/agents/realtime/events.py

@@ -115,6 +115,12 @@ class RealtimeAudioEnd:
     info: RealtimeEventInfo
     """Common info for all events, such as the context."""
 
+    item_id: str
+    """The ID of the item containing audio."""
+
+    content_index: int
+    """The index of the audio content in `item.content`"""
+
     type: Literal["audio_end"] = "audio_end"
 
 
@@ -125,6 +131,12 @@ class RealtimeAudio:
     audio: RealtimeModelAudioEvent
     """The audio event from the model layer."""
 
+    item_id: str
+    """The ID of the item containing audio."""
+
+    content_index: int
+    """The index of the audio content in `item.content`"""
+
     info: RealtimeEventInfo
     """Common info for all events, such as the context."""
 
@@ -140,6 +152,12 @@ class RealtimeAudioInterrupted:
     info: RealtimeEventInfo
     """Common info for all events, such as the context."""
 
+    item_id: str
+    """The ID of the item containing audio."""
+
+    content_index: int
+    """The index of the audio content in `item.content`"""
+
     type: Literal["audio_interrupted"] = "audio_interrupted"
 
 

src/agents/realtime/model_events.py

@@ -40,20 +40,38 @@ class RealtimeModelAudioEvent:
     data: bytes
     response_id: str
 
+    item_id: str
+    """The ID of the item containing audio."""
+
+    content_index: int
+    """The index of the audio content in `item.content`"""
+
     type: Literal["audio"] = "audio"
 
 
 @dataclass
 class RealtimeModelAudioInterruptedEvent:
     """Audio interrupted."""
 
+    item_id: str
+    """The ID of the item containing audio."""
+
+    content_index: int
+    """The index of the audio content in `item.content`"""
+
     type: Literal["audio_interrupted"] = "audio_interrupted"
 
 
 @dataclass
 class RealtimeModelAudioDoneEvent:
     """Audio done."""
 
+    item_id: str
+    """The ID of the item containing audio."""
+
+    content_index: int
+    """The index of the audio content in `item.content`"""
+
     type: Literal["audio_done"] = "audio_done"
 
 

src/agents/realtime/openai_realtime.py

@@ -302,7 +302,12 @@ async def _send_interrupt(self, event: RealtimeModelSendInterrupt) -> None:
 
         elapsed_time_ms = (datetime.now() - self._audio_start_time).total_seconds() * 1000
         if elapsed_time_ms > 0 and elapsed_time_ms < self._audio_length_ms:
-            await self._emit_event(RealtimeModelAudioInterruptedEvent())
+            await self._emit_event(
+                RealtimeModelAudioInterruptedEvent(
+                    item_id=self._current_item_id,
+                    content_index=self._current_audio_content_index or 0,
+                )
+            )
             converted = _ConversionHelper.convert_interrupt(
                 self._current_item_id,
                 self._current_audio_content_index or 0,
@@ -331,7 +336,12 @@ async def _handle_audio_delta(self, parsed: ResponseAudioDeltaEvent) -> None:
         # Calculate audio length in ms using 24KHz pcm16le
         self._audio_length_ms += self._calculate_audio_length_ms(audio_bytes)
         await self._emit_event(
-            RealtimeModelAudioEvent(data=audio_bytes, response_id=parsed.response_id)
+            RealtimeModelAudioEvent(
+                data=audio_bytes,
+                response_id=parsed.response_id,
+                item_id=parsed.item_id,
+                content_index=parsed.content_index,
+            )
         )
 
     def _calculate_audio_length_ms(self, audio_bytes: bytes) -> float:
@@ -429,7 +439,12 @@ async def _handle_ws_event(self, event: dict[str, Any]):
         if parsed.type == "response.audio.delta":
             await self._handle_audio_delta(parsed)
         elif parsed.type == "response.audio.done":
-            await self._emit_event(RealtimeModelAudioDoneEvent())
+            await self._emit_event(
+                RealtimeModelAudioDoneEvent(
+                    item_id=parsed.item_id,
+                    content_index=parsed.content_index,
+                )
+            )
         elif parsed.type == "input_audio_buffer.speech_started":
             await self._send_interrupt(RealtimeModelSendInterrupt())
         elif parsed.type == "response.created":

src/agents/realtime/session.py

@@ -188,11 +188,26 @@ async def on_event(self, event: RealtimeModelEvent) -> None:
         elif event.type == "function_call":
             await self._handle_tool_call(event)
         elif event.type == "audio":
-            await self._put_event(RealtimeAudio(info=self._event_info, audio=event))
+            await self._put_event(
+                RealtimeAudio(
+                    info=self._event_info,
+                    audio=event,
+                    item_id=event.item_id,
+                    content_index=event.content_index,
+                )
+            )
         elif event.type == "audio_interrupted":
-            await self._put_event(RealtimeAudioInterrupted(info=self._event_info))
+            await self._put_event(
+                RealtimeAudioInterrupted(
+                    info=self._event_info, item_id=event.item_id, content_index=event.content_index
+                )
+            )
         elif event.type == "audio_done":
-            await self._put_event(RealtimeAudioEnd(info=self._event_info))
+            await self._put_event(
+                RealtimeAudioEnd(
+                    info=self._event_info, item_id=event.item_id, content_index=event.content_index
+                )
+            )
         elif event.type == "input_audio_transcription_completed":
             self._history = RealtimeSession._get_new_history(self._history, event)
             await self._put_event(

tests/realtime/test_session.py

@@ -157,15 +157,17 @@ async def test_audio_events_transformation(self, mock_model, mock_agent):
         session = RealtimeSession(mock_model, mock_agent, None)
 
         # Test audio event
-        audio_event = RealtimeModelAudioEvent(data=b"audio_data", response_id="resp_1")
+        audio_event = RealtimeModelAudioEvent(
+            data=b"audio_data", response_id="resp_1", item_id="item_1", content_index=0
+        )
         await session.on_event(audio_event)
 
         # Test audio interrupted event
-        interrupted_event = RealtimeModelAudioInterruptedEvent()
+        interrupted_event = RealtimeModelAudioInterruptedEvent(item_id="item_1", content_index=0)
         await session.on_event(interrupted_event)
 
         # Test audio done event
-        done_event = RealtimeModelAudioDoneEvent()
+        done_event = RealtimeModelAudioDoneEvent(item_id="item_1", content_index=0)
         await session.on_event(done_event)
 
         # Should have 6 events total (2 per event: raw + transformed)