freddyaboulton HF staff commited on
Commit
a7ffd22
·
verified ·
1 Parent(s): 1f75936

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +26 -45
app.py CHANGED
@@ -13,8 +13,8 @@ from fastrtc import (
13
  AdditionalOutputs,
14
  AsyncStreamHandler,
15
  Stream,
16
- WebRTCError,
17
  get_twilio_turn_credentials,
 
18
  )
19
  from gradio.utils import get_space
20
  from openai.types.beta.realtime import ResponseAudioTranscriptDoneEvent
@@ -47,60 +47,41 @@ class OpenAIHandler(AsyncStreamHandler):
47
  ):
48
  """Connect to realtime API. Run forever in separate thread to keep connection open."""
49
  self.client = openai.AsyncOpenAI()
50
- try:
51
- async with self.client.beta.realtime.connect(
52
- model="gpt-4o-mini-realtime-preview-2024-12-17"
53
- ) as conn:
54
- await conn.session.update(
55
- session={"turn_detection": {"type": "server_vad"}}
56
- )
57
- self.connection = conn
58
- async for event in self.connection:
59
- if event.type == "response.audio_transcript.done":
60
- await self.output_queue.put(AdditionalOutputs(event))
61
- if event.type == "response.audio.delta":
62
- await self.output_queue.put(
63
- (
64
- self.output_sample_rate,
65
- np.frombuffer(
66
- base64.b64decode(event.delta), dtype=np.int16
67
- ).reshape(1, -1),
68
- ),
69
- )
70
- except Exception:
71
- import traceback
72
-
73
- traceback.print_exc()
74
- raise WebRTCError(str(traceback.format_exc()))
75
 
76
  async def receive(self, frame: tuple[int, np.ndarray]) -> None:
77
  if not self.connection:
78
  return
79
- try:
80
- _, array = frame
81
- array = array.squeeze()
82
- audio_message = base64.b64encode(array.tobytes()).decode("utf-8")
83
- await self.connection.input_audio_buffer.append(audio=audio_message) # type: ignore
84
- except Exception as e:
85
- # print traceback
86
- print(f"Error in receive: {str(e)}")
87
- import traceback
88
-
89
- traceback.print_exc()
90
- raise WebRTCError(str(traceback.format_exc()))
91
 
92
  async def emit(self) -> tuple[int, np.ndarray] | AdditionalOutputs | None:
93
- return await self.output_queue.get()
94
-
95
- def reset_state(self):
96
- """Reset connection state for new recording session"""
97
- self.connection = None
98
- self.args_set.clear()
99
 
100
  async def shutdown(self) -> None:
101
  if self.connection:
102
  await self.connection.close()
103
- self.reset_state()
104
 
105
 
106
  def update_chatbot(chatbot: list[dict], response: ResponseAudioTranscriptDoneEvent):
 
13
  AdditionalOutputs,
14
  AsyncStreamHandler,
15
  Stream,
 
16
  get_twilio_turn_credentials,
17
+ wait_for_item,
18
  )
19
  from gradio.utils import get_space
20
  from openai.types.beta.realtime import ResponseAudioTranscriptDoneEvent
 
47
  ):
48
  """Connect to realtime API. Run forever in separate thread to keep connection open."""
49
  self.client = openai.AsyncOpenAI()
50
+ async with self.client.beta.realtime.connect(
51
+ model="gpt-4o-mini-realtime-preview-2024-12-17"
52
+ ) as conn:
53
+ await conn.session.update(
54
+ session={"turn_detection": {"type": "server_vad"}}
55
+ )
56
+ self.connection = conn
57
+ async for event in self.connection:
58
+ if event.type == "response.audio_transcript.done":
59
+ await self.output_queue.put(AdditionalOutputs(event))
60
+ if event.type == "response.audio.delta":
61
+ await self.output_queue.put(
62
+ (
63
+ self.output_sample_rate,
64
+ np.frombuffer(
65
+ base64.b64decode(event.delta), dtype=np.int16
66
+ ).reshape(1, -1),
67
+ ),
68
+ )
 
 
 
 
 
 
69
 
70
  async def receive(self, frame: tuple[int, np.ndarray]) -> None:
71
  if not self.connection:
72
  return
73
+ _, array = frame
74
+ array = array.squeeze()
75
+ audio_message = base64.b64encode(array.tobytes()).decode("utf-8")
76
+ await self.connection.input_audio_buffer.append(audio=audio_message) # type: ignore
 
 
 
 
 
 
 
 
77
 
78
  async def emit(self) -> tuple[int, np.ndarray] | AdditionalOutputs | None:
79
+ return await wait_for_item(self.output_queue)
 
 
 
 
 
80
 
81
  async def shutdown(self) -> None:
82
  if self.connection:
83
  await self.connection.close()
84
+ self.connection = None
85
 
86
 
87
  def update_chatbot(chatbot: list[dict], response: ResponseAudioTranscriptDoneEvent):