Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Upload folder using huggingface_hub
Browse files
app.py
CHANGED
@@ -13,8 +13,8 @@ from fastrtc import (
|
|
13 |
AdditionalOutputs,
|
14 |
AsyncStreamHandler,
|
15 |
Stream,
|
16 |
-
WebRTCError,
|
17 |
get_twilio_turn_credentials,
|
|
|
18 |
)
|
19 |
from gradio.utils import get_space
|
20 |
from openai.types.beta.realtime import ResponseAudioTranscriptDoneEvent
|
@@ -47,60 +47,41 @@ class OpenAIHandler(AsyncStreamHandler):
|
|
47 |
):
|
48 |
"""Connect to realtime API. Run forever in separate thread to keep connection open."""
|
49 |
self.client = openai.AsyncOpenAI()
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
np.
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
)
|
70 |
-
except Exception:
|
71 |
-
import traceback
|
72 |
-
|
73 |
-
traceback.print_exc()
|
74 |
-
raise WebRTCError(str(traceback.format_exc()))
|
75 |
|
76 |
async def receive(self, frame: tuple[int, np.ndarray]) -> None:
|
77 |
if not self.connection:
|
78 |
return
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
await self.connection.input_audio_buffer.append(audio=audio_message) # type: ignore
|
84 |
-
except Exception as e:
|
85 |
-
# print traceback
|
86 |
-
print(f"Error in receive: {str(e)}")
|
87 |
-
import traceback
|
88 |
-
|
89 |
-
traceback.print_exc()
|
90 |
-
raise WebRTCError(str(traceback.format_exc()))
|
91 |
|
92 |
async def emit(self) -> tuple[int, np.ndarray] | AdditionalOutputs | None:
|
93 |
-
return await self.output_queue
|
94 |
-
|
95 |
-
def reset_state(self):
|
96 |
-
"""Reset connection state for new recording session"""
|
97 |
-
self.connection = None
|
98 |
-
self.args_set.clear()
|
99 |
|
100 |
async def shutdown(self) -> None:
|
101 |
if self.connection:
|
102 |
await self.connection.close()
|
103 |
-
self.
|
104 |
|
105 |
|
106 |
def update_chatbot(chatbot: list[dict], response: ResponseAudioTranscriptDoneEvent):
|
|
|
13 |
AdditionalOutputs,
|
14 |
AsyncStreamHandler,
|
15 |
Stream,
|
|
|
16 |
get_twilio_turn_credentials,
|
17 |
+
wait_for_item,
|
18 |
)
|
19 |
from gradio.utils import get_space
|
20 |
from openai.types.beta.realtime import ResponseAudioTranscriptDoneEvent
|
|
|
47 |
):
|
48 |
"""Connect to realtime API. Run forever in separate thread to keep connection open."""
|
49 |
self.client = openai.AsyncOpenAI()
|
50 |
+
async with self.client.beta.realtime.connect(
|
51 |
+
model="gpt-4o-mini-realtime-preview-2024-12-17"
|
52 |
+
) as conn:
|
53 |
+
await conn.session.update(
|
54 |
+
session={"turn_detection": {"type": "server_vad"}}
|
55 |
+
)
|
56 |
+
self.connection = conn
|
57 |
+
async for event in self.connection:
|
58 |
+
if event.type == "response.audio_transcript.done":
|
59 |
+
await self.output_queue.put(AdditionalOutputs(event))
|
60 |
+
if event.type == "response.audio.delta":
|
61 |
+
await self.output_queue.put(
|
62 |
+
(
|
63 |
+
self.output_sample_rate,
|
64 |
+
np.frombuffer(
|
65 |
+
base64.b64decode(event.delta), dtype=np.int16
|
66 |
+
).reshape(1, -1),
|
67 |
+
),
|
68 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
|
70 |
async def receive(self, frame: tuple[int, np.ndarray]) -> None:
|
71 |
if not self.connection:
|
72 |
return
|
73 |
+
_, array = frame
|
74 |
+
array = array.squeeze()
|
75 |
+
audio_message = base64.b64encode(array.tobytes()).decode("utf-8")
|
76 |
+
await self.connection.input_audio_buffer.append(audio=audio_message) # type: ignore
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
77 |
|
78 |
async def emit(self) -> tuple[int, np.ndarray] | AdditionalOutputs | None:
|
79 |
+
return await wait_for_item(self.output_queue)
|
|
|
|
|
|
|
|
|
|
|
80 |
|
81 |
async def shutdown(self) -> None:
|
82 |
if self.connection:
|
83 |
await self.connection.close()
|
84 |
+
self.connection = None
|
85 |
|
86 |
|
87 |
def update_chatbot(chatbot: list[dict], response: ResponseAudioTranscriptDoneEvent):
|