Upload folder using huggingface_hub

README.md

@@ -1,12 +1,15 @@
 ---
-title: Talk To Openai
-emoji: 📉
-colorFrom: blue
-colorTo: gray
+title: Talk to OpenAI
+emoji: 🗣️
+colorFrom: purple
+colorTo: red
 sdk: gradio
 sdk_version: 5.16.0
 app_file: app.py
 pinned: false
+license: mit
+short_description: Talk to OpenAI using their multimodal API
+tags: [webrtc, websocket, gradio, secret|TWILIO_ACCOUNT_SID, secret|TWILIO_AUTH_TOKEN, secret|OPENAI_API_KEY]
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,146 @@
+import asyncio
+import base64
+from pathlib import Path
+
+import gradio as gr
+from gradio.utils import get_space
+import numpy as np
+import json
+import openai
+from dotenv import load_dotenv
+from fastrtc import (
+    AdditionalOutputs,
+    AsyncStreamHandler,
+    Stream,
+    get_twilio_turn_credentials,
+)
+from fastapi.responses import HTMLResponse, StreamingResponse
+
+from openai.types.beta.realtime import ResponseAudioTranscriptDoneEvent
+
+load_dotenv()
+
+cur_dir = Path(__file__).parent
+
+SAMPLE_RATE = 24000
+
+
+class OpenAIHandler(AsyncStreamHandler):
+    def __init__(
+        self,
+    ) -> None:
+        super().__init__(
+            expected_layout="mono",
+            output_sample_rate=SAMPLE_RATE,
+            output_frame_size=480,
+            input_sample_rate=SAMPLE_RATE,
+        )
+        self.connection = None
+        self.connected = asyncio.Event()
+        self.output_queue = asyncio.Queue()
+
+    def copy(self):
+        return OpenAIHandler()
+
+    async def _initialize_connection(
+        self,
+    ):
+        """Connect to realtime API. Run forever in separate thread to keep connection open."""
+        self.client = openai.AsyncOpenAI()
+        async with self.client.beta.realtime.connect(
+            model="gpt-4o-mini-realtime-preview-2024-12-17"
+        ) as conn:
+            await conn.session.update(
+                session={"turn_detection": {"type": "server_vad"}}
+            )
+            self.connection = conn
+            self.connected.set()
+            async for event in self.connection:
+                if event.type == "response.audio_transcript.done":
+                    await self.output_queue.put(AdditionalOutputs(event))
+                if event.type == "response.audio.delta":
+                    await self.output_queue.put(
+                        (
+                            self.output_sample_rate,
+                            np.frombuffer(
+                                base64.b64decode(event.delta), dtype=np.int16
+                            ).reshape(1, -1),
+                        ),
+                    )
+
+    async def receive(self, frame: tuple[int, np.ndarray]) -> None:
+        if not self.connection:
+            await self.fetch_args()
+            asyncio.create_task(self._initialize_connection())
+            await self.connected.wait()
+        try:
+            _, array = frame
+            array = array.squeeze()
+            audio_message = base64.b64encode(array.tobytes()).decode("utf-8")
+            await self.connection.input_audio_buffer.append(audio=audio_message)  # type: ignore
+        except Exception as e:
+            # print traceback
+            print(f"Error in receive: {str(e)}")
+            import traceback
+
+            traceback.print_exc()
+
+    async def emit(self) -> tuple[int, np.ndarray] | AdditionalOutputs | None:
+        if not self.connection:
+            return None
+        return await self.output_queue.get()
+
+    def reset_state(self):
+        """Reset connection state for new recording session"""
+        self.connection = None
+        self.args_set.clear()
+        self.connected.clear()
+
+    async def shutdown(self) -> None:
+        if self.connection:
+            await self.connection.close()
+            self.reset_state()
+
+
+def update_chatbot(chatbot: list[dict], response: ResponseAudioTranscriptDoneEvent):
+    chatbot.append({"role": "assistant", "content": response.transcript})
+    return chatbot
+
+
+chatbot = gr.Chatbot(type="messages")
+latest_message = gr.Textbox(type="text", visible=False)
+stream = Stream(
+    OpenAIHandler(),
+    mode="send-receive",
+    modality="audio",
+    additional_inputs=[chatbot],
+    additional_outputs=[chatbot],
+    additional_outputs_handler=update_chatbot,
+    rtc_configuration=get_twilio_turn_credentials() if get_space() else None,
+)
+
+
+@stream.get("/")
+async def _():
+    rtc_config = get_twilio_turn_credentials() if get_space() else None
+    html_content = (cur_dir / "index.html").read_text()
+    html_content = html_content.replace("__RTC_CONFIGURATION__", json.dumps(rtc_config))
+    return HTMLResponse(content=html_content)
+
+
+@stream.get("/outputs")
+def _(webrtc_id: str):
+    async def output_stream():
+        import json
+
+        async for output in stream.output_stream(webrtc_id):
+            s = json.dumps({"role": "assistant", "content": output.args[0].transcript})
+            yield f"event: output\ndata: {s}\n\n"
+
+    return StreamingResponse(output_stream(), media_type="text/event-stream")
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(stream, host="0.0.0.0", port=7860)

index.html

@@ -0,0 +1,222 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>OpenAI Real-Time Chat</title>
+    <style>
+        body {
+            font-family: "SF Pro Display", -apple-system, BlinkMacSystemFont, sans-serif;
+            background-color: #0a0a0a;
+            color: #ffffff;
+            margin: 0;
+            padding: 20px;
+            height: 100vh;
+            box-sizing: border-box;
+        }
+
+        .container {
+            max-width: 800px;
+            margin: 0 auto;
+            height: calc(100% - 100px);
+        }
+
+        .logo {
+            text-align: center;
+            margin-bottom: 40px;
+        }
+
+        .chat-container {
+            border: 1px solid #333;
+            padding: 20px;
+            height: 90%;
+            box-sizing: border-box;
+            display: flex;
+            flex-direction: column;
+        }
+
+        .chat-messages {
+            flex-grow: 1;
+            overflow-y: auto;
+            margin-bottom: 20px;
+            padding: 10px;
+        }
+
+        .message {
+            margin-bottom: 20px;
+            padding: 12px;
+            border-radius: 4px;
+            font-size: 16px;
+            line-height: 1.5;
+        }
+
+        .message.user {
+            background-color: #1a1a1a;
+            margin-left: 20%;
+        }
+
+        .message.assistant {
+            background-color: #262626;
+            margin-right: 20%;
+        }
+
+        .controls {
+            text-align: center;
+            margin-top: 20px;
+        }
+
+        button {
+            background-color: transparent;
+            color: #ffffff;
+            border: 1px solid #ffffff;
+            padding: 12px 24px;
+            font-family: inherit;
+            font-size: 16px;
+            cursor: pointer;
+            transition: all 0.3s;
+            text-transform: uppercase;
+            letter-spacing: 1px;
+        }
+
+        button:hover {
+            background-color: #ffffff;
+            color: #0a0a0a;
+        }
+
+        #audio-output {
+            display: none;
+        }
+    </style>
+</head>
+
+<body>
+    <div class="container">
+        <div class="logo">
+            <h1>OpenAI Real-Time Chat</h1>
+        </div>
+        <div class="chat-container">
+            <div class="chat-messages" id="chat-messages"></div>
+        </div>
+        <div class="controls">
+            <button id="start-button">Start Conversation</button>
+        </div>
+    </div>
+    <audio id="audio-output"></audio>
+
+    <script>
+        let peerConnection;
+        let webrtc_id;
+        const audioOutput = document.getElementById('audio-output');
+        const startButton = document.getElementById('start-button');
+        const chatMessages = document.getElementById('chat-messages');
+
+        async function setupWebRTC() {
+            const config = __RTC_CONFIGURATION__;
+
+            peerConnection = new RTCPeerConnection();
+
+            try {
+                const stream = await navigator.mediaDevices.getUserMedia({
+                    audio: true
+                });
+
+                stream.getTracks().forEach(track => {
+                    peerConnection.addTrack(track, stream);
+                });
+
+                peerConnection.addEventListener('track', (evt) => {
+                    if (audioOutput.srcObject !== evt.streams[0]) {
+                        audioOutput.srcObject = evt.streams[0];
+                        audioOutput.play();
+                    }
+                });
+
+                const dataChannel = peerConnection.createDataChannel('text');
+
+                const offer = await peerConnection.createOffer();
+                await peerConnection.setLocalDescription(offer);
+
+                await new Promise((resolve) => {
+                    if (peerConnection.iceGatheringState === "complete") {
+                        resolve();
+                    } else {
+                        const checkState = () => {
+                            if (peerConnection.iceGatheringState === "complete") {
+                                peerConnection.removeEventListener("icegatheringstatechange", checkState);
+                                resolve();
+                            }
+                        };
+                        peerConnection.addEventListener("icegatheringstatechange", checkState);
+                    }
+                });
+
+                webrtc_id = Math.random().toString(36).substring(7);
+
+                const response = await fetch('/webrtc/offer', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        sdp: offer.sdp,
+                        type: offer.type,
+                        webrtc_id: webrtc_id
+                    })
+                });
+
+                const serverResponse = await response.json();
+                await peerConnection.setRemoteDescription(serverResponse);
+
+                const eventSource = new EventSource('/outputs?webrtc_id=' + webrtc_id);
+                eventSource.addEventListener("output", (event) => {
+                    const eventJson = JSON.parse(event.data);
+                    addMessage("assistant", eventJson.content);
+
+                });
+            } catch (err) {
+                console.error('Error setting up WebRTC:', err);
+            }
+        }
+
+        function addMessage(role, content) {
+            const messageDiv = document.createElement('div');
+            messageDiv.classList.add('message', role);
+            messageDiv.textContent = content;
+            chatMessages.appendChild(messageDiv);
+            chatMessages.scrollTop = chatMessages.scrollHeight;
+        }
+
+        function stop() {
+            if (peerConnection) {
+                if (peerConnection.getTransceivers) {
+                    peerConnection.getTransceivers().forEach(transceiver => {
+                        if (transceiver.stop) {
+                            transceiver.stop();
+                        }
+                    });
+                }
+
+                if (peerConnection.getSenders) {
+                    peerConnection.getSenders().forEach(sender => {
+                        if (sender.track && sender.track.stop) sender.track.stop();
+                    });
+                }
+
+                setTimeout(() => {
+                    peerConnection.close();
+                }, 500);
+            }
+        }
+
+        startButton.addEventListener('click', () => {
+            if (startButton.textContent === 'Start Conversation') {
+                setupWebRTC();
+                startButton.textContent = 'Stop Conversation';
+            } else {
+                stop();
+                startButton.textContent = 'Start Conversation';
+            }
+        });
+    </script>
+</body>
+
+</html>

requirements.txt

@@ -0,0 +1,4 @@
+fastrtc[vad]==0.0.32rc1
+openai
+twilio
+python-dotenv