feat: integrate NGROK API key and enhance media handling (main)

- Added NGROK API key retrieval to Config.py. - Improved media handling in setup.js, including restructuring audio processing logic for clarity. - Removed browser auto-launch from main.py and added automatic NGROK tunnel setup with error handling.
IRedDragonICY · Oct 12, 2024 · 872f3dd · 872f3dd
1 parent 3495603
commit 872f3dd
Show file tree

Hide file tree

Showing 3 changed files with 96 additions and 66 deletions.
diff --git a/src/Config.py b/src/Config.py
@@ -20,6 +20,7 @@ def __init__(self, config_file='config.ini'):
         self.HARM_CATEGORIES = self.config['HARM_CATEGORIES'].get('categories').split(',')
         self.FILES = self.FilesConfig(self.config['FILES'])
         self.API_KEYS = [key.strip() for key in self.config['API']['API_KEYS'].split(',') if key.strip()]
+        self.NGROK_API_KEY = self.config['NGROK'].get('api_key')
 
     @staticmethod
     def _create_default_config(config_file):

diff --git a/src/app/js/setup.js b/src/app/js/setup.js
@@ -11,20 +11,21 @@ import ModelController from './model.js';
     modelController.startBlinking();
 
     const statusDiv = document.getElementById('status');
-    const audioLink = "/temp/response.wav";
-    let audioPlaying = false;
     let isProcessing = false;
     let chunks = [];
+    const audioLink = "/temp/response.wav";
+    let audioPlaying = false;
     let audio;
 
     const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
+
     const video = document.createElement('video');
     video.srcObject = stream;
     video.autoplay = true;
     video.addEventListener('canplay', () => {
         const canvas = document.createElement('canvas');
         const context = canvas.getContext('2d');
-        (function captureFrame() {
+        const captureFrame = () => {
             if (video.readyState === 4) {
                 canvas.width = video.videoWidth;
                 canvas.height = video.videoHeight;
@@ -33,76 +34,93 @@ import ModelController from './model.js';
                     if (blob) {
                         const formData = new FormData();
                         formData.append('image', blob, 'frame.jpg');
-                        fetch('/api/upload_frame', {
-                            method: 'POST',
-                            body: formData
-                        });
+                        fetch('/api/upload_frame', { method: 'POST', body: formData });
                     }
                 }, 'image/jpeg');
             }
             setTimeout(captureFrame, 2500);
-        })();
+        };
+        captureFrame();
     });
 
     const mediaRecorder = new MediaRecorder(stream);
-    mediaRecorder.addEventListener('dataavailable', e => chunks.push(e.data));
-    mediaRecorder.addEventListener('stop', () => {
+    mediaRecorder.ondataavailable = e => chunks.push(e.data);
+    mediaRecorder.onstop = () => {
         const blob = new Blob(chunks, { type: 'audio/wav' });
         chunks = [];
         const formData = new FormData();
         formData.append('audio', blob, 'audio.wav');
-        fetch('/api/upload_audio', {
-            method: 'POST',
-            body: formData
-        }).then(() => {
-            statusDiv.textContent = "";
-            isProcessing = false;
-            (async function playAudioWhenReady() {
-                while (!(await fetch('/api/audio_status').then(res => res.json()).then(data => data.audio_ready))) {
-                    await new Promise(r => setTimeout(r, 500));
-                }
-                audioPlaying = true;
-                audio = new Audio(audioLink);
-                audio.addEventListener('ended', () => {
-                    fetch('/api/reset_audio_status', { method: 'POST' });
-                    audioPlaying = false;
-                });
-                audio.play().then(() => {
-                    const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
-                    const analyser = audioCtx.createAnalyser();
-                    const source = audioCtx.createMediaElementSource(audio);
-                    source.connect(analyser);
-                    analyser.connect(audioCtx.destination);
-                    const dataArray = new Uint8Array(analyser.frequencyBinCount);
-                    (function analyseVolume() {
-                        analyser.getByteFrequencyData(dataArray);
-                        modelController.setMouthOpenY(dataArray.reduce((a, b) => a + b, 0) / dataArray.length / 255);
-                        if (!audio.paused) requestAnimationFrame(analyseVolume);
-                    })();
-                });
-            })();
-        });
-    });
+        fetch('/api/upload_audio', { method: 'POST', body: formData })
+            .then(() => {
+                statusDiv.textContent = "";
+                isProcessing = false;
+                checkAudioReady();
+            });
+    };
 
-    vad.MicVAD.new({
-        onSpeechStart: () => {
-            if (audioPlaying) {
-                audio.pause();
-                audio.currentTime = 0;
-                fetch('/api/reset_audio_status', { method: 'POST' });
-                audioPlaying = false;
+    const checkAudioReady = async () => {
+        while (true) {
+            const response = await fetch('/api/audio_status');
+            const data = await response.json();
+            if (data.audio_ready) {
+                playBotAudio();
+                break;
             }
-            if (!isProcessing && mediaRecorder.state !== 'recording') {
-                mediaRecorder.start();
-                statusDiv.textContent = "Listening...";
-            }
-        },
-        onSpeechEnd: () => {
-            if (mediaRecorder.state === 'recording') {
-                mediaRecorder.stop();
-                isProcessing = true;
-                statusDiv.textContent = "Processing...";
+            await new Promise(r => setTimeout(r, 500));
+        }
+    };
+
+    const playBotAudio = () => {
+        audioPlaying = true;
+        audio = new Audio(audioLink);
+        audio.addEventListener('ended', () => {
+            audioPlaying = false;
+        });
+        audio.play().then(() => {
+            const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+            const analyser = audioCtx.createAnalyser();
+            const source = audioCtx.createMediaElementSource(audio);
+            source.connect(analyser);
+            analyser.connect(audioCtx.destination);
+            const dataArray = new Uint8Array(analyser.frequencyBinCount);
+            const analyseVolume = () => {
+                analyser.getByteFrequencyData(dataArray);
+                modelController.setMouthOpenY(dataArray.reduce((a, b) => a + b, 0) / dataArray.length / 255);
+                if (!audio.paused) requestAnimationFrame(analyseVolume);
+            };
+            analyseVolume();
+        });
+    };
+
+    try {
+        const MicVAD = window.vad.MicVAD;
+        if (!MicVAD) throw new Error('MicVAD is not available');
+
+        const micVAD = await MicVAD.new({
+            onSpeechStart: () => {
+                if (audioPlaying) {
+                    audio.pause();
+                    audio.currentTime = 0;
+                    audioPlaying = false;
+                }
+                if (!isProcessing && mediaRecorder.state !== 'recording') {
+                    mediaRecorder.start();
+                    statusDiv.textContent = "Listening...";
+                }
+            },
+            onSpeechEnd: () => {
+                if (mediaRecorder.state === 'recording') {
+                    mediaRecorder.stop();
+                    isProcessing = true;
+                    statusDiv.textContent = "Processing...";
+                }
             }
+        });
+
+        if (typeof micVAD.start === 'function') {
+            micVAD.start();
         }
-    }).start();
-})();
+    } catch (error) {
+        console.error('Error initializing MicVAD:', error);
+    }
+})();
diff --git a/src/main.py b/src/main.py
@@ -1,7 +1,6 @@
 import logging
 import threading
 import urllib.request
-import webbrowser
 from fastapi import FastAPI, File, UploadFile, Form, Cookie
 from fastapi.responses import HTMLResponse, JSONResponse
 from starlette.middleware.cors import CORSMiddleware
@@ -119,12 +118,24 @@ def check_internet_connection():
             return False
 
     def run(self):
-        threading.Thread(target=self.open_browser).start()
+        self.start_ngrok_automatically()
         uvicorn.run(self.app, host="localhost", port=8000)
 
-    @staticmethod
-    def open_browser():
-        webbrowser.open_new("http://localhost:8000")
+    def start_ngrok_automatically(self):
+        api_key = self.chatbot.config.NGROK_API_KEY
+        if not self.check_internet_connection():
+            print("No internet connection.")
+            return
+        try:
+            ngrok.set_auth_token(api_key)
+            tunnel = ngrok.connect("8000")
+            self.public_url = tunnel.public_url
+            self.ngrok_process = ngrok.get_ngrok_process()
+            threading.Thread(target=self.ngrok_process.proc.wait).start()
+            print(f"Localhost URL: http://localhost:8000")
+            print(f"Public URL: {self.public_url}")
+        except Exception as e:
+            print(f"Error starting ngrok: {e}")
 
 if __name__ == "__main__":
     config = Config()