From 872f3dd2a30baad3355cd0ef07257c44d34e72cf Mon Sep 17 00:00:00 2001 From: IRedDragonICY Date: Sat, 12 Oct 2024 19:04:51 +0800 Subject: [PATCH] feat: integrate NGROK API key and enhance media handling (main) - Added NGROK API key retrieval to Config.py. - Improved media handling in setup.js, including restructuring audio processing logic for clarity. - Removed browser auto-launch from main.py and added automatic NGROK tunnel setup with error handling. --- src/Config.py | 1 + src/app/js/setup.js | 140 +++++++++++++++++++++++++------------------- src/main.py | 21 +++++-- 3 files changed, 96 insertions(+), 66 deletions(-) diff --git a/src/Config.py b/src/Config.py index 5485fed..34bd444 100644 --- a/src/Config.py +++ b/src/Config.py @@ -20,6 +20,7 @@ def __init__(self, config_file='config.ini'): self.HARM_CATEGORIES = self.config['HARM_CATEGORIES'].get('categories').split(',') self.FILES = self.FilesConfig(self.config['FILES']) self.API_KEYS = [key.strip() for key in self.config['API']['API_KEYS'].split(',') if key.strip()] + self.NGROK_API_KEY = self.config['NGROK'].get('api_key') @staticmethod def _create_default_config(config_file): diff --git a/src/app/js/setup.js b/src/app/js/setup.js index 2b06047..68965f6 100644 --- a/src/app/js/setup.js +++ b/src/app/js/setup.js @@ -11,20 +11,21 @@ import ModelController from './model.js'; modelController.startBlinking(); const statusDiv = document.getElementById('status'); - const audioLink = "/temp/response.wav"; - let audioPlaying = false; let isProcessing = false; let chunks = []; + const audioLink = "/temp/response.wav"; + let audioPlaying = false; let audio; const stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true }); + const video = document.createElement('video'); video.srcObject = stream; video.autoplay = true; video.addEventListener('canplay', () => { const canvas = document.createElement('canvas'); const context = canvas.getContext('2d'); - (function captureFrame() { + const captureFrame = () => { if (video.readyState === 4) { canvas.width = video.videoWidth; canvas.height = video.videoHeight; @@ -33,76 +34,93 @@ import ModelController from './model.js'; if (blob) { const formData = new FormData(); formData.append('image', blob, 'frame.jpg'); - fetch('/api/upload_frame', { - method: 'POST', - body: formData - }); + fetch('/api/upload_frame', { method: 'POST', body: formData }); } }, 'image/jpeg'); } setTimeout(captureFrame, 2500); - })(); + }; + captureFrame(); }); const mediaRecorder = new MediaRecorder(stream); - mediaRecorder.addEventListener('dataavailable', e => chunks.push(e.data)); - mediaRecorder.addEventListener('stop', () => { + mediaRecorder.ondataavailable = e => chunks.push(e.data); + mediaRecorder.onstop = () => { const blob = new Blob(chunks, { type: 'audio/wav' }); chunks = []; const formData = new FormData(); formData.append('audio', blob, 'audio.wav'); - fetch('/api/upload_audio', { - method: 'POST', - body: formData - }).then(() => { - statusDiv.textContent = ""; - isProcessing = false; - (async function playAudioWhenReady() { - while (!(await fetch('/api/audio_status').then(res => res.json()).then(data => data.audio_ready))) { - await new Promise(r => setTimeout(r, 500)); - } - audioPlaying = true; - audio = new Audio(audioLink); - audio.addEventListener('ended', () => { - fetch('/api/reset_audio_status', { method: 'POST' }); - audioPlaying = false; - }); - audio.play().then(() => { - const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); - const analyser = audioCtx.createAnalyser(); - const source = audioCtx.createMediaElementSource(audio); - source.connect(analyser); - analyser.connect(audioCtx.destination); - const dataArray = new Uint8Array(analyser.frequencyBinCount); - (function analyseVolume() { - analyser.getByteFrequencyData(dataArray); - modelController.setMouthOpenY(dataArray.reduce((a, b) => a + b, 0) / dataArray.length / 255); - if (!audio.paused) requestAnimationFrame(analyseVolume); - })(); - }); - })(); - }); - }); + fetch('/api/upload_audio', { method: 'POST', body: formData }) + .then(() => { + statusDiv.textContent = ""; + isProcessing = false; + checkAudioReady(); + }); + }; - vad.MicVAD.new({ - onSpeechStart: () => { - if (audioPlaying) { - audio.pause(); - audio.currentTime = 0; - fetch('/api/reset_audio_status', { method: 'POST' }); - audioPlaying = false; + const checkAudioReady = async () => { + while (true) { + const response = await fetch('/api/audio_status'); + const data = await response.json(); + if (data.audio_ready) { + playBotAudio(); + break; } - if (!isProcessing && mediaRecorder.state !== 'recording') { - mediaRecorder.start(); - statusDiv.textContent = "Listening..."; - } - }, - onSpeechEnd: () => { - if (mediaRecorder.state === 'recording') { - mediaRecorder.stop(); - isProcessing = true; - statusDiv.textContent = "Processing..."; + await new Promise(r => setTimeout(r, 500)); + } + }; + + const playBotAudio = () => { + audioPlaying = true; + audio = new Audio(audioLink); + audio.addEventListener('ended', () => { + audioPlaying = false; + }); + audio.play().then(() => { + const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); + const analyser = audioCtx.createAnalyser(); + const source = audioCtx.createMediaElementSource(audio); + source.connect(analyser); + analyser.connect(audioCtx.destination); + const dataArray = new Uint8Array(analyser.frequencyBinCount); + const analyseVolume = () => { + analyser.getByteFrequencyData(dataArray); + modelController.setMouthOpenY(dataArray.reduce((a, b) => a + b, 0) / dataArray.length / 255); + if (!audio.paused) requestAnimationFrame(analyseVolume); + }; + analyseVolume(); + }); + }; + + try { + const MicVAD = window.vad.MicVAD; + if (!MicVAD) throw new Error('MicVAD is not available'); + + const micVAD = await MicVAD.new({ + onSpeechStart: () => { + if (audioPlaying) { + audio.pause(); + audio.currentTime = 0; + audioPlaying = false; + } + if (!isProcessing && mediaRecorder.state !== 'recording') { + mediaRecorder.start(); + statusDiv.textContent = "Listening..."; + } + }, + onSpeechEnd: () => { + if (mediaRecorder.state === 'recording') { + mediaRecorder.stop(); + isProcessing = true; + statusDiv.textContent = "Processing..."; + } } + }); + + if (typeof micVAD.start === 'function') { + micVAD.start(); } - }).start(); -})(); \ No newline at end of file + } catch (error) { + console.error('Error initializing MicVAD:', error); + } +})(); diff --git a/src/main.py b/src/main.py index 295c60b..d89f9df 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,6 @@ import logging import threading import urllib.request -import webbrowser from fastapi import FastAPI, File, UploadFile, Form, Cookie from fastapi.responses import HTMLResponse, JSONResponse from starlette.middleware.cors import CORSMiddleware @@ -119,12 +118,24 @@ def check_internet_connection(): return False def run(self): - threading.Thread(target=self.open_browser).start() + self.start_ngrok_automatically() uvicorn.run(self.app, host="localhost", port=8000) - @staticmethod - def open_browser(): - webbrowser.open_new("http://localhost:8000") + def start_ngrok_automatically(self): + api_key = self.chatbot.config.NGROK_API_KEY + if not self.check_internet_connection(): + print("No internet connection.") + return + try: + ngrok.set_auth_token(api_key) + tunnel = ngrok.connect("8000") + self.public_url = tunnel.public_url + self.ngrok_process = ngrok.get_ngrok_process() + threading.Thread(target=self.ngrok_process.proc.wait).start() + print(f"Localhost URL: http://localhost:8000") + print(f"Public URL: {self.public_url}") + except Exception as e: + print(f"Error starting ngrok: {e}") if __name__ == "__main__": config = Config()