From b45a4ee5df4aa5d3afd22bd2f4e3012c14a9a714 Mon Sep 17 00:00:00 2001 From: WolfwithSword <12175651+WolfwithSword@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:31:37 -0400 Subject: [PATCH] Initial Overlay UI and Flow --- src/chatdnd/events/session_events.py | 3 + src/chatdnd/events/web_events.py | 3 + src/chatdnd/session_manager.py | 14 +- src/data/session.py | 3 + src/helpers/event.py | 2 +- src/server/app.py | 99 ++++++--- src/server/static/overlay.html | 291 ++++++++++++++++++++++++++- src/tts/local_tts.py | 7 +- src/twitch/chat.py | 5 +- src/ui/tabs/home.py | 4 +- 10 files changed, 391 insertions(+), 40 deletions(-) create mode 100644 src/chatdnd/events/session_events.py create mode 100644 src/chatdnd/events/web_events.py diff --git a/src/chatdnd/events/session_events.py b/src/chatdnd/events/session_events.py new file mode 100644 index 0000000..f161201 --- /dev/null +++ b/src/chatdnd/events/session_events.py @@ -0,0 +1,3 @@ +from helpers import Event + +on_party_update = Event() \ No newline at end of file diff --git a/src/chatdnd/events/web_events.py b/src/chatdnd/events/web_events.py new file mode 100644 index 0000000..a1a84b6 --- /dev/null +++ b/src/chatdnd/events/web_events.py @@ -0,0 +1,3 @@ +from helpers import Event + +on_overlay_open = Event() \ No newline at end of file diff --git a/src/chatdnd/session_manager.py b/src/chatdnd/session_manager.py index 45b41f7..95f0579 100644 --- a/src/chatdnd/session_manager.py +++ b/src/chatdnd/session_manager.py @@ -3,9 +3,14 @@ from data import Session, Member, SessionState from custom_logger.logger import logger +from chatdnd.events.session_events import on_party_update +from chatdnd.events.web_events import on_overlay_open + class SessionManager(): def __init__(self): self.session = Session() + on_overlay_open.addListener(self.trigger_update) + on_party_update.trigger([self.session.get_party()]) def join_queue(self, member: Member): self.session.queue.add(member) @@ -17,11 +22,18 @@ def start_session(self, party_size: int = 4) -> bool: self.session.party.clear() self.session.party.update(random.sample(sorted(self.session.queue), party_size)) self.session.queue.clear() + on_party_update.trigger([self.session.get_party()]) return True def end(self): self.session.clear() self.session.state = SessionState.NONE + on_party_update.trigger([self.session.get_party()]) def open(self): - self.session.state = SessionState.OPEN \ No newline at end of file + self.session.clear() + self.session.state = SessionState.OPEN + on_party_update.trigger([self.session.get_party()]) + + def trigger_update(self): + on_party_update.trigger([self.session.get_party()]) diff --git a/src/data/session.py b/src/data/session.py index 7d32950..a48795c 100644 --- a/src/data/session.py +++ b/src/data/session.py @@ -24,3 +24,6 @@ def clear(self): self.queue.clear() self.party.clear() self.state: SessionState = SessionState.NONE + + def get_party(self) -> List[Member]: + return sorted(self.party) diff --git a/src/helpers/event.py b/src/helpers/event.py index 7f81a52..ec3fc45 100644 --- a/src/helpers/event.py +++ b/src/helpers/event.py @@ -61,4 +61,4 @@ async def _run_async_func(self, func, *args): try: await func(*args) except Exception as e: - print(f"Async Event Listener error: {e}") \ No newline at end of file + logger.error(f"Async Event Listener error: {e}") \ No newline at end of file diff --git a/src/server/app.py b/src/server/app.py index b023917..9be8702 100644 --- a/src/server/app.py +++ b/src/server/app.py @@ -10,12 +10,16 @@ from custom_logger.logger import logger from chatdnd.events.chat_events import chat_say_command +from chatdnd.events.session_events import on_party_update +from chatdnd.events.web_events import on_overlay_open from helpers.utils import get_resource_path STATIC_DIR = get_resource_path("../server/static", from_resources=True) message_queue = Queue() +members_queue = Queue() clients = set() +overlay_clients = set() class ServerApp(): def __init__(self, config: Config): @@ -24,58 +28,81 @@ def __init__(self, config: Config): self.tts = LocalTTS(config) # TODO both local and cloud self._setup_routes() + self._party: set[Member] = set() + # Setup here temporarily for POC chat_say_command.addListener(self.chat_say) - + on_party_update.addListener(self.send_members) + on_overlay_open.trigger() def _setup_routes(self): - @self.app.route('/test') # Temp / Testing - async def test(): - logger.info("TEST QUART") - return jsonify({"test": "123"}) - @self.app.websocket("/tts") + @self.app.websocket("/ws/tts") async def audio_stream(): clients.add(websocket) try: - logger.debug("ws opened") + logger.debug("tts ws opened") + await websocket.send_json({"type":"heartbeat"}) while True: if not message_queue.empty(): member, message = await message_queue.get() - logger.info(f"saying {message} from {member}") - - async for chunk in self.tts.get_stream(message, '' if not member else member.preferred_tts): + speech_message = { + "type": "speech", + "name": member.name, + "message": message + } + logger.info(f"saying '{message}' from {member}") + duration = 0 + last_chunk_duration = 0 + send_bounce = False + async for chunk, _duration in self.tts.get_stream(message, '' if not member else member.preferred_tts): + # TODO: Allow for break / interruption from emergency stuff - also hide stuff. Or yknow, just instruct to hide the browser source. + if not send_bounce: + send_bounce = True + await self.animate_member(member.name, "bounce") + await members_queue.put(speech_message) await asyncio.wait_for(websocket.send(chunk), timeout=10) + duration += _duration + last_chunk_duration = _duration + await asyncio.sleep(last_chunk_duration) + speech_message = { + "type": "endspeech" + } + await self.animate_member(member.name, "idle") + await asyncio.sleep(0.3) + await members_queue.put(speech_message) + await asyncio.sleep(0.1) except Exception as e: logger.error(e) pass finally: - logger.debug("ws closed") + logger.debug("tts ws closed") clients.discard(websocket) - # This POST is *purely* for testing. We want actual triggers to be event driven to put text into the queue. - # Final version, this will not be enabled. We will rely on event processing - @self.app.route('/trigger-tts', methods=['POST']) # doesnt have to be a post - async def trigger_tts(): - - text = (await request.get_json()).get('text', 'Default message') # Read text from POST body - - if not text: - return jsonify({"error": "Text is required"}), 400 - - if clients: - await message_queue.put((None, text)) - return jsonify({"status": "success", "message": "Text sent to WebSocket for TTS."}) - else: - return jsonify({"error": "No active WebSocket connection."}), 400 + @self.app.websocket("/ws/members") + async def user_overlay_ws(): + overlay_clients.add(websocket) + try: + await websocket.send_json({"type":"heartbeat"}) + logger.debug('overlay ws opened') + on_overlay_open.trigger() + await asyncio.sleep(0.5) + while True: + while not members_queue.empty(): + message = await members_queue.get() + logger.info(f"member msg {message}") + await asyncio.wait_for(websocket.send_json(message), timeout=5) + await asyncio.sleep(0.5) + finally: + logger.debug('overlay ws closed') + overlay_clients.discard(websocket) @self.app.route('/overlay') async def overlay(): return await send_from_directory(STATIC_DIR, 'overlay.html') async def chat_say(self, member: Member, text: str): - logger.info(f"TTS saying '{text}' from {'UNKNOWN' if not member else member.name}") # If client was connected but dc'd, this can revive connection when it runs. But also, we don't want things to queue up forever... Might not be a problem, needs hard testing later # But cannot simply do an if check here for clients await message_queue.put((member, text)) @@ -84,3 +111,21 @@ async def chat_say(self, member: Member, text: str): async def run_task(self, host="0.0.0.0", **kwargs): # TODO on port change, request app restart await self.app.run_task(host=host, port=self.config.getint(section="SERVER", option="port"), **kwargs) + + async def send_members(self, members: list[Member] = []): + user_data = [{"name": member.name, "pfp_url": member.pfp_url} for member in sorted(members)] + if not user_data: + speech_message = { + "type": "endspeech" + } + await members_queue.put(speech_message) + message = {"type": "update_users", "users": user_data} + await members_queue.put(message) + + async def animate_member(self, name, anim_type): + message = { + "type": "animate", + "name": name, + "animation": anim_type + } + await members_queue.put(message) diff --git a/src/server/static/overlay.html b/src/server/static/overlay.html index ad5ab9a..4e04dd1 100644 --- a/src/server/static/overlay.html +++ b/src/server/static/overlay.html @@ -2,12 +2,111 @@ - Audio Overlay + Chat DnD Overlay + - -

Audio Overlay

+
\ No newline at end of file diff --git a/src/tts/local_tts.py b/src/tts/local_tts.py index f7e55e9..8a1e8b6 100644 --- a/src/tts/local_tts.py +++ b/src/tts/local_tts.py @@ -43,7 +43,7 @@ def __init__(self, config: Config): self.bits_per_sample = 16 self.num_channels = 1 - self.max_chunk_size = 1024*8*8*2 # 128kb + self.max_chunk_size = 1024*8*8*2*2 # 256kb engine = pyttsx4.init() for v in engine.getProperty('voices'): @@ -78,8 +78,9 @@ async def get_stream(self, text="Hello World!", voice: str = ''): chunk = output.read(chunk_size) while chunk: - await asyncio.sleep((len(chunk) / (self.sample_rate * self.num_channels * (self.bits_per_sample // 8)))) - yield header + chunk + duration = (len(chunk) / (self.sample_rate * self.num_channels * (self.bits_per_sample // 8))) + await asyncio.sleep(duration) + yield (header + chunk, duration) chunk = output.read(chunk_size) def test_speak(self, text:str ="Hello there. How are you?", voice:str = None): diff --git a/src/twitch/chat.py b/src/twitch/chat.py index 4ab29ab..2db988d 100644 --- a/src/twitch/chat.py +++ b/src/twitch/chat.py @@ -83,6 +83,7 @@ def update_bot_settings(self): self._say, command_middleware=[UserRestriction(allowed_users=[x.name for x in self.session_mgr.session.party]), ChannelCommandCooldown(10), # TODO Config cooldown times ChannelUserCommandCooldown(15) ]) + # self.end_session() def stop(self): @@ -136,10 +137,12 @@ def start_session(self, party_size) -> bool: self.send_message(f"Not enough party members in the queue! Type {self.chat._prefix}{self.command_list['join']} to join ({len(self.session_mgr.session.queue)}/{party_size})") return False + def end_session(self): if self.session_mgr: self.session_mgr.end() - self.chat.unregister_command('say') + self.chat.unregister_command(self.command_list['say']) + self.chat.unregister_command(self.command_list['join']) chat_on_session_end.trigger() diff --git a/src/ui/tabs/home.py b/src/ui/tabs/home.py index 96cc3bd..9fe4855 100644 --- a/src/ui/tabs/home.py +++ b/src/ui/tabs/home.py @@ -5,7 +5,6 @@ from twitch.chat import ChatController from chatdnd.events.chat_events import chat_on_join_queue, chat_bot_on_connect -# TODO: Display current active party on session start as MemberCards (smaller than on users page) class HomeTab(): def __init__(self, parent, chat_ctrl: ChatController): @@ -94,6 +93,7 @@ def _fill_party_frame(self): def _allow_session_management(self, status: bool): if status: self.open_button.configure(state="normal") + self._end_session() else: self.open_button.configure(state="disabled") @@ -122,7 +122,7 @@ def _start_session(self): self.start_session.configure(state="disabled") self.end_button.configure(state="normal") self._fill_party_frame() - + def _end_session(self): self.chat_ctrl.end_session()