add webhook launcher logger

launcher.py

@@ -1,4 +1,7 @@
 import asyncio
+from datetime import datetime
+
+import discord
 import logging
 import multiprocessing
 import os
@@ -11,7 +14,7 @@
 
 # from bot_mp import ClusterBot
 from bot.bot import Abyss
-from config import TOKEN, SPLASHES
+from config import DEBUG_WEBHOOK, TOKEN, SPLASHES
 
 log = logging.getLogger("Cluster#Launcher")
 log.setLevel(logging.DEBUG)
@@ -29,6 +32,8 @@
 )
 NAMES = iter(CLUSTER_NAMES)
 
+webhook_logger = discord.Webhook.from_url(DEBUG_WEBHOOK, adapter=discord.RequestsWebhookAdapter())
+
 
 def get_shard_count():
     data = requests.get('https://discordapp.com/api/v7/gateway/bot', headers={
@@ -58,12 +63,26 @@ def __init__(self, loop, *, ipc=False):
         self.start_ipc = ipc
         self.ipc = None
 
+    def info(self, message):
+        embed = discord.Embed(colour=discord.Colour.green(), title=message, timestamp=datetime.utcnow())
+        webhook_logger.send(embed=embed)
+
+    def warn(self, message):
+        embed = discord.Embed(colour=discord.Colour.gold(), title=message, timestamp=datetime.utcnow())
+        webhook_logger.send(embed=embed)
+
+    def error(self, message):
+        embed = discord.Embed(colour=discord.Colour.red(), title=message, timestamp=datetime.utcnow())
+        webhook_logger.send(embed=embed)
+
     def start(self):
+        self.info("[Launcher] Starting up")
         self.fut = asyncio.ensure_future(self.startup(), loop=self.loop)
 
         try:
             self.loop.run_forever()
         except KeyboardInterrupt:
+            self.info("[Launcher] Received KeyboardInterrupt")
             self.shutdown()
         finally:
             self.cleanup()
@@ -85,6 +104,7 @@ def _cleanup(self):
             self.loop.run_until_complete(self.loop.shutdown_default_executor())
 
     def cleanup(self):
+        self.info("[Launcher] Cleaning up tasks")
         self._cleanup()
         self.loop.stop()
         if sys.platform == 'win32':
@@ -111,15 +131,18 @@ async def startup(self):
         shards = list(range(get_shard_count()))
         size = [shards[x:x + 4] for x in range(0, len(shards), 4)]
         log.info(f"Preparing {len(size)} clusters")
+        self.info(f"[Launcher] Starting {len(size)}C / {len(shards)}S")
         for shard_ids in size:
             self.cluster_queue.append(Cluster(self, next(NAMES), shard_ids, len(shards)))
 
         await self.start_cluster()
         self.keep_alive = self.loop.create_task(self.rebooter())
         self.keep_alive.add_done_callback(self.task_complete)
         log.info(f"Startup completed in {time.perf_counter() - self.init:.2f}s")
+        self.info(f"Startup completed in {time.perf_counter() - self.init:.2f}s")
 
     def shutdown(self):
+        self.info("Shutting down clusters")
         log.info("Shutting down clusters")
         self.alive = False
         if self.keep_alive:
@@ -134,6 +157,7 @@ async def rebooter(self):
             # log.info("Cycle!")
             if not self.clusters and self.alive:
                 self.alive = False
+                self.warn("[Launcher] found all clusters dead")
                 log.warning("All clusters appear to be dead")
                 raise KeyboardInterrupt
 
@@ -146,10 +170,11 @@ async def rebooter(self):
                 if not cluster.process.is_alive():
                     if cluster.process.exitcode != 0:
                         # ignore safe exits
-                        log.info(f"Cluster#{cluster.name} exited with code {cluster.process.exitcode}")
-                        log.info(f"Restarting cluster#{cluster.name}")
+                        self.warn(f'[Cluster#{cluster.name}] Exited with status {cluster.process.exitcode}, restarting')
+                        log.info(f"Cluster#{cluster.name} exited with code {cluster.process.exitcode}, restarting")
                         await cluster.start()
                     else:
+                        self.warn(f"[Launcher] Found Cluster#{cluster.name} dead with status 0.")
                         log.info(f"Cluster#{cluster.name} found dead")
                         to_remove.append(cluster)
                         cluster.stop()  # ensure stopped
@@ -160,13 +185,15 @@ async def rebooter(self):
     async def start_cluster(self):
         if self.cluster_queue:
             cluster = self.cluster_queue.pop(0)
+            self.info(f"[Launcher] Starting Cluster#{cluster.name}")
             log.info(f"Starting Cluster#{cluster.name}")
             await cluster.start()
             log.info("Done!")
             self.clusters.append(cluster)
             await self.start_cluster()
         else:
             log.info("All clusters launched")
+            self.info("[Launcher] Successfully launched all clusters")
 
 
 class Cluster:
@@ -188,12 +215,25 @@ def __init__(self, launcher, name, shard_ids, max_shards):
         self.log.handlers = [hdlr, fhdlr]
         self.log.info(f"Initialized with shard ids {shard_ids}, total shards {max_shards}")
 
+    def info(self, message):
+        embed = discord.Embed(colour=discord.Colour.green(), title=message, timestamp=datetime.utcnow())
+        webhook_logger.send(embed=embed)
+
+    def warn(self, message):
+        embed = discord.Embed(colour=discord.Colour.gold(), title=message, timestamp=datetime.utcnow())
+        webhook_logger.send(embed=embed)
+
+    def error(self, message):
+        embed = discord.Embed(colour=discord.Colour.red(), title=message, timestamp=datetime.utcnow())
+        webhook_logger.send(embed=embed)
+
     def wait_close(self):
         return self.process.join()
 
     async def start(self, *, force=False):
         if self.process and self.process.is_alive():
             if not force:
+                self.warn(f"[Cluster#{self.name}] Attempted to restart while running")
                 self.log.warning("Start called with already running cluster, pass `force=True` to override")
                 return
             self.log.info("Terminating existing process")
@@ -210,10 +250,12 @@ async def start(self, *, force=False):
         if await self.launcher.loop.run_in_executor(None, stdout.recv) == 1:
             stdout.close()
             self.log.info("Process started successfully")
+            self.info(f"[Cluster#{self.name}] Successfully loaded")
 
         return True
 
     def stop(self, sign=signal.SIGINT):
+        self.info(f"[Cluster#{self.name}] Requested to close with signal {sign!r}")
         self.log.info(f"Shutting down with signal {sign!r}")
         try:
             os.kill(self.process.pid, sign)