New API (#125)

src/client/Streamer.ts

@@ -46,11 +46,11 @@ export class Streamer {
                 this.client.user.id,
                 channel_id,
                 options ?? {},
-                (voiceUdp) => {
-                    resolve(voiceUdp);
+                (udp) => {
+                    udp.mediaConnection.setProtocols().then(() => resolve(udp))
                 }
             );
-            this.signalVideo(guild_id, channel_id, false);
+            this.signalVideo(false);
         });
     }
 
@@ -67,18 +67,15 @@ export class Streamer {
                 return;
             }
 
-            this.signalStream(
-                this.voiceConnection.guildId,
-                this.voiceConnection.channelId
-            );
+            this.signalStream();
 
             this.voiceConnection.streamConnection = new StreamConnection(
                 this.voiceConnection.guildId,
                 this.client.user.id,
                 this.voiceConnection.channelId,
                 options ?? {},
-                (voiceUdp) => {
-                    resolve(voiceUdp);
+                (udp) => {
+                    udp.mediaConnection.setProtocols().then(() => resolve(udp))
                 }
             );
         });
@@ -91,7 +88,7 @@ export class Streamer {
 
         stream.stop();
 
-        this.signalStopStream(stream.guildId, stream.channelId);
+        this.signalStopStream();
 
         this.voiceConnection.streamConnection = undefined;
     }
@@ -104,7 +101,13 @@ export class Streamer {
         this._voiceConnection = undefined;
     }
 
-    public signalVideo(guild_id: string, channel_id: string, video_enabled: boolean): void {
+    public signalVideo(video_enabled: boolean): void {
+        if (!this.voiceConnection)
+            return;
+        const {
+            guildId: guild_id,
+            channelId: channel_id
+        } = this.voiceConnection;
         this.sendOpcode(GatewayOpCodes.VOICE_STATE_UPDATE, {
             guild_id,
             channel_id,
@@ -114,7 +117,13 @@ export class Streamer {
         });
     }
 
-    public signalStream(guild_id: string, channel_id: string): void {
+    public signalStream(): void {
+        if (!this.voiceConnection)
+            return;
+        const {
+            guildId: guild_id,
+            channelId: channel_id
+        } = this.voiceConnection;
         this.sendOpcode(GatewayOpCodes.STREAM_CREATE, {
             type: "guild",
             guild_id,
@@ -128,7 +137,13 @@ export class Streamer {
         });
     }
 
-    public signalStopStream(guild_id: string, channel_id: string): void {
+    public signalStopStream(): void {
+        if (!this.voiceConnection)
+            return;
+        const {
+            guildId: guild_id,
+            channelId: channel_id
+        } = this.voiceConnection;
         this.sendOpcode(GatewayOpCodes.STREAM_DELETE, {
             stream_key: `guild:${guild_id}:${channel_id}:${this.client.user!.id}`
         });

src/client/packet/AudioPacketizer.ts

@@ -1,9 +1,10 @@
 import { MediaUdp } from "../voice/MediaUdp.js";
 import { BaseMediaPacketizer } from "./BaseMediaPacketizer.js";
+import { CodecPayloadType } from "../voice/BaseMediaConnection.js";
 
 export class AudioPacketizer extends BaseMediaPacketizer {
     constructor(connection: MediaUdp) {
-        super(connection, 0x78);
+        super(connection, CodecPayloadType.opus.payload_type);
         this.srInterval = 5 * 1000 / 20; // ~5 seconds for 20ms frame time
     }
 

src/client/packet/BaseMediaPacketizer.ts

@@ -13,7 +13,7 @@ let sodium: Promise<sp.SodiumPlus> | undefined;
 export class BaseMediaPacketizer {
     private _loggerRtcpSr = new Log("packetizer:rtcp-sr");
 
-    private _ssrc: number;
+    private _ssrc?: number;
     private _payloadType: number;
     private _mtu: number;
     private _sequence: number;
@@ -40,11 +40,10 @@ export class BaseMediaPacketizer {
         this._mtu = 1200;
         this._extensionEnabled = extensionEnabled;
 
-        this._ssrc = 0;
         this._srInterval = 512; // Sane fallback value for interval
     }
 
-    public get ssrc(): number
+    public get ssrc(): number | undefined
     {
         return this._ssrc;
     }
@@ -129,6 +128,9 @@ export class BaseMediaPacketizer {
     }
 
     public makeRtpHeader(isLastPacket: boolean = true): Buffer {
+        if (!this._ssrc)
+            throw new Error("SSRC is not set");
+
         const packetHeader = Buffer.alloc(12);
 
         packetHeader[0] = 2 << 6 | ((this._extensionEnabled ? 1 : 0) << 4); // set version and flags
@@ -143,6 +145,9 @@ export class BaseMediaPacketizer {
     }
 
     public async makeRtcpSenderReport(): Promise<Buffer> {
+        if (!this._ssrc)
+            throw new Error("SSRC is not set");
+
         const packetHeader = Buffer.allocUnsafe(8);
 
         packetHeader[0] = 0x80; // RFC1889 v2, no padding, no reception report count

src/client/packet/VideoPacketizerAnnexB.ts

@@ -7,6 +7,7 @@ import {
 } from "../processing/AnnexBHelper.js";
 import { extensions } from "../../utils.js";
 import { splitNalu } from "../processing/AnnexBHelper.js";
+import { CodecPayloadType } from "../voice/BaseMediaConnection.js";
 
 /**
  * Annex B format
@@ -60,8 +61,8 @@ import { splitNalu } from "../processing/AnnexBHelper.js";
 class VideoPacketizerAnnexB extends BaseMediaPacketizer {
     private _nalFunctions: AnnexBHelpers;
 
-    constructor(connection: MediaUdp, nalFunctions: AnnexBHelpers) {
-        super(connection, 0x65, true);
+    constructor(connection: MediaUdp, payloadType: number, nalFunctions: AnnexBHelpers) {
+        super(connection, payloadType, true);
         this.srInterval = 5 * connection.mediaConnection.streamOptions.fps * 3; // ~5 seconds, assuming ~3 packets per frame
         this._nalFunctions = nalFunctions;
     }
@@ -157,7 +158,7 @@ class VideoPacketizerAnnexB extends BaseMediaPacketizer {
 
 export class VideoPacketizerH264 extends VideoPacketizerAnnexB {
     constructor(connection: MediaUdp) {
-        super(connection, H264Helpers);
+        super(connection, CodecPayloadType.H264.payload_type, H264Helpers);
     }
     /**
      * The FU indicator octet has the following format:
@@ -213,7 +214,7 @@ export class VideoPacketizerH264 extends VideoPacketizerAnnexB {
 
 export class VideoPacketizerH265 extends VideoPacketizerAnnexB {
     constructor(connection: MediaUdp) {
-        super(connection, H265Helpers);
+        super(connection, CodecPayloadType.H265.payload_type, H265Helpers);
     }
     /**
      * The FU indicator octet has the following format:

src/client/packet/VideoPacketizerVP8.ts

@@ -1,6 +1,7 @@
 import { extensions, max_int16bit } from "../../utils.js";
 import { MediaUdp } from "../voice/MediaUdp.js";
 import { BaseMediaPacketizer } from "./BaseMediaPacketizer.js";
+import { CodecPayloadType } from "../voice/BaseMediaConnection.js";
 
 /**
  * VP8 payload format
@@ -10,7 +11,7 @@ export class VideoPacketizerVP8 extends BaseMediaPacketizer {
     private _pictureId: number;
 
     constructor(connection: MediaUdp) {
-        super(connection, 0x65, true);
+        super(connection, CodecPayloadType.VP8.payload_type, true);
         this._pictureId = 0;
         this.srInterval = 5 * connection.mediaConnection.streamOptions.fps * 3; // ~5 seconds, assuming ~3 packets per frame
     }

src/client/voice/BaseMediaConnection.ts

@@ -5,6 +5,7 @@ import { MediaUdp } from "./MediaUdp.js";
 import { normalizeVideoCodec, STREAMS_SIMULCAST, SupportedEncryptionModes, SupportedVideoCodec } from "../../utils.js";
 import type { ReadyMessage, SelectProtocolAck } from "./VoiceMessageTypes.js";
 import WebSocket from 'ws';
+import EventEmitter from "node:events";
 
 type VoiceConnectionStatus =
 {
@@ -14,6 +15,27 @@ type VoiceConnectionStatus =
     resuming: boolean;
 }
 
+export const CodecPayloadType = {
+    "opus": {
+        name: "opus", type: "audio", priority: 1000, payload_type: 120
+    },
+    "H264": {
+        name: "H264", type: "video", priority: 1000, payload_type: 101, rtx_payload_type: 102, encode: true, decode: true
+    },
+    "H265": {
+        name: "H265", type: "video", priority: 1000, payload_type: 103, rtx_payload_type: 104, encode: true, decode: true
+    },
+    "VP8": {
+        name: "VP8", type: "video", priority: 1000, payload_type: 105, rtx_payload_type: 106, encode: true, decode: true
+    },
+    "VP9": {
+        name: "VP9", type: "video", priority: 1000, payload_type: 107, rtx_payload_type: 108, encode: true, decode: true
+    },
+    "AV1": {
+        name: "AV1", type: "video", priority: 1000, payload_type: 107, rtx_payload_type: 108, encode: true, decode: true
+    }
+}
+
 export interface StreamOptions {
     /**
      * Video output width
@@ -76,7 +98,7 @@ const defaultStreamOptions: StreamOptions = {
     forceChacha20Encryption: false,
 }
 
-export abstract class BaseMediaConnection {
+export abstract class BaseMediaConnection extends EventEmitter {
     private interval: NodeJS.Timeout | null = null;
     public udp: MediaUdp;
     public guildId: string;
@@ -97,8 +119,10 @@ export abstract class BaseMediaConnection {
     public secretkeyAes256: Promise<webcrypto.CryptoKey> | null = null;
     public secretkeyChacha20: sp.CryptographyKey | null = null;
     private _streamOptions: StreamOptions;
+    private _supportedEncryptionMode?: SupportedEncryptionModes[];
 
     constructor(guildId: string, botId: string, channelId: string, options: Partial<StreamOptions>, callback: (udp: MediaUdp) => void) {
+        super();
         this.status = {
             hasSession: false,
             hasToken: false,
@@ -194,23 +218,13 @@ export abstract class BaseMediaConnection {
         this.ssrc = d.ssrc;
         this.address = d.ip;
         this.port = d.port;
-
-        // select encryption mode
-        // From Discord docs: 
-        // You must support aead_xchacha20_poly1305_rtpsize. You should prefer to use aead_aes256_gcm_rtpsize when it is available.
-        if(d.modes.includes(SupportedEncryptionModes.AES256) && !this.streamOptions.forceChacha20Encryption) {
-            this.udp.encryptionMode = SupportedEncryptionModes.AES256
-        } else {
-            this.udp.encryptionMode = SupportedEncryptionModes.XCHACHA20
-        }
+        this._supportedEncryptionMode = d.modes;
 
         // we hardcoded the STREAMS_SIMULCAST, which will always be array of 1
         const stream = d.streams[0];
         this.videoSsrc = stream.ssrc;
         this.rtxSsrc = stream.rtx_ssrc;
-
-        this.udp.audioPacketizer.ssrc = this.ssrc;
-        this.udp.videoPacketizer.ssrc = this.videoSsrc;
+        this.udp.updatePacketizer();
     }
 
     handleProtocolAck(d: SelectProtocolAck): void {
@@ -224,9 +238,7 @@ export abstract class BaseMediaConnection {
             false, ["encrypt"]
         );
         this.secretkeyChacha20 = new sp.CryptographyKey(this.secretkey);
-
-        this.ready(this.udp);
-        this.udp.ready = true;
+        this.emit("select_protocol_ack");
     }
 
     setupEvents(): void {
@@ -235,7 +247,7 @@ export abstract class BaseMediaConnection {
 
             if (op == VoiceOpCodes.READY) { // ready
                 this.handleReady(d);
-                this.sendVoice();
+                this.sendVoice().then(() => this.ready(this.udp));
                 this.setVideoStatus(false);
             }
             else if (op >= 4000) {
@@ -306,24 +318,34 @@ export abstract class BaseMediaConnection {
     ** Uses vp8 for video
     ** Uses opus for audio
     */
-    setProtocols(ip: string, port: number): void {
-        this.sendOpcode(VoiceOpCodes.SELECT_PROTOCOL, {
-            protocol: "udp",
-            codecs: [
-                { name: "opus", type: "audio", priority: 1000, payload_type: 120 },
-                { name: normalizeVideoCodec(this.streamOptions.videoCodec), type: "video", priority: 1000, payload_type: 101, rtx_payload_type: 102, encode: true, decode: true}
-                //{ name: "VP8", type: "video", priority: 3000, payload_type: 103, rtx_payload_type: 104, encode: true, decode: true }
-                //{ name: "VP9", type: "video", priority: 3000, payload_type: 105, rtx_payload_type: 106 },
-            ],
-            data: {
+    setProtocols(): Promise<void> {
+        const { ip, port } = this.udp;
+        // select encryption mode
+        // From Discord docs: 
+        // You must support aead_xchacha20_poly1305_rtpsize. You should prefer to use aead_aes256_gcm_rtpsize when it is available.
+        if (
+            this._supportedEncryptionMode!.includes(SupportedEncryptionModes.AES256) &&
+            !this.streamOptions.forceChacha20Encryption
+        ) {
+            this.udp.encryptionMode = SupportedEncryptionModes.AES256
+        } else {
+            this.udp.encryptionMode = SupportedEncryptionModes.XCHACHA20
+        }
+        return new Promise((resolve) => {
+            this.sendOpcode(VoiceOpCodes.SELECT_PROTOCOL, {
+                protocol: "udp",
+                codecs: Object.values(CodecPayloadType),
+                data: {
+                    address: ip,
+                    port: port,
+                    mode: this.udp.encryptionMode
+                },
                 address: ip,
                 port: port,
                 mode: this.udp.encryptionMode
-            },
-            address: ip,
-            port: port,
-            mode: this.udp.encryptionMode
-        });
+            });
+            this.once("select_protocol_ack", () => resolve());
+        })
     }
 
     /*
@@ -372,10 +394,6 @@ export abstract class BaseMediaConnection {
     ** Start media connection
     */
     public sendVoice(): Promise<void> {
-        return new Promise<void>((resolve, reject) => {
-            this.udp.createUdp().then(() => {
-                resolve();
-            });
-        })
+        return this.udp.createUdp();
     }
 }