From e2fed64b390cf02af3515eb41a5656e2732ad488 Mon Sep 17 00:00:00 2001
From: WolverinDEV <git@did.science>
Date: Sun, 15 Nov 2020 23:28:00 +0100
Subject: [PATCH] Fixed the video connection for firefox and some minor
 bugfixes

---
 ChangeLog.md                            |  3 +
 shared/js/ConnectionHandler.ts          |  6 +-
 shared/js/tree/ChannelTree.tsx          | 31 +++++----
 shared/js/ui/frames/video/Controller.ts |  7 +-
 shared/js/ui/frames/video/Renderer.tsx  | 11 +--
 web/app/rtc/Connection.ts               | 92 +++++++++++++++++++++++--
 web/app/rtc/RemoteTrack.ts              |  6 ++
 web/app/rtc/SdpUtils.ts                 |  5 +-
 web/app/rtc/video/Connection.ts         | 27 +++++---
 web/app/rtc/voice/Connection.ts         |  2 +
 10 files changed, 149 insertions(+), 41 deletions(-)

diff --git a/ChangeLog.md b/ChangeLog.md
index b241800c..68ea38cc 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,4 +1,7 @@
 # Changelog:
+** 14.11.20**
+    - Fixed bug where the microphone has been requested when muting it.
+    
 * **07.11.20**
     - Added video broadcasting to the web client
     - Added various new user interfaces related to video broadcasting
diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts
index c012cb1e..219fe63c 100644
--- a/shared/js/ConnectionHandler.ts
+++ b/shared/js/ConnectionHandler.ts
@@ -826,11 +826,7 @@ export class ConnectionHandler {
             return;
         }
 
-        if(this.connection_state === ConnectionState.CONNECTED) {
-            await this.startVoiceRecorder(true);
-        } else {
-            this.setInputHardwareState(InputHardwareState.VALID);
-        }
+        /* our voice status will be updated automatically due to the notify_recorder_changed event which should be fired when the acquired recorder changed */
     }
 
     async startVoiceRecorder(notifyError: boolean) : Promise<{ state: "success" | "no-input" } | { state: "error", message: string }> {
diff --git a/shared/js/tree/ChannelTree.tsx b/shared/js/tree/ChannelTree.tsx
index bc119c73..ab2651a4 100644
--- a/shared/js/tree/ChannelTree.tsx
+++ b/shared/js/tree/ChannelTree.tsx
@@ -588,7 +588,7 @@ export class ChannelTree {
     deleteClient(client: ClientEntry, reason: { reason: ViewReasonId, message?: string, serverLeave: boolean }) {
         const oldChannel = client.currentChannel();
         oldChannel?.unregisterClient(client);
-        this.clients.remove(client);
+        this.unregisterClient(client);
 
         if(oldChannel) {
             this.events.fire("notify_client_leave_view", { client: client, message: reason.message, reason: reason.reason, isServerLeave: reason.serverLeave, sourceChannel: oldChannel });
@@ -597,30 +597,23 @@ export class ChannelTree {
             logWarn(LogCategory.CHANNEL, tr("Deleting client %s from channel tree which hasn't a channel."), client.clientId());
         }
 
-        const voiceConnection = this.client.serverConnection.getVoiceConnection();
-        if(client.getVoiceClient()) {
-            voiceConnection.unregisterVoiceClient(client.getVoiceClient());
-            client.setVoiceClient(undefined);
-        }
-
-        const videoConnection = this.client.serverConnection.getVideoConnection();
-        if(client.getVideoClient()) {
-            videoConnection.unregisterVideoClient(client.getVideoClient());
-            client.setVideoClient(undefined);
-        }
         client.destroy();
     }
 
     registerClient(client: ClientEntry) {
         this.clients.push(client);
 
-        if(client instanceof LocalClientEntry) {
+        const isLocalClient = client instanceof LocalClientEntry;
+        if(isLocalClient) {
             if(client.channelTree !== this) {
                 throw tr("client channel tree missmatch");
             }
         } else {
             client.channelTree = this;
+        }
 
+        /* for debug purposes, the server might send back the own audio/video stream */
+        if(!isLocalClient || __build.mode === "debug") {
             const voiceConnection = this.client.serverConnection.getVoiceConnection();
             try {
                 client.setVoiceClient(voiceConnection.registerVoiceClient(client.clientId()));
@@ -641,6 +634,18 @@ export class ChannelTree {
         if(!this.clients.remove(client)) {
             return;
         }
+
+        const voiceConnection = this.client.serverConnection.getVoiceConnection();
+        if(client.getVoiceClient()) {
+            voiceConnection.unregisterVoiceClient(client.getVoiceClient());
+            client.setVoiceClient(undefined);
+        }
+
+        const videoConnection = this.client.serverConnection.getVideoConnection();
+        if(client.getVideoClient()) {
+            videoConnection.unregisterVideoClient(client.getVideoClient());
+            client.setVideoClient(undefined);
+        }
     }
 
     insertClient(client: ClientEntry, channel: ChannelEntry, reason: { reason: ViewReasonId, isServerJoin: boolean }) : ClientEntry {
diff --git a/shared/js/ui/frames/video/Controller.ts b/shared/js/ui/frames/video/Controller.ts
index 3859ffdb..a924563e 100644
--- a/shared/js/ui/frames/video/Controller.ts
+++ b/shared/js/ui/frames/video/Controller.ts
@@ -351,7 +351,12 @@ class ChannelVideoController {
         if(localClient.currentChannel()) {
             this.currentChannelId = localClient.currentChannel().channelId;
             localClient.currentChannel().channelClientsOrdered().forEach(client => {
-                if(client instanceof LocalClientEntry || ChannelVideoController.shouldIgnoreClient(client)) {
+                /* in some instances the server might return our own stream for debug purposes */
+                if(client instanceof LocalClientEntry && __build.mode !== "debug") {
+                    return;
+                }
+
+                if(ChannelVideoController.shouldIgnoreClient(client)) {
                     return;
                 }
 
diff --git a/shared/js/ui/frames/video/Renderer.tsx b/shared/js/ui/frames/video/Renderer.tsx
index 84572d75..16988145 100644
--- a/shared/js/ui/frames/video/Renderer.tsx
+++ b/shared/js/ui/frames/video/Renderer.tsx
@@ -77,16 +77,19 @@ const VideoStreamReplay = React.memo((props: { stream: MediaStream | undefined,
     const refVideo = useRef<HTMLVideoElement>();
 
     useEffect(() => {
+        const video = refVideo.current;
         if(props.stream) {
-            refVideo.current.style.opacity = "1";
-            refVideo.current.srcObject = props.stream;
+            video.style.opacity = "1";
+            video.srcObject = props.stream;
+            video.autoplay = true;
+            video.play().then(undefined);
         } else {
-            refVideo.current.style.opacity = "0";
+            video.style.opacity = "0";
         }
     }, [ props.stream ]);
 
     return (
-        <video ref={refVideo} autoPlay={true} className={cssStyle.video + " " + props.className} />
+        <video ref={refVideo} className={cssStyle.video + " " + props.className} />
     )
 });
 
diff --git a/web/app/rtc/Connection.ts b/web/app/rtc/Connection.ts
index e10a8960..f2784056 100644
--- a/web/app/rtc/Connection.ts
+++ b/web/app/rtc/Connection.ts
@@ -14,9 +14,53 @@ import {
     TrackClientInfo
 } from "tc-backend/web/rtc/RemoteTrack";
 import {SdpCompressor, SdpProcessor} from "tc-backend/web/rtc/SdpUtils";
+import {context} from "tc-backend/web/audio/player";
 
 const kSdpCompressionMode = 1;
 
+declare global {
+    interface RTCIceCandidate {
+        /* Firefox has this */
+        address: string | undefined;
+    }
+
+    interface HTMLCanvasElement {
+        captureStream(framed: number) : MediaStream;
+    }
+}
+
+let dummyVideoTrack: MediaStreamTrack | undefined;
+let dummyAudioTrack: MediaStreamTrack | undefined;
+
+/*
+ * For Firefox as soon we stop a sender we're never able to get the sender starting again...
+ * (This only applies after the initial negotiation. Before values of null are allowed)
+ * So we've to keep it alive with a dummy track.
+ */
+function getIdleTrack(kind: "video" | "audio") : MediaStreamTrack | null {
+    if(window.detectedBrowser?.name === "firefox" || true) {
+        if(kind === "video") {
+            if(!dummyVideoTrack) {
+                const canvas = document.createElement("canvas");
+                canvas.getContext("2d");
+                const stream = canvas.captureStream(1);
+                dummyVideoTrack = stream.getVideoTracks()[0];
+            }
+
+            return dummyVideoTrack;
+        } else if(kind === "audio") {
+            if(!dummyAudioTrack) {
+                const dest = context().createMediaStreamDestination();
+                dummyAudioTrack = dest.stream.getAudioTracks()[0];
+            }
+
+            return dummyAudioTrack;
+        }
+    }
+
+    return null;
+}
+
 class CommandHandler extends AbstractCommandHandler {
     private readonly handle: RTCConnection;
     private readonly sdpProcessor: SdpProcessor;
@@ -71,9 +115,20 @@ class CommandHandler extends AbstractCommandHandler {
                     sdp: sdp,
                     type: "offer"
                 }).then(() => this.handle["peer"].createAnswer())
+                .then(async answer => {
+                    await this.handle["peer"].setLocalDescription(answer);
+                    return answer;
+                })
                 .then(answer => {
+                    if(RTCConnection.kEnableSdpTrace) {
+                        logTrace(LogCategory.WEBRTC, tr("Sending answer to remote %s:\n%s"), data.mode, answer.sdp);
+                    }
                     answer.sdp = this.sdpProcessor.processOutgoingSdp(answer.sdp, "answer");
+
                     answer.sdp = SdpCompressor.compressSdp(answer.sdp, kSdpCompressionMode);
+                    if(RTCConnection.kEnableSdpTrace) {
+                        logTrace(LogCategory.WEBRTC, tr("Patched answer to remote %s:\n%s"), data.mode, answer.sdp);
+                    }
 
                     return this.connection.send_command("rtcsessiondescribe", {
                         mode: "answer",
@@ -295,7 +350,7 @@ export interface RTCConnectionEvents {
 }
 
 export class RTCConnection {
-    public static readonly kEnableSdpTrace = false;
+    public static readonly kEnableSdpTrace = true;
     private readonly events: Registry<RTCConnectionEvents>;
     private readonly connection: ServerConnection;
     private readonly commandHandler: CommandHandler;
@@ -304,6 +359,8 @@ export class RTCConnection {
     private connectionState: RTPConnectionState;
     private failedReason: string;
 
+    private retryTimeout: number;
+
     private peer: RTCPeerConnection;
     private localCandidateCount: number;
 
@@ -392,6 +449,9 @@ export class RTCConnection {
         this.temporaryStreams = {};
         this.localCandidateCount = 0;
 
+        clearTimeout(this.retryTimeout);
+        this.retryTimeout = 0;
+
         if(updateConnectionState) {
             this.updateConnectionState(RTPConnectionState.DISCONNECTED);
         }
@@ -461,7 +521,7 @@ export class RTCConnection {
 
         /* FIXME: Generate a log message! */
         if(retryThreshold > 0) {
-            setTimeout(() => {
+            this.retryTimeout = setTimeout(() => {
                 console.error("XXXX Retry");
                 this.doInitialSetup();
             }, 5000);
@@ -497,7 +557,7 @@ export class RTCConnection {
             iceServers: [{ urls: ["stun:stun.l.google.com:19302", "stun:stun1.l.google.com:19302"] }]
         });
 
-        this.currentTransceiver["audio"] = this.peer.addTransceiver("audio", { sendEncodings: [{ }] });
+        this.currentTransceiver["audio"] = this.peer.addTransceiver("audio");
         this.enableDtx(this.currentTransceiver["audio"].sender);
 
         this.currentTransceiver["audio-whisper"] = this.peer.addTransceiver("audio");
@@ -537,7 +597,19 @@ export class RTCConnection {
 
     private async updateTracks() {
         for(const type of kRtcSourceTrackTypes) {
-            await this.currentTransceiver[type]?.sender.replaceTrack(this.currentTracks[type]);
+            let fallback;
+            switch (type) {
+                case "audio":
+                case "audio-whisper":
+                    fallback = getIdleTrack("audio");
+                    break;
+
+                case "video":
+                case "video-screen":
+                    fallback = getIdleTrack("video");
+                    break;
+            }
+            await this.currentTransceiver[type]?.sender.replaceTrack(this.currentTracks[type] || fallback);
         }
     }
 
@@ -546,6 +618,7 @@ export class RTCConnection {
 
         const peer = this.peer;
         await this.updateTracks();
+
         const offer = await peer.createOffer({ iceRestart: false, offerToReceiveAudio: true, offerToReceiveVideo: true });
         if(offer.type !== "offer") { throw tr("created ofer isn't of type offer"); }
         if(this.peer !== peer) { return; }
@@ -595,10 +668,14 @@ export class RTCConnection {
 
     private handleIceCandidate(candidate: RTCIceCandidate | undefined) {
         if(candidate) {
+            if(candidate.address?.endsWith(".local")) {
+                logTrace(LogCategory.WEBRTC, tr("Skipping local fqdn ICE candidate %s"), candidate.toJSON().candidate);
+                return;
+            }
             this.localCandidateCount++;
 
             const json = candidate.toJSON();
-            logTrace(LogCategory.WEBRTC, tr("Received ICE candidate %s"), json.candidate);
+            logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate %s"), json.candidate);
             this.connection.send_command("rtcicecandidate", {
                 media_line: json.sdpMLineIndex,
                 candidate: json.candidate
@@ -611,7 +688,7 @@ export class RTCConnection {
                 this.handleFatalError(tr("Failed to gather any ICE candidates"), 0);
                 return;
             } else {
-                logTrace(LogCategory.WEBRTC, tr("Received ICE candidate finish"));
+                logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate finish"));
             }
             this.connection.send_command("rtcicecandidate", { }).catch(error => {
                 logWarn(LogCategory.WEBRTC, tr("Failed to transmit local ICE candidate finish to server: %o"), error);
@@ -644,7 +721,8 @@ export class RTCConnection {
         logTrace(LogCategory.WEBRTC, tr("Peer connection state changed to %s"), this.peer.connectionState);
         switch (this.peer.connectionState) {
             case "connecting":
-                this.updateConnectionState(RTPConnectionState.CONNECTING);
+                //this.updateConnectionState(RTPConnectionState.CONNECTING);
+                this.updateConnectionState(RTPConnectionState.CONNECTED);
                 break;
 
             case "connected":
diff --git a/web/app/rtc/RemoteTrack.ts b/web/app/rtc/RemoteTrack.ts
index 9306100e..6486226b 100644
--- a/web/app/rtc/RemoteTrack.ts
+++ b/web/app/rtc/RemoteTrack.ts
@@ -101,6 +101,12 @@ export class RemoteRTPVideoTrack extends RemoteRTPTrack {
 
         this.mediaStream = new MediaStream();
         this.mediaStream.addTrack(transceiver.receiver.track);
+
+        const track = transceiver.receiver.track;
+        track.onended = () => console.error("TRACK %d ended", ssrc);
+        track.onmute = () => console.error("TRACK %d muted", ssrc);
+        track.onunmute = () => console.error("TRACK %d unmuted", ssrc);
+        track.onisolationchange = () => console.error("TRACK %d onisolationchange", ssrc);
     }
 
     getMediaStream() : MediaStream {
diff --git a/web/app/rtc/SdpUtils.ts b/web/app/rtc/SdpUtils.ts
index e2eb5511..12c59336 100644
--- a/web/app/rtc/SdpUtils.ts
+++ b/web/app/rtc/SdpUtils.ts
@@ -1,6 +1,5 @@
 import * as sdpTransform from "sdp-transform";
 import {MediaDescription} from "sdp-transform";
-import {guid} from "tc-shared/crypto/uid";
 
 interface SdpCodec {
     payload: number;
@@ -14,7 +13,7 @@ interface SdpCodec {
 /* These MUST be the payloads used by the remote as well */
 const OPUS_VOICE_PAYLOAD_TYPE = 111;
 const OPUS_MUSIC_PAYLOAD_TYPE = 112;
-const VP8_PAYLOAD_TYPE = 96;
+const VP8_PAYLOAD_TYPE = 122; /* Using 122 for testing purposes */ //96;
 
 type SdpMedia = {
     type: string;
@@ -31,7 +30,7 @@ export class SdpProcessor {
             codec: "opus",
             rate: 48000,
             encoding: 2,
-            fmtp: { minptime: 1, maxptime: 20, useinbandfec: 1, stereo: 1 },
+            fmtp: { minptime: 1, maxptime: 20, useinbandfec: 1, stereo: 0 },
             rtcpFb: [ "transport-cc" ]
         },
         {
diff --git a/web/app/rtc/video/Connection.ts b/web/app/rtc/video/Connection.ts
index e694af4b..0fa85687 100644
--- a/web/app/rtc/video/Connection.ts
+++ b/web/app/rtc/video/Connection.ts
@@ -13,6 +13,7 @@ import {LogCategory, logDebug, logError, logWarn} from "tc-shared/log";
 import {Settings, settings} from "tc-shared/settings";
 import {RtpVideoClient} from "tc-backend/web/rtc/video/VideoClient";
 import {tr} from "tc-shared/i18n/localize";
+import {ConnectionState} from "tc-shared/ConnectionHandler";
 
 type VideoBroadcast = {
     readonly source: VideoSource;
@@ -26,6 +27,7 @@ export class RtpVideoConnection implements VideoConnection {
     private readonly events: Registry<VideoConnectionEvent>;
     private readonly listenerClientMoved;
     private readonly listenerRtcStateChanged;
+    private readonly listenerConnectionStateChanged;
     private connectionState: VideoConnectionStatus;
 
     private broadcasts: {[T in VideoBroadcastType]: VideoBroadcast} = {
@@ -54,6 +56,12 @@ export class RtpVideoConnection implements VideoConnection {
                 }
             }
         });
+        this.listenerConnectionStateChanged = this.rtcConnection.getConnection().events.on("notify_connection_state_changed", event => {
+            if(event.newState !== ConnectionState.CONNECTED) {
+                this.stopBroadcasting("camera");
+                this.stopBroadcasting("screen");
+            }
+        });
 
         this.listenerRtcStateChanged = this.rtcConnection.getEvents().on("notify_state_changed", event => this.handleRtcConnectionStateChanged(event));
 
@@ -95,6 +103,7 @@ export class RtpVideoConnection implements VideoConnection {
     destroy() {
         this.listenerClientMoved();
         this.listenerRtcStateChanged();
+        this.listenerConnectionStateChanged();
     }
 
     getEvents(): Registry<VideoConnectionEvent> {
@@ -163,20 +172,22 @@ export class RtpVideoConnection implements VideoConnection {
     }
 
     stopBroadcasting(type: VideoBroadcastType, skipRtcStop?: boolean) {
+        const broadcast = this.broadcasts[type];
+        if(!broadcast) {
+            return;
+        }
+
         if(!skipRtcStop) {
             this.rtcConnection.stopTrackBroadcast(type === "camera" ? "video" : "video-screen");
         }
 
         this.rtcConnection.setTrackSource(type === "camera" ? "video" : "video-screen", null).then(undefined);
-        if(this.broadcasts[type]) {
-            const broadcast = this.broadcasts[type];
-            const oldState = this.broadcasts[type].state;
-            this.broadcasts[type].active = false;
-            this.broadcasts[type] = undefined;
-            broadcast.source.deref();
+        const oldState = this.broadcasts[type].state;
+        this.broadcasts[type].active = false;
+        this.broadcasts[type] = undefined;
+        broadcast.source.deref();
 
-            this.events.fire("notify_local_broadcast_state_changed", { oldState: oldState, newState: VideoBroadcastState.Stopped, broadcastType: type });
-        }
+        this.events.fire("notify_local_broadcast_state_changed", { oldState: oldState, newState: VideoBroadcastState.Stopped, broadcastType: type });
     }
 
     registerVideoClient(clientId: number) {
diff --git a/web/app/rtc/voice/Connection.ts b/web/app/rtc/voice/Connection.ts
index c38d09f5..eb1e2640 100644
--- a/web/app/rtc/voice/Connection.ts
+++ b/web/app/rtc/voice/Connection.ts
@@ -49,6 +49,8 @@ export class RtpVoiceConnection extends AbstractVoiceConnection {
         this.rtcConnection.getEvents().on("notify_state_changed",
             this.listenerRtcStateChanged = event => this.handleRtcConnectionStateChanged(event));
 
+        /* FIXME: Listener for audio! */
+
         this.listenerClientMoved = this.rtcConnection.getConnection().command_handler_boss().register_explicit_handler("notifyclientmoved", event => {
             const localClientId = this.rtcConnection.getConnection().client.getClientId();
             for(const data of event.arguments) {