Improved RTC connection handling and logging

shared/js/ConnectionHandler.ts

@@ -194,12 +194,7 @@ export class ConnectionHandler {
             this.setInputHardwareState(this.getVoiceRecorder() ? InputHardwareState.VALID : InputHardwareState.MISSING);
             this.update_voice_status();
         });
-        this.serverConnection.getVoiceConnection().events.on("notify_connection_status_changed", event => {
+        this.serverConnection.getVoiceConnection().events.on("notify_connection_status_changed", () => this.update_voice_status());
-            this.update_voice_status();
-            if(event.newStatus === VoiceConnectionStatus.Failed) {
-                createErrorModal(tr("Voice connection failed"), tra("Failed to establish a voice connection:\n{}", this.serverConnection.getVoiceConnection().getFailedMessage() || tr("Lookup the console for more detail"))).open();
-            }
-        });
         this.serverConnection.getVoiceConnection().setWhisperSessionInitializer(this.initializeWhisperSession.bind(this));
 
         this.serverFeatures = new ServerFeatures(this);

shared/js/connection/DummyVoiceConnection.ts

@@ -154,4 +154,8 @@ export class DummyVoiceConnection extends AbstractVoiceConnection {
             bytesSend: 0
         }
     }
+
+    getRetryTimestamp(): number | 0 {
+        return 0;
+    }
 }

shared/js/connection/VideoConnection.ts

@@ -45,6 +45,9 @@ export interface VideoConnection {
     getEvents() : Registry<VideoConnectionEvent>;
 
     getStatus() : VideoConnectionStatus;
+    getRetryTimestamp() : number | 0;
+    getFailedMessage() : string;
+
     getConnectionStats() : Promise<ConnectionStatistics>;
 
     isBroadcasting(type: VideoBroadcastType);

shared/js/connection/VoiceConnection.ts

@@ -62,6 +62,8 @@ export abstract class AbstractVoiceConnection {
 
     abstract getConnectionState() : VoiceConnectionStatus;
     abstract getFailedMessage() : string;
+    abstract getRetryTimestamp() : number | 0;
+    
     abstract getConnectionStats() : Promise<ConnectionStatistics>;
 
     abstract encodingSupported(codec: number) : boolean;

shared/js/ui/frames/footer/Renderer.scss

@@ -110,6 +110,10 @@
             font-size: .8em;
             text-align: left;
             line-height: 1.2em;
+
+            &.error {
+                color: #a63030;
+            }
         }
 
         &.title {
@@ -120,8 +124,20 @@
             }
         }
 
-        &.doubleSize {
+        &.error {
-            height: 2.8em;
+            justify-content: flex-start;
+            flex-direction: column;
+
+            min-height: 2.8em;
+            height: min-content;
         }
     }
+
+    .errorRow {
+        display: flex;
+        flex-direction: row;
+        justify-content: space-between;
+
+        width: 100%;
+    }
 }

shared/js/ui/frames/footer/StatusController.ts

@@ -97,8 +97,7 @@ export class StatusController {
                 const videoConnection = this.currentConnectionHandler.getServerConnection().getVideoConnection();
                 switch (videoConnection.getStatus()) {
                     case VideoConnectionStatus.Failed:
-                        /* FIXME: Reason! */
+                        return { type: "unhealthy", reason: videoConnection.getFailedMessage(), retryTimestamp: videoConnection.getRetryTimestamp() };
-                        return { type: "unhealthy", reason: tr("Unknown") };
                     case VideoConnectionStatus.Connected:
                         if(detailed) {
                             const statistics = await videoConnection.getConnectionStats();
@@ -130,7 +129,7 @@ export class StatusController {
                 const voiceConnection = this.currentConnectionHandler.getServerConnection().getVoiceConnection();
                 switch (voiceConnection.getConnectionState()) {
                     case VoiceConnectionStatus.Failed:
-                        return { type: "unhealthy", reason: voiceConnection.getFailedMessage() };
+                        return { type: "unhealthy", reason: voiceConnection.getFailedMessage(), retryTimestamp: voiceConnection.getRetryTimestamp() };
 
                     case VoiceConnectionStatus.Connected:
                         if(detailed) {
@@ -205,11 +204,11 @@ export class StatusController {
             } else if(componentState.type === "disconnected" && component !== "signaling") {
                 switch (component) {
                     case "voice":
-                        componentState = { type: "unhealthy", reason: tr("No voice connection") };
+                        componentState = { type: "unhealthy", reason: tr("No voice connection"), retryTimestamp: 0 };
                         break;
 
                     case "video":
-                        componentState = { type: "unhealthy", reason: tr("No video connection") };
+                        componentState = { type: "unhealthy", reason: tr("No video connection"), retryTimestamp: 0 };
                         break;
                 }
             }

shared/js/ui/frames/footer/StatusDefinitions.ts

@@ -6,7 +6,7 @@ export type ConnectionStatus = {
 } | {
     type: "unhealthy",
     reason: string,
-    /* try reconnect attribute */
+    retryTimestamp: number
 } | {
     type: "connecting-signalling",
     state: "initializing" | "connecting" | "authentication"

shared/js/ui/frames/footer/StatusRenderer.tsx

@@ -6,8 +6,9 @@ import {
 } from "tc-shared/ui/frames/footer/StatusDefinitions";
 import * as React from "react";
 import {useContext, useEffect, useRef, useState} from "react";
-import {Translatable} from "tc-shared/ui/react-elements/i18n";
+import {Translatable, VariadicTranslatable} from "tc-shared/ui/react-elements/i18n";
 import {network} from "tc-shared/ui/frames/chat";
+import {date_format} from "tc-shared/utils/DateUtils";
 
 const cssStyle = require("./Renderer.scss");
 export const StatusEvents = React.createContext<Registry<ConnectionStatusEvents>>(undefined);
@@ -162,16 +163,26 @@ const ComponentStatusRenderer = React.memo((props: { component: ConnectionCompon
                     break;
             }
             body = (
-                <div className={cssStyle.row + " " + cssStyle.doubleSize} key={"description"}>
+                <div className={cssStyle.row + " " + cssStyle.error} key={"description"}>
                     <div className={cssStyle.text}>{text}</div>
                 </div>
             );
             break;
 
         case "unhealthy":
+            let errorText;
+            if(status.retryTimestamp) {
+                let time = Math.ceil((status.retryTimestamp - Date.now()) / 1000);
+                let minutes = Math.floor(time / 60);
+                let seconds = time % 60;
+                errorText = <VariadicTranslatable key={"retry"} text={"Error occurred. Retry in {}."}>{(minutes > 0 ? minutes + "m" : "") + seconds + "s"}</VariadicTranslatable>;
+            } else {
+                errorText = <Translatable key={"no-retry"}>Error occurred. No retry.</Translatable>;
+            }
             body = (
-                <div className={cssStyle.row + " " + cssStyle.doubleSize} key={"error"}>
+                <div className={cssStyle.row + " " + cssStyle.error} key={"error"}>
-                    <div className={cssStyle.text}><Translatable>Some error occured</Translatable></div>
+                    <div className={cssStyle.text + " " + cssStyle.errorRow}>{errorText}</div>
+                    <div className={cssStyle.text + " " + cssStyle.error} title={status.reason}>{status.reason}</div>
                 </div>
             );
             break;

shared/js/ui/frames/log/Definitions.ts

@@ -66,7 +66,9 @@ export enum EventType {
 
     RECONNECT_SCHEDULED = "reconnect.scheduled",
     RECONNECT_EXECUTE = "reconnect.execute",
-    RECONNECT_CANCELED = "reconnect.canceled"
+    RECONNECT_CANCELED = "reconnect.canceled",
+
+    WEBRTC_FATAL_ERROR = "webrtc.fatal.error"
 }
 
 export type EventClient = {
@@ -249,6 +251,11 @@ export namespace event {
         sender: EventClient,
         message: string
     }
+
+    export type EventWebrtcFatalError = {
+        message: string,
+        retryTimeout: number | 0
+    }
 }
 
 export type LogMessage = {
@@ -301,7 +308,7 @@ export interface TypeInfo {
 
     "client.nickname.change.failed": event.EventClientNicknameChangeFailed,
     "client.nickname.changed": event.EventClientNicknameChanged,
-    "client.nickname.changed.own": event.EventClientNicknameChanged
+    "client.nickname.changed.own": event.EventClientNicknameChanged,
 
     "channel.create": event.EventChannelCreate;
     "channel.delete": event.EventChannelDelete;
@@ -312,10 +319,11 @@ export interface TypeInfo {
     "client.poke.received": event.EventClientPokeReceived,
     "client.poke.send": event.EventClientPokeSend,
 
-
     "private.message.received": event.EventPrivateMessageReceived,
     "private.message.send": event.EventPrivateMessageSend,
 
+    "webrtc.fatal.error": event.EventWebrtcFatalError
+
     "disconnected": any;
 }
 

shared/js/ui/frames/log/DispatcherLog.tsx

@@ -629,3 +629,28 @@ registerDispatcher(EventType.CLIENT_POKE_RECEIVED,(data, handlerId) => {
 
 registerDispatcher(EventType.PRIVATE_MESSAGE_RECEIVED, () => undefined);
 registerDispatcher(EventType.PRIVATE_MESSAGE_SEND, () => undefined);
+
+registerDispatcher(EventType.WEBRTC_FATAL_ERROR, (data) => {
+    if(data.retryTimeout) {
+        let time = Math.ceil(data.retryTimeout / 1000);
+        let minutes = Math.floor(time / 60);
+        let seconds = time % 60;
+
+        return (
+            <div className={cssStyleRenderer.errorMessage}>
+                <VariadicTranslatable text={"WebRTC connection closed due to a fatal error:\n{}\nRetry scheduled in {}."}>
+                    <>{data.message}</>
+                    <>{(minutes > 0 ? minutes + "m" : "") + seconds + "s"}</>
+                </VariadicTranslatable>
+            </div>
+        );
+    } else {
+        return (
+            <div className={cssStyleRenderer.errorMessage}>
+                <VariadicTranslatable text={"WebRTC connection closed due to a fatal error:\n{}\nNo retry scheduled."}>
+                    <>{data.message}</>
+                </VariadicTranslatable>
+            </div>
+        );
+    }
+});

shared/js/ui/frames/log/DispatcherNotifications.ts

@@ -480,6 +480,22 @@ registerDispatcher(EventType.PRIVATE_MESSAGE_RECEIVED, (data, handlerId) => {
     });
 });
 
+registerDispatcher(EventType.WEBRTC_FATAL_ERROR, (data, handlerId) => {
+   if(data.retryTimeout) {
+       let time = Math.ceil(data.retryTimeout / 1000);
+       let minutes = Math.floor(time / 60);
+       let seconds = time % 60;
+
+       spawnServerNotification(handlerId, {
+           body: tra("WebRTC connection closed due to a fatal error:\n{}\nRetry scheduled in {}.", data.message, (minutes > 0 ? minutes + "m" : "") + seconds + "s")
+       });
+   } else {
+       spawnServerNotification(handlerId, {
+           body: tra("WebRTC connection closed due to a fatal error:\n{}\nNo retry scheduled.", data.message)
+       });
+   }
+});
+
 /* snipped PRIVATE_MESSAGE_SEND */
 
 loader.register_task(Stage.LOADED, {

shared/js/ui/frames/log/Renderer.scss

@@ -47,6 +47,7 @@
 
     .timestamp {
         padding-right: 5px;
+        vertical-align: top;
     }
 
     .errorMessage {

shared/js/ui/react-elements/i18n/index.tsx

@@ -50,6 +50,7 @@ export class Translatable extends React.Component<{
     }
 }
 
+let renderBrElementIndex = 0;
 export type VariadicTranslatableChild = React.ReactElement | string;
 export const VariadicTranslatable = (props: { text: string, __cacheKey?: string, children?: VariadicTranslatableChild[] | VariadicTranslatableChild }) => {
     const args = Array.isArray(props.children) ? props.children : [props.children];
@@ -60,8 +61,15 @@ export const VariadicTranslatable = (props: { text: string, __cacheKey?: string,
     return (<>
         {
             parseMessageWithArguments(translated, args.length).map(e => {
-                if(typeof e === "string")
+                if(typeof e === "string") {
-                    return e;
+                    return e.split("\n").reduce((result, element) => {
+                        if(result.length > 0) {
+                            result.push(<br key={++this.renderBrElementIndex}/>);
+                        }
+                        result.push(element);
+                        return result;
+                    }, []);
+                }
 
                 let element = args[e];
                 if(argsUseCount[e]) {

web/app/rtc/Connection.ts

@@ -29,6 +29,40 @@ declare global {
     }
 }
 
+class RetryTimeCalculator {
+    private readonly minTime: number;
+    private readonly maxTime: number;
+    private readonly increment: number;
+
+    private retryCount: number;
+    private currentTime: number;
+
+    constructor(minTime: number, maxTime: number, increment: number) {
+        this.minTime = minTime;
+        this.maxTime = maxTime;
+        this.increment = increment;
+
+        this.reset();
+    }
+
+    calculateRetryTime() {
+        if(this.retryCount >= 5) {
+            /* no more retries */
+            return 0;
+        }
+        this.retryCount++;
+        const time = this.currentTime;
+        this.currentTime = Math.min(this.currentTime + this.increment, this.maxTime);
+        console.error(time + " - " + this.retryCount);
+        return time;
+    }
+
+    reset() {
+        this.currentTime = this.minTime;
+        this.retryCount = 0;
+    }
+}
+
 let dummyVideoTrack: MediaStreamTrack | undefined;
 let dummyAudioTrack: MediaStreamTrack | undefined;
 
@@ -86,7 +120,7 @@ class CommandHandler extends AbstractCommandHandler {
                 sdp = SdpCompressor.decompressSdp(sdp, 1);
             } catch (error) {
                 logError(LogCategory.WEBRTC, tr("Failed to decompress remote SDP: %o"), error);
-                this.handle["handleFatalError"](tr("Failed to decompress remote SDP"), 5000);
+                this.handle["handleFatalError"](tr("Failed to decompress remote SDP"), true);
                 return;
             }
             if(RTCConnection.kEnableSdpTrace) {
@@ -96,7 +130,7 @@ class CommandHandler extends AbstractCommandHandler {
                 sdp = this.sdpProcessor.processIncomingSdp(sdp, data.mode);
             } catch (error) {
                 logError(LogCategory.WEBRTC, tr("Failed to reprocess SDP %s: %o"), data.mode, error);
-                this.handle["handleFatalError"](tra("Failed to preprocess SDP {}", data.mode as string), 5000);
+                this.handle["handleFatalError"](tra("Failed to preprocess SDP {}", data.mode as string), true);
                 return;
             }
             if(RTCConnection.kEnableSdpTrace) {
@@ -108,7 +142,7 @@ class CommandHandler extends AbstractCommandHandler {
                     type: "answer"
                 }).catch(error => {
                     logError(LogCategory.WEBRTC, tr("Failed to set the remote description: %o"), error);
-                    this.handle["handleFatalError"](tr("Failed to set the remote description (answer)"), 5000);
+                    this.handle["handleFatalError"](tr("Failed to set the remote description (answer)"), true);
                 })
             } else if(data.mode === "offer") {
                 this.handle["peer"].setRemoteDescription({
@@ -137,7 +171,7 @@ class CommandHandler extends AbstractCommandHandler {
                     });
                 }).catch(error => {
                     logError(LogCategory.WEBRTC, tr("Failed to set the remote description and execute the renegotiation: %o"), error);
-                    this.handle["handleFatalError"](tr("Failed to set the remote description (offer/renegotiation)"), 5000);
+                    this.handle["handleFatalError"](tr("Failed to set the remote description (offer/renegotiation)"), true);
                 });
             } else {
                 logWarn(LogCategory.NETWORKING, tr("Received invalid mode for rtc session description (%s)."), data.mode);
@@ -366,7 +400,8 @@ export class RTCConnection {
 
     private connectionState: RTPConnectionState;
     private failedReason: string;
-
+    private retryCalculator: RetryTimeCalculator;
+    private retryTimestamp: number;
     private retryTimeout: number;
 
     private peer: RTCPeerConnection;
@@ -394,6 +429,7 @@ export class RTCConnection {
         this.connection = connection;
         this.sdpProcessor = new SdpProcessor();
         this.commandHandler = new CommandHandler(connection, this, this.sdpProcessor);
+        this.retryCalculator = new RetryTimeCalculator(5000, 30000, 10000);
 
         this.connection.command_handler_boss().register_handler(this.commandHandler);
         this.reset(true);
@@ -421,6 +457,10 @@ export class RTCConnection {
         return this.failedReason;
     }
 
+    getRetryTimestamp() : number | 0 {
+        return this.retryTimestamp;
+    }
+
     reset(updateConnectionState: boolean) {
         if(this.peer) {
             if(this.getConnection().connected()) {
@@ -459,6 +499,12 @@ export class RTCConnection {
 
         clearTimeout(this.retryTimeout);
         this.retryTimeout = 0;
+        this.retryTimestamp = 0;
+        /*
+         * We do not reset the retry timer here since we might get called when a fatal error occurs.
+         * Instead we're resetting it every time we've changed the server connection state.
+         */
+        /* this.retryCalculator.reset(); */
 
         if(updateConnectionState) {
             this.updateConnectionState(RTPConnectionState.DISCONNECTED);
@@ -522,18 +568,34 @@ export class RTCConnection {
         this.events.fire("notify_state_changed", { oldState: oldState, newState: newState });
     }
 
-    private handleFatalError(error: string, retryThreshold: number) {
+    private handleFatalError(error: string, allowRetry: boolean) {
         this.reset(false);
         this.failedReason = error;
         this.updateConnectionState(RTPConnectionState.FAILED);
 
-        /* FIXME: Generate a log message! */
+        const log = this.connection.client.log;
-        if(retryThreshold > 0) {
+        if(allowRetry) {
-            this.retryTimeout = setTimeout(() => {
+            const time = this.retryCalculator.calculateRetryTime();
-                console.error("XXXX Retry");
+            if(time > 0) {
-                this.doInitialSetup();
+                this.retryTimestamp = Date.now() + time;
-            }, 5000);
+                this.retryTimeout = setTimeout(() => {
-            /* TODO: Schedule a retry? */
+                    this.doInitialSetup();
+                }, time);
+
+                log.log("webrtc.fatal.error", {
+                    message: error,
+                    retryTimeout: time
+                });
+            } else {
+                allowRetry = false;
+            }
+        }
+
+        if(!allowRetry) {
+            log.log("webrtc.fatal.error", {
+                message: error,
+                retryTimeout: 0
+            });
         }
     }
 
@@ -555,7 +617,7 @@ export class RTCConnection {
 
     private doInitialSetup() {
         if(!('RTCPeerConnection' in window)) {
-            this.handleFatalError(tr("WebRTC has been disabled (RTCPeerConnection is not defined)"), 0);
+            this.handleFatalError(tr("WebRTC has been disabled (RTCPeerConnection is not defined)"), false);
             return;
         }
 
@@ -598,7 +660,7 @@ export class RTCConnection {
 
         this.updateConnectionState(RTPConnectionState.CONNECTING);
         this.doInitialSetup0().catch(error => {
-            this.handleFatalError(tr("initial setup failed"), 5000);
+            this.handleFatalError(tr("initial setup failed"), true);
             logError(LogCategory.WEBRTC, tr("Connection setup failed: %o"), error);
         });
     }
@@ -639,7 +701,7 @@ export class RTCConnection {
             logTrace(LogCategory.WEBRTC, tr("Patched initial local offer:\n%s"), offer.sdp);
         } catch (error) {
             logError(LogCategory.WEBRTC, tr("Failed to preprocess outgoing initial offer: %o"), error);
-            this.handleFatalError(tr("Failed to preprocess outgoing initial offer"), 10000);
+            this.handleFatalError(tr("Failed to preprocess outgoing initial offer"), true);
             return;
         }
 
@@ -668,6 +730,7 @@ export class RTCConnection {
     private handleConnectionStateChanged(event: ServerConnectionEvents["notify_connection_state_changed"]) {
         if(event.newState === ConnectionState.CONNECTED) {
             /* initialize rtc connection */
+            this.retryCalculator.reset();
             this.doInitialSetup();
         } else {
             this.reset(true);
@@ -680,7 +743,7 @@ export class RTCConnection {
                 logTrace(LogCategory.WEBRTC, tr("Skipping local fqdn ICE candidate %s"), candidate.toJSON().candidate);
                 return;
             }
-            this.localCandidateCount++;
+            //sthis.localCandidateCount++;
 
             const json = candidate.toJSON();
             logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate %s"), json.candidate);
@@ -692,8 +755,8 @@ export class RTCConnection {
             });
         } else {
             if(this.localCandidateCount === 0) {
-                logError(LogCategory.WEBRTC, tr("Received local ICE candidate finish, without having any candidates."));
+                logError(LogCategory.WEBRTC, tr("Received local ICE candidate finish, without having any candidates"));
-                this.handleFatalError(tr("Failed to gather any ICE candidates"), 0);
+                this.handleFatalError(tr("Failed to gather any local ICE candidates."), false);
                 return;
             } else {
                 logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate finish"));
@@ -729,17 +792,17 @@ export class RTCConnection {
         logTrace(LogCategory.WEBRTC, tr("Peer connection state changed to %s"), this.peer.connectionState);
         switch (this.peer.connectionState) {
             case "connecting":
-                //this.updateConnectionState(RTPConnectionState.CONNECTING);
+                this.updateConnectionState(RTPConnectionState.CONNECTING);
-                this.updateConnectionState(RTPConnectionState.CONNECTED);
                 break;
 
             case "connected":
+                this.retryCalculator.reset();
                 this.updateConnectionState(RTPConnectionState.CONNECTED);
                 break;
 
             case "failed":
                 if(this.connectionState !== RTPConnectionState.FAILED) {
-                    this.handleFatalError(tr("peer connection failed"), 5000);
+                    this.handleFatalError(tr("peer connection failed"), true);
                 }
                 break;
 

web/app/rtc/RemoteTrack.ts

@@ -157,6 +157,11 @@ export class RemoteRTPAudioTrack extends RemoteRTPTrack {
         */
 
         aplayer.on_ready(() => {
+            if(!this.mediaStream) {
+                /* we've already been destroyed */
+                return;
+            }
+
             const audioContext = aplayer.context();
             this.audioNode = audioContext.createMediaStreamSource(this.mediaStream);
             this.gainNode = audioContext.createGain();

web/app/rtc/video/Connection.ts

@@ -115,6 +115,14 @@ export class RtpVideoConnection implements VideoConnection {
         return this.connectionState;
     }
 
+    getRetryTimestamp(): number | 0 {
+        return this.rtcConnection.getRetryTimestamp();
+    }
+
+    getFailedMessage(): string {
+        return this.rtcConnection.getFailReason();
+    }
+
     getBroadcastingState(type: VideoBroadcastType): VideoBroadcastState {
         return this.broadcasts[type] ? this.broadcasts[type].state : VideoBroadcastState.Stopped;
     }

web/app/rtc/voice/Connection.ts

@@ -392,4 +392,8 @@ export class RtpVoiceConnection extends AbstractVoiceConnection {
         this.speakerMuted = newState;
         this.voiceClients.forEach(client => client.setGloballyMuted(this.speakerMuted));
     }
+
+    getRetryTimestamp(): number | 0 {
+        return this.rtcConnection.getRetryTimestamp();
+    }
 }