Fixed the video connection for firefox and some minor bugfixes

canary
WolverinDEV 2020-11-15 23:28:00 +01:00
parent 1ca64c82e2
commit e2fed64b39
10 changed files with 149 additions and 41 deletions

View File

@ -1,4 +1,7 @@
# Changelog: # Changelog:
** 14.11.20**
- Fixed bug where the microphone has been requested when muting it.
* **07.11.20** * **07.11.20**
- Added video broadcasting to the web client - Added video broadcasting to the web client
- Added various new user interfaces related to video broadcasting - Added various new user interfaces related to video broadcasting

View File

@ -826,11 +826,7 @@ export class ConnectionHandler {
return; return;
} }
if(this.connection_state === ConnectionState.CONNECTED) { /* our voice status will be updated automatically due to the notify_recorder_changed event which should be fired when the acquired recorder changed */
await this.startVoiceRecorder(true);
} else {
this.setInputHardwareState(InputHardwareState.VALID);
}
} }
async startVoiceRecorder(notifyError: boolean) : Promise<{ state: "success" | "no-input" } | { state: "error", message: string }> { async startVoiceRecorder(notifyError: boolean) : Promise<{ state: "success" | "no-input" } | { state: "error", message: string }> {

View File

@ -588,7 +588,7 @@ export class ChannelTree {
deleteClient(client: ClientEntry, reason: { reason: ViewReasonId, message?: string, serverLeave: boolean }) { deleteClient(client: ClientEntry, reason: { reason: ViewReasonId, message?: string, serverLeave: boolean }) {
const oldChannel = client.currentChannel(); const oldChannel = client.currentChannel();
oldChannel?.unregisterClient(client); oldChannel?.unregisterClient(client);
this.clients.remove(client); this.unregisterClient(client);
if(oldChannel) { if(oldChannel) {
this.events.fire("notify_client_leave_view", { client: client, message: reason.message, reason: reason.reason, isServerLeave: reason.serverLeave, sourceChannel: oldChannel }); this.events.fire("notify_client_leave_view", { client: client, message: reason.message, reason: reason.reason, isServerLeave: reason.serverLeave, sourceChannel: oldChannel });
@ -597,30 +597,23 @@ export class ChannelTree {
logWarn(LogCategory.CHANNEL, tr("Deleting client %s from channel tree which hasn't a channel."), client.clientId()); logWarn(LogCategory.CHANNEL, tr("Deleting client %s from channel tree which hasn't a channel."), client.clientId());
} }
const voiceConnection = this.client.serverConnection.getVoiceConnection();
if(client.getVoiceClient()) {
voiceConnection.unregisterVoiceClient(client.getVoiceClient());
client.setVoiceClient(undefined);
}
const videoConnection = this.client.serverConnection.getVideoConnection();
if(client.getVideoClient()) {
videoConnection.unregisterVideoClient(client.getVideoClient());
client.setVideoClient(undefined);
}
client.destroy(); client.destroy();
} }
registerClient(client: ClientEntry) { registerClient(client: ClientEntry) {
this.clients.push(client); this.clients.push(client);
if(client instanceof LocalClientEntry) { const isLocalClient = client instanceof LocalClientEntry;
if(isLocalClient) {
if(client.channelTree !== this) { if(client.channelTree !== this) {
throw tr("client channel tree missmatch"); throw tr("client channel tree missmatch");
} }
} else { } else {
client.channelTree = this; client.channelTree = this;
}
/* for debug purposes, the server might send back the own audio/video stream */
if(!isLocalClient || __build.mode === "debug") {
const voiceConnection = this.client.serverConnection.getVoiceConnection(); const voiceConnection = this.client.serverConnection.getVoiceConnection();
try { try {
client.setVoiceClient(voiceConnection.registerVoiceClient(client.clientId())); client.setVoiceClient(voiceConnection.registerVoiceClient(client.clientId()));
@ -641,6 +634,18 @@ export class ChannelTree {
if(!this.clients.remove(client)) { if(!this.clients.remove(client)) {
return; return;
} }
const voiceConnection = this.client.serverConnection.getVoiceConnection();
if(client.getVoiceClient()) {
voiceConnection.unregisterVoiceClient(client.getVoiceClient());
client.setVoiceClient(undefined);
}
const videoConnection = this.client.serverConnection.getVideoConnection();
if(client.getVideoClient()) {
videoConnection.unregisterVideoClient(client.getVideoClient());
client.setVideoClient(undefined);
}
} }
insertClient(client: ClientEntry, channel: ChannelEntry, reason: { reason: ViewReasonId, isServerJoin: boolean }) : ClientEntry { insertClient(client: ClientEntry, channel: ChannelEntry, reason: { reason: ViewReasonId, isServerJoin: boolean }) : ClientEntry {

View File

@ -351,7 +351,12 @@ class ChannelVideoController {
if(localClient.currentChannel()) { if(localClient.currentChannel()) {
this.currentChannelId = localClient.currentChannel().channelId; this.currentChannelId = localClient.currentChannel().channelId;
localClient.currentChannel().channelClientsOrdered().forEach(client => { localClient.currentChannel().channelClientsOrdered().forEach(client => {
if(client instanceof LocalClientEntry || ChannelVideoController.shouldIgnoreClient(client)) { /* in some instances the server might return our own stream for debug purposes */
if(client instanceof LocalClientEntry && __build.mode !== "debug") {
return;
}
if(ChannelVideoController.shouldIgnoreClient(client)) {
return; return;
} }

View File

@ -77,16 +77,19 @@ const VideoStreamReplay = React.memo((props: { stream: MediaStream | undefined,
const refVideo = useRef<HTMLVideoElement>(); const refVideo = useRef<HTMLVideoElement>();
useEffect(() => { useEffect(() => {
const video = refVideo.current;
if(props.stream) { if(props.stream) {
refVideo.current.style.opacity = "1"; video.style.opacity = "1";
refVideo.current.srcObject = props.stream; video.srcObject = props.stream;
video.autoplay = true;
video.play().then(undefined);
} else { } else {
refVideo.current.style.opacity = "0"; video.style.opacity = "0";
} }
}, [ props.stream ]); }, [ props.stream ]);
return ( return (
<video ref={refVideo} autoPlay={true} className={cssStyle.video + " " + props.className} /> <video ref={refVideo} className={cssStyle.video + " " + props.className} />
) )
}); });

View File

@ -14,9 +14,53 @@ import {
TrackClientInfo TrackClientInfo
} from "tc-backend/web/rtc/RemoteTrack"; } from "tc-backend/web/rtc/RemoteTrack";
import {SdpCompressor, SdpProcessor} from "tc-backend/web/rtc/SdpUtils"; import {SdpCompressor, SdpProcessor} from "tc-backend/web/rtc/SdpUtils";
import {context} from "tc-backend/web/audio/player";
const kSdpCompressionMode = 1; const kSdpCompressionMode = 1;
declare global {
interface RTCIceCandidate {
/* Firefox has this */
address: string | undefined;
}
interface HTMLCanvasElement {
captureStream(framed: number) : MediaStream;
}
}
let dummyVideoTrack: MediaStreamTrack | undefined;
let dummyAudioTrack: MediaStreamTrack | undefined;
/*
* For Firefox as soon we stop a sender we're never able to get the sender starting again...
* (This only applies after the initial negotiation. Before values of null are allowed)
* So we've to keep it alive with a dummy track.
*/
function getIdleTrack(kind: "video" | "audio") : MediaStreamTrack | null {
if(window.detectedBrowser?.name === "firefox" || true) {
if(kind === "video") {
if(!dummyVideoTrack) {
const canvas = document.createElement("canvas");
canvas.getContext("2d");
const stream = canvas.captureStream(1);
dummyVideoTrack = stream.getVideoTracks()[0];
}
return dummyVideoTrack;
} else if(kind === "audio") {
if(!dummyAudioTrack) {
const dest = context().createMediaStreamDestination();
dummyAudioTrack = dest.stream.getAudioTracks()[0];
}
return dummyAudioTrack;
}
}
return null;
}
class CommandHandler extends AbstractCommandHandler { class CommandHandler extends AbstractCommandHandler {
private readonly handle: RTCConnection; private readonly handle: RTCConnection;
private readonly sdpProcessor: SdpProcessor; private readonly sdpProcessor: SdpProcessor;
@ -71,9 +115,20 @@ class CommandHandler extends AbstractCommandHandler {
sdp: sdp, sdp: sdp,
type: "offer" type: "offer"
}).then(() => this.handle["peer"].createAnswer()) }).then(() => this.handle["peer"].createAnswer())
.then(async answer => {
await this.handle["peer"].setLocalDescription(answer);
return answer;
})
.then(answer => { .then(answer => {
if(RTCConnection.kEnableSdpTrace) {
logTrace(LogCategory.WEBRTC, tr("Sending answer to remote %s:\n%s"), data.mode, answer.sdp);
}
answer.sdp = this.sdpProcessor.processOutgoingSdp(answer.sdp, "answer"); answer.sdp = this.sdpProcessor.processOutgoingSdp(answer.sdp, "answer");
answer.sdp = SdpCompressor.compressSdp(answer.sdp, kSdpCompressionMode); answer.sdp = SdpCompressor.compressSdp(answer.sdp, kSdpCompressionMode);
if(RTCConnection.kEnableSdpTrace) {
logTrace(LogCategory.WEBRTC, tr("Patched answer to remote %s:\n%s"), data.mode, answer.sdp);
}
return this.connection.send_command("rtcsessiondescribe", { return this.connection.send_command("rtcsessiondescribe", {
mode: "answer", mode: "answer",
@ -295,7 +350,7 @@ export interface RTCConnectionEvents {
} }
export class RTCConnection { export class RTCConnection {
public static readonly kEnableSdpTrace = false; public static readonly kEnableSdpTrace = true;
private readonly events: Registry<RTCConnectionEvents>; private readonly events: Registry<RTCConnectionEvents>;
private readonly connection: ServerConnection; private readonly connection: ServerConnection;
private readonly commandHandler: CommandHandler; private readonly commandHandler: CommandHandler;
@ -304,6 +359,8 @@ export class RTCConnection {
private connectionState: RTPConnectionState; private connectionState: RTPConnectionState;
private failedReason: string; private failedReason: string;
private retryTimeout: number;
private peer: RTCPeerConnection; private peer: RTCPeerConnection;
private localCandidateCount: number; private localCandidateCount: number;
@ -392,6 +449,9 @@ export class RTCConnection {
this.temporaryStreams = {}; this.temporaryStreams = {};
this.localCandidateCount = 0; this.localCandidateCount = 0;
clearTimeout(this.retryTimeout);
this.retryTimeout = 0;
if(updateConnectionState) { if(updateConnectionState) {
this.updateConnectionState(RTPConnectionState.DISCONNECTED); this.updateConnectionState(RTPConnectionState.DISCONNECTED);
} }
@ -461,7 +521,7 @@ export class RTCConnection {
/* FIXME: Generate a log message! */ /* FIXME: Generate a log message! */
if(retryThreshold > 0) { if(retryThreshold > 0) {
setTimeout(() => { this.retryTimeout = setTimeout(() => {
console.error("XXXX Retry"); console.error("XXXX Retry");
this.doInitialSetup(); this.doInitialSetup();
}, 5000); }, 5000);
@ -497,7 +557,7 @@ export class RTCConnection {
iceServers: [{ urls: ["stun:stun.l.google.com:19302", "stun:stun1.l.google.com:19302"] }] iceServers: [{ urls: ["stun:stun.l.google.com:19302", "stun:stun1.l.google.com:19302"] }]
}); });
this.currentTransceiver["audio"] = this.peer.addTransceiver("audio", { sendEncodings: [{ }] }); this.currentTransceiver["audio"] = this.peer.addTransceiver("audio");
this.enableDtx(this.currentTransceiver["audio"].sender); this.enableDtx(this.currentTransceiver["audio"].sender);
this.currentTransceiver["audio-whisper"] = this.peer.addTransceiver("audio"); this.currentTransceiver["audio-whisper"] = this.peer.addTransceiver("audio");
@ -537,7 +597,19 @@ export class RTCConnection {
private async updateTracks() { private async updateTracks() {
for(const type of kRtcSourceTrackTypes) { for(const type of kRtcSourceTrackTypes) {
await this.currentTransceiver[type]?.sender.replaceTrack(this.currentTracks[type]); let fallback;
switch (type) {
case "audio":
case "audio-whisper":
fallback = getIdleTrack("audio");
break;
case "video":
case "video-screen":
fallback = getIdleTrack("video");
break;
}
await this.currentTransceiver[type]?.sender.replaceTrack(this.currentTracks[type] || fallback);
} }
} }
@ -546,6 +618,7 @@ export class RTCConnection {
const peer = this.peer; const peer = this.peer;
await this.updateTracks(); await this.updateTracks();
const offer = await peer.createOffer({ iceRestart: false, offerToReceiveAudio: true, offerToReceiveVideo: true }); const offer = await peer.createOffer({ iceRestart: false, offerToReceiveAudio: true, offerToReceiveVideo: true });
if(offer.type !== "offer") { throw tr("created ofer isn't of type offer"); } if(offer.type !== "offer") { throw tr("created ofer isn't of type offer"); }
if(this.peer !== peer) { return; } if(this.peer !== peer) { return; }
@ -595,10 +668,14 @@ export class RTCConnection {
private handleIceCandidate(candidate: RTCIceCandidate | undefined) { private handleIceCandidate(candidate: RTCIceCandidate | undefined) {
if(candidate) { if(candidate) {
if(candidate.address?.endsWith(".local")) {
logTrace(LogCategory.WEBRTC, tr("Skipping local fqdn ICE candidate %s"), candidate.toJSON().candidate);
return;
}
this.localCandidateCount++; this.localCandidateCount++;
const json = candidate.toJSON(); const json = candidate.toJSON();
logTrace(LogCategory.WEBRTC, tr("Received ICE candidate %s"), json.candidate); logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate %s"), json.candidate);
this.connection.send_command("rtcicecandidate", { this.connection.send_command("rtcicecandidate", {
media_line: json.sdpMLineIndex, media_line: json.sdpMLineIndex,
candidate: json.candidate candidate: json.candidate
@ -611,7 +688,7 @@ export class RTCConnection {
this.handleFatalError(tr("Failed to gather any ICE candidates"), 0); this.handleFatalError(tr("Failed to gather any ICE candidates"), 0);
return; return;
} else { } else {
logTrace(LogCategory.WEBRTC, tr("Received ICE candidate finish")); logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate finish"));
} }
this.connection.send_command("rtcicecandidate", { }).catch(error => { this.connection.send_command("rtcicecandidate", { }).catch(error => {
logWarn(LogCategory.WEBRTC, tr("Failed to transmit local ICE candidate finish to server: %o"), error); logWarn(LogCategory.WEBRTC, tr("Failed to transmit local ICE candidate finish to server: %o"), error);
@ -644,7 +721,8 @@ export class RTCConnection {
logTrace(LogCategory.WEBRTC, tr("Peer connection state changed to %s"), this.peer.connectionState); logTrace(LogCategory.WEBRTC, tr("Peer connection state changed to %s"), this.peer.connectionState);
switch (this.peer.connectionState) { switch (this.peer.connectionState) {
case "connecting": case "connecting":
this.updateConnectionState(RTPConnectionState.CONNECTING); //this.updateConnectionState(RTPConnectionState.CONNECTING);
this.updateConnectionState(RTPConnectionState.CONNECTED);
break; break;
case "connected": case "connected":

View File

@ -101,6 +101,12 @@ export class RemoteRTPVideoTrack extends RemoteRTPTrack {
this.mediaStream = new MediaStream(); this.mediaStream = new MediaStream();
this.mediaStream.addTrack(transceiver.receiver.track); this.mediaStream.addTrack(transceiver.receiver.track);
const track = transceiver.receiver.track;
track.onended = () => console.error("TRACK %d ended", ssrc);
track.onmute = () => console.error("TRACK %d muted", ssrc);
track.onunmute = () => console.error("TRACK %d unmuted", ssrc);
track.onisolationchange = () => console.error("TRACK %d onisolationchange", ssrc);
} }
getMediaStream() : MediaStream { getMediaStream() : MediaStream {

View File

@ -1,6 +1,5 @@
import * as sdpTransform from "sdp-transform"; import * as sdpTransform from "sdp-transform";
import {MediaDescription} from "sdp-transform"; import {MediaDescription} from "sdp-transform";
import {guid} from "tc-shared/crypto/uid";
interface SdpCodec { interface SdpCodec {
payload: number; payload: number;
@ -14,7 +13,7 @@ interface SdpCodec {
/* These MUST be the payloads used by the remote as well */ /* These MUST be the payloads used by the remote as well */
const OPUS_VOICE_PAYLOAD_TYPE = 111; const OPUS_VOICE_PAYLOAD_TYPE = 111;
const OPUS_MUSIC_PAYLOAD_TYPE = 112; const OPUS_MUSIC_PAYLOAD_TYPE = 112;
const VP8_PAYLOAD_TYPE = 96; const VP8_PAYLOAD_TYPE = 122; /* Using 122 for testing purposes */ //96;
type SdpMedia = { type SdpMedia = {
type: string; type: string;
@ -31,7 +30,7 @@ export class SdpProcessor {
codec: "opus", codec: "opus",
rate: 48000, rate: 48000,
encoding: 2, encoding: 2,
fmtp: { minptime: 1, maxptime: 20, useinbandfec: 1, stereo: 1 }, fmtp: { minptime: 1, maxptime: 20, useinbandfec: 1, stereo: 0 },
rtcpFb: [ "transport-cc" ] rtcpFb: [ "transport-cc" ]
}, },
{ {

View File

@ -13,6 +13,7 @@ import {LogCategory, logDebug, logError, logWarn} from "tc-shared/log";
import {Settings, settings} from "tc-shared/settings"; import {Settings, settings} from "tc-shared/settings";
import {RtpVideoClient} from "tc-backend/web/rtc/video/VideoClient"; import {RtpVideoClient} from "tc-backend/web/rtc/video/VideoClient";
import {tr} from "tc-shared/i18n/localize"; import {tr} from "tc-shared/i18n/localize";
import {ConnectionState} from "tc-shared/ConnectionHandler";
type VideoBroadcast = { type VideoBroadcast = {
readonly source: VideoSource; readonly source: VideoSource;
@ -26,6 +27,7 @@ export class RtpVideoConnection implements VideoConnection {
private readonly events: Registry<VideoConnectionEvent>; private readonly events: Registry<VideoConnectionEvent>;
private readonly listenerClientMoved; private readonly listenerClientMoved;
private readonly listenerRtcStateChanged; private readonly listenerRtcStateChanged;
private readonly listenerConnectionStateChanged;
private connectionState: VideoConnectionStatus; private connectionState: VideoConnectionStatus;
private broadcasts: {[T in VideoBroadcastType]: VideoBroadcast} = { private broadcasts: {[T in VideoBroadcastType]: VideoBroadcast} = {
@ -54,6 +56,12 @@ export class RtpVideoConnection implements VideoConnection {
} }
} }
}); });
this.listenerConnectionStateChanged = this.rtcConnection.getConnection().events.on("notify_connection_state_changed", event => {
if(event.newState !== ConnectionState.CONNECTED) {
this.stopBroadcasting("camera");
this.stopBroadcasting("screen");
}
});
this.listenerRtcStateChanged = this.rtcConnection.getEvents().on("notify_state_changed", event => this.handleRtcConnectionStateChanged(event)); this.listenerRtcStateChanged = this.rtcConnection.getEvents().on("notify_state_changed", event => this.handleRtcConnectionStateChanged(event));
@ -95,6 +103,7 @@ export class RtpVideoConnection implements VideoConnection {
destroy() { destroy() {
this.listenerClientMoved(); this.listenerClientMoved();
this.listenerRtcStateChanged(); this.listenerRtcStateChanged();
this.listenerConnectionStateChanged();
} }
getEvents(): Registry<VideoConnectionEvent> { getEvents(): Registry<VideoConnectionEvent> {
@ -163,20 +172,22 @@ export class RtpVideoConnection implements VideoConnection {
} }
stopBroadcasting(type: VideoBroadcastType, skipRtcStop?: boolean) { stopBroadcasting(type: VideoBroadcastType, skipRtcStop?: boolean) {
const broadcast = this.broadcasts[type];
if(!broadcast) {
return;
}
if(!skipRtcStop) { if(!skipRtcStop) {
this.rtcConnection.stopTrackBroadcast(type === "camera" ? "video" : "video-screen"); this.rtcConnection.stopTrackBroadcast(type === "camera" ? "video" : "video-screen");
} }
this.rtcConnection.setTrackSource(type === "camera" ? "video" : "video-screen", null).then(undefined); this.rtcConnection.setTrackSource(type === "camera" ? "video" : "video-screen", null).then(undefined);
if(this.broadcasts[type]) { const oldState = this.broadcasts[type].state;
const broadcast = this.broadcasts[type]; this.broadcasts[type].active = false;
const oldState = this.broadcasts[type].state; this.broadcasts[type] = undefined;
this.broadcasts[type].active = false; broadcast.source.deref();
this.broadcasts[type] = undefined;
broadcast.source.deref();
this.events.fire("notify_local_broadcast_state_changed", { oldState: oldState, newState: VideoBroadcastState.Stopped, broadcastType: type }); this.events.fire("notify_local_broadcast_state_changed", { oldState: oldState, newState: VideoBroadcastState.Stopped, broadcastType: type });
}
} }
registerVideoClient(clientId: number) { registerVideoClient(clientId: number) {

View File

@ -49,6 +49,8 @@ export class RtpVoiceConnection extends AbstractVoiceConnection {
this.rtcConnection.getEvents().on("notify_state_changed", this.rtcConnection.getEvents().on("notify_state_changed",
this.listenerRtcStateChanged = event => this.handleRtcConnectionStateChanged(event)); this.listenerRtcStateChanged = event => this.handleRtcConnectionStateChanged(event));
/* FIXME: Listener for audio! */
this.listenerClientMoved = this.rtcConnection.getConnection().command_handler_boss().register_explicit_handler("notifyclientmoved", event => { this.listenerClientMoved = this.rtcConnection.getConnection().command_handler_boss().register_explicit_handler("notifyclientmoved", event => {
const localClientId = this.rtcConnection.getConnection().client.getClientId(); const localClientId = this.rtcConnection.getConnection().client.getClientId();
for(const data of event.arguments) { for(const data of event.arguments) {