TeaSpeak
/
TeaWeb
1
0
Fork 0
Code Issues 1 Pull Requests Activity

Fixed the video connection for firefox and some minor bugfixes

canary
WolverinDEV 2020-11-15 23:28:00 +01:00
parent 1ca64c82e2
commit e2fed64b39
10 changed files with 149 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ChangeLog.md
View File

@ -1,4 +1,7 @@
# Changelog:
** 14.11.20**
- Fixed bug where the microphone has been requested when muting it.
* **07.11.20**
- Added video broadcasting to the web client
- Added various new user interfaces related to video broadcasting

6
shared/js/ConnectionHandler.ts
View File

@ -826,11 +826,7 @@ export class ConnectionHandler {
return;
}
if(this.connection_state === ConnectionState.CONNECTED) {
await this.startVoiceRecorder(true);
} else {
this.setInputHardwareState(InputHardwareState.VALID);
}
/* our voice status will be updated automatically due to the notify_recorder_changed event which should be fired when the acquired recorder changed */
}
async startVoiceRecorder(notifyError: boolean) : Promise<{ state: "success" | "no-input" } | { state: "error", message: string }> {

31
shared/js/tree/ChannelTree.tsx
View File

@ -588,7 +588,7 @@ export class ChannelTree {
deleteClient(client: ClientEntry, reason: { reason: ViewReasonId, message?: string, serverLeave: boolean }) {
const oldChannel = client.currentChannel();
oldChannel?.unregisterClient(client);
this.clients.remove(client);
this.unregisterClient(client);
if(oldChannel) {
this.events.fire("notify_client_leave_view", { client: client, message: reason.message, reason: reason.reason, isServerLeave: reason.serverLeave, sourceChannel: oldChannel });
@ -597,30 +597,23 @@ export class ChannelTree {
logWarn(LogCategory.CHANNEL, tr("Deleting client %s from channel tree which hasn't a channel."), client.clientId());
}
const voiceConnection = this.client.serverConnection.getVoiceConnection();
if(client.getVoiceClient()) {
voiceConnection.unregisterVoiceClient(client.getVoiceClient());
client.setVoiceClient(undefined);
}
const videoConnection = this.client.serverConnection.getVideoConnection();
if(client.getVideoClient()) {
videoConnection.unregisterVideoClient(client.getVideoClient());
client.setVideoClient(undefined);
}
client.destroy();
}
registerClient(client: ClientEntry) {
this.clients.push(client);
if(client instanceof LocalClientEntry) {
const isLocalClient = client instanceof LocalClientEntry;
if(isLocalClient) {
if(client.channelTree !== this) {
throw tr("client channel tree missmatch");
}
} else {
client.channelTree = this;
}
/* for debug purposes, the server might send back the own audio/video stream */
if(!isLocalClient || __build.mode === "debug") {
const voiceConnection = this.client.serverConnection.getVoiceConnection();
try {
client.setVoiceClient(voiceConnection.registerVoiceClient(client.clientId()));
@ -641,6 +634,18 @@ export class ChannelTree {
if(!this.clients.remove(client)) {
return;
}
const voiceConnection = this.client.serverConnection.getVoiceConnection();
if(client.getVoiceClient()) {
voiceConnection.unregisterVoiceClient(client.getVoiceClient());
client.setVoiceClient(undefined);
}
const videoConnection = this.client.serverConnection.getVideoConnection();
if(client.getVideoClient()) {
videoConnection.unregisterVideoClient(client.getVideoClient());
client.setVideoClient(undefined);
}
}
insertClient(client: ClientEntry, channel: ChannelEntry, reason: { reason: ViewReasonId, isServerJoin: boolean }) : ClientEntry {

7
shared/js/ui/frames/video/Controller.ts
View File

@ -351,7 +351,12 @@ class ChannelVideoController {
if(localClient.currentChannel()) {
this.currentChannelId = localClient.currentChannel().channelId;
localClient.currentChannel().channelClientsOrdered().forEach(client => {
if(client instanceof LocalClientEntry || ChannelVideoController.shouldIgnoreClient(client)) {
/* in some instances the server might return our own stream for debug purposes */
if(client instanceof LocalClientEntry && __build.mode !== "debug") {
return;
}
if(ChannelVideoController.shouldIgnoreClient(client)) {
return;
}

11
shared/js/ui/frames/video/Renderer.tsx
View File

@ -77,16 +77,19 @@ const VideoStreamReplay = React.memo((props: { stream: MediaStream | undefined,
const refVideo = useRef<HTMLVideoElement>();
useEffect(() => {
const video = refVideo.current;
if(props.stream) {
refVideo.current.style.opacity = "1";
refVideo.current.srcObject = props.stream;
video.style.opacity = "1";
video.srcObject = props.stream;
video.autoplay = true;
video.play().then(undefined);
} else {
refVideo.current.style.opacity = "0";
video.style.opacity = "0";
}
}, [ props.stream ]);
return (
<video ref={refVideo} autoPlay={true} className={cssStyle.video + " " + props.className} />
<video ref={refVideo} className={cssStyle.video + " " + props.className} />
)
});

92
web/app/rtc/Connection.ts
View File

@ -14,9 +14,53 @@ import {
TrackClientInfo
} from "tc-backend/web/rtc/RemoteTrack";
import {SdpCompressor, SdpProcessor} from "tc-backend/web/rtc/SdpUtils";
import {context} from "tc-backend/web/audio/player";
const kSdpCompressionMode = 1;
declare global {
interface RTCIceCandidate {
/* Firefox has this */
address: string | undefined;
}
interface HTMLCanvasElement {
captureStream(framed: number) : MediaStream;
}
}
let dummyVideoTrack: MediaStreamTrack | undefined;
let dummyAudioTrack: MediaStreamTrack | undefined;
/*
* For Firefox as soon we stop a sender we're never able to get the sender starting again...
* (This only applies after the initial negotiation. Before values of null are allowed)
* So we've to keep it alive with a dummy track.
*/
function getIdleTrack(kind: "video" | "audio") : MediaStreamTrack | null {
if(window.detectedBrowser?.name === "firefox" || true) {
if(kind === "video") {
if(!dummyVideoTrack) {
const canvas = document.createElement("canvas");
canvas.getContext("2d");
const stream = canvas.captureStream(1);
dummyVideoTrack = stream.getVideoTracks()[0];
}
return dummyVideoTrack;
} else if(kind === "audio") {
if(!dummyAudioTrack) {
const dest = context().createMediaStreamDestination();
dummyAudioTrack = dest.stream.getAudioTracks()[0];
}
return dummyAudioTrack;
}
}
return null;
}
class CommandHandler extends AbstractCommandHandler {
private readonly handle: RTCConnection;
private readonly sdpProcessor: SdpProcessor;
@ -71,9 +115,20 @@ class CommandHandler extends AbstractCommandHandler {
sdp: sdp,
type: "offer"
}).then(() => this.handle["peer"].createAnswer())
.then(async answer => {
await this.handle["peer"].setLocalDescription(answer);
return answer;
})
.then(answer => {
if(RTCConnection.kEnableSdpTrace) {
logTrace(LogCategory.WEBRTC, tr("Sending answer to remote %s:\n%s"), data.mode, answer.sdp);
}
answer.sdp = this.sdpProcessor.processOutgoingSdp(answer.sdp, "answer");
answer.sdp = SdpCompressor.compressSdp(answer.sdp, kSdpCompressionMode);
if(RTCConnection.kEnableSdpTrace) {
logTrace(LogCategory.WEBRTC, tr("Patched answer to remote %s:\n%s"), data.mode, answer.sdp);
}
return this.connection.send_command("rtcsessiondescribe", {
mode: "answer",
@ -295,7 +350,7 @@ export interface RTCConnectionEvents {
}
export class RTCConnection {
public static readonly kEnableSdpTrace = false;
public static readonly kEnableSdpTrace = true;
private readonly events: Registry<RTCConnectionEvents>;
private readonly connection: ServerConnection;
private readonly commandHandler: CommandHandler;
@ -304,6 +359,8 @@ export class RTCConnection {
private connectionState: RTPConnectionState;
private failedReason: string;
private retryTimeout: number;
private peer: RTCPeerConnection;
private localCandidateCount: number;
@ -392,6 +449,9 @@ export class RTCConnection {
this.temporaryStreams = {};
this.localCandidateCount = 0;
clearTimeout(this.retryTimeout);
this.retryTimeout = 0;
if(updateConnectionState) {
this.updateConnectionState(RTPConnectionState.DISCONNECTED);
}
@ -461,7 +521,7 @@ export class RTCConnection {
/* FIXME: Generate a log message! */
if(retryThreshold > 0) {
setTimeout(() => {
this.retryTimeout = setTimeout(() => {
console.error("XXXX Retry");
this.doInitialSetup();
}, 5000);
@ -497,7 +557,7 @@ export class RTCConnection {
iceServers: [{ urls: ["stun:stun.l.google.com:19302", "stun:stun1.l.google.com:19302"] }]
});
this.currentTransceiver["audio"] = this.peer.addTransceiver("audio", { sendEncodings: [{ }] });
this.currentTransceiver["audio"] = this.peer.addTransceiver("audio");
this.enableDtx(this.currentTransceiver["audio"].sender);
this.currentTransceiver["audio-whisper"] = this.peer.addTransceiver("audio");
@ -537,7 +597,19 @@ export class RTCConnection {
private async updateTracks() {
for(const type of kRtcSourceTrackTypes) {
await this.currentTransceiver[type]?.sender.replaceTrack(this.currentTracks[type]);
let fallback;
switch (type) {
case "audio":
case "audio-whisper":
fallback = getIdleTrack("audio");
break;
case "video":
case "video-screen":
fallback = getIdleTrack("video");
break;
}
await this.currentTransceiver[type]?.sender.replaceTrack(this.currentTracks[type] || fallback);
}
}
@ -546,6 +618,7 @@ export class RTCConnection {
const peer = this.peer;
await this.updateTracks();
const offer = await peer.createOffer({ iceRestart: false, offerToReceiveAudio: true, offerToReceiveVideo: true });
if(offer.type !== "offer") { throw tr("created ofer isn't of type offer"); }
if(this.peer !== peer) { return; }
@ -595,10 +668,14 @@ export class RTCConnection {
private handleIceCandidate(candidate: RTCIceCandidate | undefined) {
if(candidate) {
if(candidate.address?.endsWith(".local")) {
logTrace(LogCategory.WEBRTC, tr("Skipping local fqdn ICE candidate %s"), candidate.toJSON().candidate);
return;
}
this.localCandidateCount++;
const json = candidate.toJSON();
logTrace(LogCategory.WEBRTC, tr("Received ICE candidate %s"), json.candidate);
logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate %s"), json.candidate);
this.connection.send_command("rtcicecandidate", {
media_line: json.sdpMLineIndex,
candidate: json.candidate
@ -611,7 +688,7 @@ export class RTCConnection {
this.handleFatalError(tr("Failed to gather any ICE candidates"), 0);
return;
} else {
logTrace(LogCategory.WEBRTC, tr("Received ICE candidate finish"));
logTrace(LogCategory.WEBRTC, tr("Received local ICE candidate finish"));
}
this.connection.send_command("rtcicecandidate", { }).catch(error => {
logWarn(LogCategory.WEBRTC, tr("Failed to transmit local ICE candidate finish to server: %o"), error);
@ -644,7 +721,8 @@ export class RTCConnection {
logTrace(LogCategory.WEBRTC, tr("Peer connection state changed to %s"), this.peer.connectionState);
switch (this.peer.connectionState) {
case "connecting":
this.updateConnectionState(RTPConnectionState.CONNECTING);
//this.updateConnectionState(RTPConnectionState.CONNECTING);
this.updateConnectionState(RTPConnectionState.CONNECTED);
break;
case "connected":

6
web/app/rtc/RemoteTrack.ts
View File

@ -101,6 +101,12 @@ export class RemoteRTPVideoTrack extends RemoteRTPTrack {
this.mediaStream = new MediaStream();
this.mediaStream.addTrack(transceiver.receiver.track);
const track = transceiver.receiver.track;
track.onended = () => console.error("TRACK %d ended", ssrc);
track.onmute = () => console.error("TRACK %d muted", ssrc);
track.onunmute = () => console.error("TRACK %d unmuted", ssrc);
track.onisolationchange = () => console.error("TRACK %d onisolationchange", ssrc);
}
getMediaStream() : MediaStream {

5
web/app/rtc/SdpUtils.ts
View File

@ -1,6 +1,5 @@
import * as sdpTransform from "sdp-transform";
import {MediaDescription} from "sdp-transform";
import {guid} from "tc-shared/crypto/uid";
interface SdpCodec {
payload: number;
@ -14,7 +13,7 @@ interface SdpCodec {
/* These MUST be the payloads used by the remote as well */
const OPUS_VOICE_PAYLOAD_TYPE = 111;
const OPUS_MUSIC_PAYLOAD_TYPE = 112;
const VP8_PAYLOAD_TYPE = 96;
const VP8_PAYLOAD_TYPE = 122; /* Using 122 for testing purposes */ //96;
type SdpMedia = {
type: string;
@ -31,7 +30,7 @@ export class SdpProcessor {
codec: "opus",
rate: 48000,
encoding: 2,
fmtp: { minptime: 1, maxptime: 20, useinbandfec: 1, stereo: 1 },
fmtp: { minptime: 1, maxptime: 20, useinbandfec: 1, stereo: 0 },
rtcpFb: [ "transport-cc" ]
},
{

17
web/app/rtc/video/Connection.ts
View File

@ -13,6 +13,7 @@ import {LogCategory, logDebug, logError, logWarn} from "tc-shared/log";
import {Settings, settings} from "tc-shared/settings";
import {RtpVideoClient} from "tc-backend/web/rtc/video/VideoClient";
import {tr} from "tc-shared/i18n/localize";
import {ConnectionState} from "tc-shared/ConnectionHandler";
type VideoBroadcast = {
readonly source: VideoSource;
@ -26,6 +27,7 @@ export class RtpVideoConnection implements VideoConnection {
private readonly events: Registry<VideoConnectionEvent>;
private readonly listenerClientMoved;
private readonly listenerRtcStateChanged;
private readonly listenerConnectionStateChanged;
private connectionState: VideoConnectionStatus;
private broadcasts: {[T in VideoBroadcastType]: VideoBroadcast} = {
@ -54,6 +56,12 @@ export class RtpVideoConnection implements VideoConnection {
}
}
});
this.listenerConnectionStateChanged = this.rtcConnection.getConnection().events.on("notify_connection_state_changed", event => {
if(event.newState !== ConnectionState.CONNECTED) {
this.stopBroadcasting("camera");
this.stopBroadcasting("screen");
}
});
this.listenerRtcStateChanged = this.rtcConnection.getEvents().on("notify_state_changed", event => this.handleRtcConnectionStateChanged(event));
@ -95,6 +103,7 @@ export class RtpVideoConnection implements VideoConnection {
destroy() {
this.listenerClientMoved();
this.listenerRtcStateChanged();
this.listenerConnectionStateChanged();
}
getEvents(): Registry<VideoConnectionEvent> {
@ -163,13 +172,16 @@ export class RtpVideoConnection implements VideoConnection {
}
stopBroadcasting(type: VideoBroadcastType, skipRtcStop?: boolean) {
const broadcast = this.broadcasts[type];
if(!broadcast) {
return;
}
if(!skipRtcStop) {
this.rtcConnection.stopTrackBroadcast(type === "camera" ? "video" : "video-screen");
}
this.rtcConnection.setTrackSource(type === "camera" ? "video" : "video-screen", null).then(undefined);
if(this.broadcasts[type]) {
const broadcast = this.broadcasts[type];
const oldState = this.broadcasts[type].state;
this.broadcasts[type].active = false;
this.broadcasts[type] = undefined;
@ -177,7 +189,6 @@ export class RtpVideoConnection implements VideoConnection {
this.events.fire("notify_local_broadcast_state_changed", { oldState: oldState, newState: VideoBroadcastState.Stopped, broadcastType: type });
}
}
registerVideoClient(clientId: number) {
if(typeof this.registeredClients[clientId] !== "undefined") {

2
web/app/rtc/voice/Connection.ts
View File

@ -49,6 +49,8 @@ export class RtpVoiceConnection extends AbstractVoiceConnection {
this.rtcConnection.getEvents().on("notify_state_changed",
this.listenerRtcStateChanged = event => this.handleRtcConnectionStateChanged(event));
/* FIXME: Listener for audio! */
this.listenerClientMoved = this.rtcConnection.getConnection().command_handler_boss().register_explicit_handler("notifyclientmoved", event => {
const localClientId = this.rtcConnection.getConnection().client.getClientId();
for(const data of event.arguments) {