Moved the RTP part to the shared part and trick the user to activate audio before connecting to any server.

This commit is contained in:
WolverinDEV 2020-11-28 19:33:54 +01:00
parent 1cc91668b7
commit 4c16b8373a
14 changed files with 100 additions and 114 deletions

View file

@ -32,7 +32,6 @@ import {W2GPluginCmdHandler} from "./video-viewer/W2GPlugin";
import {VoiceConnectionStatus, WhisperSessionInitializeData} from "./connection/VoiceConnection";
import {getServerConnectionFactory} from "./connection/ConnectionFactory";
import {WhisperSession} from "./voice/VoiceWhisper";
import {spawnEchoTestModal} from "./ui/modal/echo-test/Controller";
import {ServerFeature, ServerFeatures} from "./connection/ServerFeatures";
import {ChannelTree} from "./tree/ChannelTree";
import {LocalClientEntry} from "./tree/Client";
@ -371,17 +370,19 @@ export class ConnectionHandler {
this.permissions.requestPermissionList();
/*
There is no need to request the server groups since they must be send by the server
if(this.groups.serverGroups.length == 0)
if(this.groups.serverGroups.length == 0) {
this.groups.requestGroups();
}
*/
this.settings.setServer(this.channelTree.server.properties.virtualserver_unique_identifier);
/* apply the server settings */
if(this.client_status.channel_subscribe_all)
if(this.client_status.channel_subscribe_all) {
this.channelTree.subscribe_all_channels();
else
} else {
this.channelTree.unsubscribe_all_channels();
}
this.channelTree.toggle_server_queries(this.client_status.queries_visible);
this.sync_status_with_server();
@ -398,6 +399,7 @@ export class ConnectionHandler {
if(!result) {
return;
}
if(this.serverFeatures.supportsFeature(ServerFeature.WHISPER_ECHO)) {
global_client_actions.fire("action_open_window", { window: "server-echo-test", connection: this });
}

View file

@ -7,13 +7,8 @@ import {AbstractCommandHandler} from "tc-shared/connection/AbstractCommandHandle
import {CommandResult} from "tc-shared/connection/ServerConnectionDeclaration";
import {tr} from "tc-shared/i18n/localize";
import {Registry} from "tc-shared/events";
import {
RemoteRTPAudioTrack,
RemoteRTPTrackState,
RemoteRTPVideoTrack,
TrackClientInfo
} from "tc-backend/web/rtc/RemoteTrack";
import {SdpCompressor, SdpProcessor} from "tc-backend/web/rtc/SdpUtils";
import {RemoteRTPAudioTrack, RemoteRTPTrackState, RemoteRTPVideoTrack, TrackClientInfo} from "./RemoteTrack";
import {SdpCompressor, SdpProcessor} from "./SdpUtils";
import {context} from "tc-backend/web/audio/player";
import {ErrorCode} from "tc-shared/connection/ErrorCode";
import {WhisperTarget} from "tc-shared/voice/VoiceWhisper";
@ -461,6 +456,8 @@ export interface RTCConnectionEvents {
export class RTCConnection {
public static readonly kEnableSdpTrace = true;
private readonly audioSupport: boolean;
private readonly events: Registry<RTCConnectionEvents>;
private readonly connection: ServerConnection;
private readonly commandHandler: CommandHandler;
@ -492,12 +489,13 @@ export class RTCConnection {
private remoteVideoTracks: {[key: number]: InternalRemoteRTPVideoTrack};
private temporaryStreams: {[key: number]: TemporaryRtpStream} = {};
constructor(connection: ServerConnection) {
constructor(connection: ServerConnection, audioSupport: boolean) {
this.events = new Registry<RTCConnectionEvents>();
this.connection = connection;
this.sdpProcessor = new SdpProcessor();
this.commandHandler = new CommandHandler(connection, this, this.sdpProcessor);
this.retryCalculator = new RetryTimeCalculator(5000, 30000, 10000);
this.audioSupport = audioSupport;
this.connection.command_handler_boss().register_handler(this.commandHandler);
this.reset(true);
@ -509,6 +507,10 @@ export class RTCConnection {
this.connection.command_handler_boss().unregister_handler(this.commandHandler);
}
isAudioEnabled() : boolean {
return this.audioSupport;
}
getConnection() : ServerConnection {
return this.connection;
}
@ -593,6 +595,7 @@ export class RTCConnection {
switch (type) {
case "audio":
case "audio-whisper":
if(!this.audioSupport) { throw tr("audio support isn't enabled"); }
if(source && source.kind !== "audio") { throw tr("invalid track type"); }
break;
case "video":
@ -618,6 +621,21 @@ export class RTCConnection {
throw tr("missing transceiver");
}
switch (type) {
case "audio":
if(!this.audioSupport) {
throw tr("audio support isn't enabled");
}
break;
case "video":
case "video-screen":
break;
default:
throw tr("invalid broadcast type");
}
try {
await this.connection.send_command("rtcbroadcast", {
type: broadcastableTrackTypeToNumber(type),
@ -630,6 +648,10 @@ export class RTCConnection {
}
public async startWhisper(target: WhisperTarget) : Promise<void> {
if(!this.audioSupport) {
throw tr("audio support isn't enabled");
}
const transceiver = this.currentTransceiver["audio-whisper"];
if(typeof transceiver === "undefined") {
throw tr("missing transceiver");
@ -718,8 +740,6 @@ export class RTCConnection {
}
}
private enableDtx(_sender: RTCRtpSender) { }
public doInitialSetup() {
/* initialize rtc connection */
this.retryCalculator.reset();
@ -740,24 +760,23 @@ export class RTCConnection {
iceServers: [{ urls: ["stun:stun.l.google.com:19302", "stun:stun1.l.google.com:19302"] }]
});
this.currentTransceiver["audio"] = this.peer.addTransceiver("audio");
this.enableDtx(this.currentTransceiver["audio"].sender);
if(this.audioSupport) {
this.currentTransceiver["audio"] = this.peer.addTransceiver("audio");
this.currentTransceiver["audio-whisper"] = this.peer.addTransceiver("audio");
this.currentTransceiver["audio-whisper"] = this.peer.addTransceiver("audio");
this.enableDtx(this.currentTransceiver["audio-whisper"].sender);
/* add some other transceivers for later use */
for(let i = 0; i < 8; i++) {
this.peer.addTransceiver("audio");
}
}
this.currentTransceiver["video"] = this.peer.addTransceiver("video");
this.currentTransceiver["video-screen"] = this.peer.addTransceiver("video");
/* add some other transceivers for later use */
/*
for(let i = 0; i < 8; i++) {
this.peer.addTransceiver("audio");
}
for(let i = 0; i < 4; i++) {
this.peer.addTransceiver("video");
}
*/
this.peer.onicecandidate = event => this.handleIceCandidate(event.candidate);
this.peer.onicecandidateerror = event => this.handleIceCandidateError(event);
@ -805,8 +824,7 @@ export class RTCConnection {
}
await this.currentTransceiver[type].sender.replaceTrack(target);
target.enabled = true;
console.error("Replaced track for %o (Fallback: %o)", type, target === fallback);
logTrace(LogCategory.WEBRTC, "Replaced track for %o (Fallback: %o)", type, target === fallback);
}
}
@ -816,7 +834,7 @@ export class RTCConnection {
const peer = this.peer;
await this.updateTracks();
const offer = await peer.createOffer({ iceRestart: false, offerToReceiveAudio: true, offerToReceiveVideo: true });
const offer = await peer.createOffer({ iceRestart: false, offerToReceiveAudio: this.audioSupport, offerToReceiveVideo: true });
if(offer.type !== "offer") { throw tr("created ofer isn't of type offer"); }
if(this.peer !== peer) { return; }
@ -970,6 +988,10 @@ export class RTCConnection {
const tempInfo = this.releaseTemporaryStream(ssrc);
if(event.track.kind === "audio") {
if(!this.audioSupport) {
logWarn(LogCategory.WEBRTC, tr("Received remote audio track %d but audio has been disabled. Dropping track."), ssrc);
return;
}
const track = new InternalRemoteRTPAudioTrack(ssrc, event.transceiver);
logDebug(LogCategory.WEBRTC, tr("Received remote audio track on ssrc %d"), ssrc);
if(tempInfo?.info !== undefined) {

View file

@ -1,4 +1,4 @@
import {MediaDescription, SessionDescription} from "sdp-transform";
import {SessionDescription} from "sdp-transform";
import * as sdpTransform from "sdp-transform";
export interface RTCNegotiationMediaMapping {

View file

@ -3,7 +3,6 @@ import {LogCategory, logWarn} from "tc-shared/log";
import {tr} from "tc-shared/i18n/localize";
import * as aplayer from "tc-backend/web/audio/player";
export interface TrackClientInfo {
media?: number,

View file

@ -15,13 +15,6 @@ const OPUS_VOICE_PAYLOAD_TYPE = 111;
const OPUS_MUSIC_PAYLOAD_TYPE = 112;
const H264_PAYLOAD_TYPE = 126;
type SdpMedia = {
type: string;
port: number;
protocol: string;
payloads?: string;
} & MediaDescription;
export class SdpProcessor {
private static readonly kAudioCodecs: SdpCodec[] = [
// Primary audio format

View file

@ -9,10 +9,10 @@ import {
} from "tc-shared/connection/VideoConnection";
import {Registry} from "tc-shared/events";
import {VideoSource} from "tc-shared/video/VideoSource";
import {RTCConnection, RTCConnectionEvents, RTPConnectionState} from "tc-backend/web/rtc/Connection";
import {RTCConnection, RTCConnectionEvents, RTPConnectionState} from "../Connection";
import {LogCategory, logDebug, logError, logWarn} from "tc-shared/log";
import {Settings, settings} from "tc-shared/settings";
import {RtpVideoClient} from "tc-backend/web/rtc/video/VideoClient";
import {RtpVideoClient} from "./VideoClient";
import {tr} from "tc-shared/i18n/localize";
import {ConnectionState} from "tc-shared/ConnectionHandler";
import {ConnectionStatistics} from "tc-shared/connection/ConnectionBase";

View file

@ -5,7 +5,7 @@ import {
VideoClientEvents
} from "tc-shared/connection/VideoConnection";
import {Registry} from "tc-shared/events";
import {RemoteRTPTrackState, RemoteRTPVideoTrack} from "tc-backend/web/rtc/RemoteTrack";
import {RemoteRTPTrackState, RemoteRTPVideoTrack} from "../RemoteTrack";
import {LogCategory, logWarn} from "tc-shared/log";
export class RtpVideoClient implements VideoClient {

View file

@ -125,6 +125,17 @@ export function handle_connect_request(properties: ConnectRequestData, connectio
if(profile && profile.valid()) {
settings.changeGlobal(Settings.KEY_USER_IS_NEW, false);
if(!aplayer.initialized()) {
spawnYesNo(tra("Connect to {}", properties.address), tra("Would you like to connect to {}?", properties.address), result => {
if(result) {
aplayer.on_ready(() => handle_connect_request(properties, connection));
} else {
/* Well... the client don't want to... */
}
}).open();
return;
}
connection.startConnection(properties.address, profile, true, {
nickname: username,
password: password.length > 0 ? {
@ -359,7 +370,7 @@ const task_connect_handler: loader.Task = {
if(chandler && !settings.static(Settings.KEY_CONNECT_NO_SINGLE_INSTANCE)) {
try {
await chandler.post_connect_request(connect_data, () => new Promise<boolean>((resolve, reject) => {
await chandler.post_connect_request(connect_data, () => new Promise<boolean>(resolve => {
spawnYesNo(tr("Another TeaWeb instance is already running"), tra("Another TeaWeb instance is already running.{:br:}Would you like to connect there?"), response => {
resolve(response);
}, {
@ -412,50 +423,6 @@ const task_connect_handler: loader.Task = {
const task_certificate_callback: loader.Task = {
name: "certificate accept tester",
function: async () => {
/*
This is not needed any more. If we would use the certificate accept stuff, we would have an extra loader target.
I'm just keeping this, so later I've not to to any work, writing this, again.
const certificate_accept = settings.static_global(Settings.KEY_CERTIFICATE_CALLBACK, undefined);
if(certificate_accept) {
log.info(LogCategory.IPC, tr("Using this instance as certificate callback. ID: %s"), certificate_accept);
try {
try {
await bipc.getInstance().post_certificate_accpected(certificate_accept);
} catch(e) {} //FIXME remove!
log.info(LogCategory.IPC, tr("Other instance has acknowledged out work. Closing this window."));
const seconds_tag = $.spawn("a");
let seconds = 5;
let interval_id;
interval_id = setInterval(() => {
seconds--;
seconds_tag.text(seconds.toString());
if(seconds <= 0) {
clearTimeout(interval_id);
log.info(LogCategory.GENERAL, tr("Closing window"));
window.close();
return;
}
}, 1000);
createInfoModal(
tr("Certificate acccepted successfully"),
formatMessage(tr("You've successfully accepted the certificate.{:br:}This page will close in {0} seconds."), seconds_tag),
{
closeable: false,
footer: undefined
}
).open();
return;
} catch(error) {
log.warn(LogCategory.IPC, tr("Failed to successfully post certificate accept status: %o"), error);
}
} else {
log.info(LogCategory.IPC, tr("We're not used to accept certificated. Booting app."));
}
*/
loader.register_task(loader.Stage.LOADED, task_connect_handler);
},
priority: 10

View file

@ -10,7 +10,7 @@ import {global_client_actions} from "tc-shared/events/GlobalEvents";
import {VoiceConnectionStatus} from "tc-shared/connection/VoiceConnection";
import {Settings, settings} from "tc-shared/settings";
import {CommandResult} from "tc-shared/connection/ServerConnectionDeclaration";
import {LogCategory, logError} from "tc-shared/log";
import {LogCategory, logError, logTrace, logWarn} from "tc-shared/log";
import {ServerFeature} from "tc-shared/connection/ServerFeatures";
export function spawnEchoTestModal(connection: ConnectionHandler) {
@ -40,7 +40,7 @@ export function spawnEchoTestModal(connection: ConnectionHandler) {
modal.destroy();
});
modal.events.on("close", () => events.fire("notify_close"));
modal.events.on("close", () => events.fire_react("notify_close"));
modal.events.on("destroy", () => {
events.fire("notify_destroy");
events.destroy();
@ -72,8 +72,8 @@ function initializeController(connection: ConnectionHandler, events: Registry<Ec
if (event.status === "success") {
events.fire("action_close");
} else {
events.fire("action_stop_test");
events.fire("notify_test_phase", {phase: "troubleshooting"});
events.fire_react("action_stop_test");
events.fire_react("notify_test_phase", {phase: "troubleshooting"});
}
});
@ -81,7 +81,7 @@ function initializeController(connection: ConnectionHandler, events: Registry<Ec
if (event.status === "aborted") {
events.fire("action_close");
} else {
events.fire("notify_test_phase", {phase: "testing"});
events.fire_react("notify_test_phase", {phase: "testing"});
events.fire("action_start_test");
}
});
@ -94,28 +94,28 @@ function initializeController(connection: ConnectionHandler, events: Registry<Ec
}
switch (state) {
case VoiceConnectionStatus.Connected:
events.fire("notify_voice_connection_state", {state: "connected"});
events.fire_react("notify_voice_connection_state", {state: "connected"});
break;
case VoiceConnectionStatus.Disconnected:
case VoiceConnectionStatus.Disconnecting:
events.fire("notify_voice_connection_state", {state: "disconnected"});
events.fire_react("notify_voice_connection_state", {state: "disconnected"});
break;
case VoiceConnectionStatus.Connecting:
events.fire("notify_voice_connection_state", {state: "connecting"});
events.fire_react("notify_voice_connection_state", {state: "connecting"});
break;
case VoiceConnectionStatus.ClientUnsupported:
events.fire("notify_voice_connection_state", {state: "unsupported-client"});
events.fire_react("notify_voice_connection_state", {state: "unsupported-client"});
break;
case VoiceConnectionStatus.ServerUnsupported:
events.fire("notify_voice_connection_state", {state: "unsupported-server"});
events.fire_react("notify_voice_connection_state", {state: "unsupported-server"});
break;
case VoiceConnectionStatus.Failed:
events.fire("notify_voice_connection_state", {state: "failed", message: connection.getServerConnection().getVoiceConnection().getFailedMessage() });
events.fire_react("notify_voice_connection_state", {state: "failed", message: connection.getServerConnection().getVoiceConnection().getFailedMessage() });
break;
}
};
@ -142,7 +142,7 @@ function initializeController(connection: ConnectionHandler, events: Registry<Ec
const setTestState = (state: TestState) => {
testState = state;
events.fire("notify_test_state", {state: state});
events.fire_react("notify_test_state", {state: state});
}
let testId = 0;
@ -170,8 +170,10 @@ function initializeController(connection: ConnectionHandler, events: Registry<Ec
return;
}
logTrace(LogCategory.VOICE, tr("Echo test started."));
setTestState({state: "running"});
}).catch(error => {
logWarn(LogCategory.VOICE, tr("Failed to start echo test: %o"), error);
if (currentTestId !== testId) {
return;
}

View file

@ -19,13 +19,13 @@ import {WrappedWebSocket} from "tc-backend/web/connection/WrappedWebSocket";
import {AbstractVoiceConnection} from "tc-shared/connection/VoiceConnection";
import {parseCommand} from "tc-backend/web/connection/CommandParser";
import {ServerAddress} from "tc-shared/tree/Server";
import {RTCConnection} from "tc-backend/web/rtc/Connection";
import {RtpVoiceConnection} from "tc-backend/web/rtc/voice/Connection";
import {RtpVideoConnection} from "tc-backend/web/rtc/video/Connection";
import {RtpVoiceConnection} from "tc-backend/web/voice/Connection";
import {VideoConnection} from "tc-shared/connection/VideoConnection";
import {VoiceConnection} from "tc-backend/web/legacy/voice/VoiceHandler";
import {LegacySupportVoiceBridge} from "tc-backend/web/connection/LegacySupportVoiceBridge";
import {ServerFeature} from "tc-shared/connection/ServerFeatures";
import {RTCConnection} from "tc-shared/connection/rtc/Connection";
import {RtpVideoConnection} from "tc-shared/connection/rtc/video/Connection";
class ReturnListener<T> {
resolve: (value?: T | PromiseLike<T>) => void;
@ -82,7 +82,7 @@ export class ServerConnection extends AbstractServerConnection {
this.commandHandlerBoss.register_handler(this.defaultCommandHandler);
this.command_helper.initialize();
this.rtcConnection = new RTCConnection(this);
this.rtcConnection = new RTCConnection(this, true);
this.voiceConnection = new RtpVoiceConnection(this, this.rtcConnection);
this.videoConnection = new RtpVideoConnection(this.rtcConnection);

View file

@ -11,16 +11,16 @@ import {
WhisperSessionState,
WhisperTarget
} from "tc-shared/voice/VoiceWhisper";
import {RTCConnection, RTCConnectionEvents, RTPConnectionState} from "tc-backend/web/rtc/Connection";
import {RTCConnection, RTCConnectionEvents, RTPConnectionState} from "tc-shared/connection/rtc/Connection";
import {AbstractServerConnection, ConnectionStatistics} from "tc-shared/connection/ConnectionBase";
import {VoicePlayerState} from "tc-shared/voice/VoicePlayer";
import * as log from "tc-shared/log";
import {LogCategory, logDebug, logError, logTrace, logWarn} from "tc-shared/log";
import * as aplayer from "../../audio/player";
import * as aplayer from "../audio/player";
import {tr} from "tc-shared/i18n/localize";
import {RtpVoiceClient} from "tc-backend/web/rtc/voice/VoiceClient";
import {RtpVoiceClient} from "tc-backend/web/voice/VoiceClient";
import {InputConsumerType} from "tc-shared/voice/RecorderBase";
import {RtpWhisperSession} from "tc-backend/web/rtc/voice/WhisperClient";
import {RtpWhisperSession} from "tc-backend/web/voice/WhisperClient";
type CancelableWhisperTarget = WhisperTarget & { canceled: boolean };
export class RtpVoiceConnection extends AbstractVoiceConnection {
@ -504,6 +504,8 @@ export class RtpVoiceConnection extends AbstractVoiceConnection {
this.dropWhisperSession(session);
});
this.events.fire("notify_whisper_created", { session: session });
} else {
session.setRtpTrack(event.track);
}
break;
}

View file

@ -1,7 +1,7 @@
import {VoiceClient} from "tc-shared/voice/VoiceClient";
import {RtpVoicePlayer} from "./RtpVoicePlayer";
import {VoicePlayer} from "./VoicePlayer";
export class RtpVoiceClient extends RtpVoicePlayer implements VoiceClient {
export class RtpVoiceClient extends VoicePlayer implements VoiceClient {
private readonly clientId: number;
constructor(clientId: number) {

View file

@ -1,18 +1,17 @@
import {
VoicePlayer,
VoicePlayerEvents,
VoicePlayerLatencySettings,
VoicePlayerState
} from "tc-shared/voice/VoicePlayer";
import {Registry} from "tc-shared/events";
import {LogCategory, logWarn} from "tc-shared/log";
import {RemoteRTPAudioTrack, RemoteRTPTrackState} from "tc-backend/web/rtc/RemoteTrack";
import {RemoteRTPAudioTrack, RemoteRTPTrackState} from "tc-shared/connection/rtc/RemoteTrack";
export interface RtpVoicePlayerEvents {
notify_state_changed: { oldState: VoicePlayerState, newState: VoicePlayerState }
}
export class RtpVoicePlayer implements VoicePlayer {
export class VoicePlayer implements VoicePlayer {
readonly events: Registry<VoicePlayerEvents>;
private readonly listenerTrackStateChanged;
@ -111,5 +110,5 @@ export class RtpVoicePlayer implements VoicePlayer {
return { minBufferTime: 0, maxBufferTime: 0 };
}
resetLatencySettings() { }
setLatencySettings(settings: VoicePlayerLatencySettings) { }
setLatencySettings(_settings: VoicePlayerLatencySettings) { }
}

View file

@ -1,9 +1,9 @@
import {WhisperSession, WhisperSessionEvents, WhisperSessionState} from "tc-shared/voice/VoiceWhisper";
import {Registry} from "tc-shared/events";
import {VoicePlayer, VoicePlayerState} from "tc-shared/voice/VoicePlayer";
import {VoicePlayerState} from "tc-shared/voice/VoicePlayer";
import {WhisperSessionInitializeData} from "tc-shared/connection/VoiceConnection";
import {RtpVoicePlayer} from "./RtpVoicePlayer";
import {RemoteRTPAudioTrack, TrackClientInfo} from "tc-backend/web/rtc/RemoteTrack";
import {VoicePlayer} from "./VoicePlayer";
import {RemoteRTPAudioTrack, TrackClientInfo} from "tc-shared/connection/rtc/RemoteTrack";
export class RtpWhisperSession implements WhisperSession {
readonly events: Registry<WhisperSessionEvents>;
@ -19,7 +19,7 @@ export class RtpWhisperSession implements WhisperSession {
private sessionTimeoutId: number;
private lastWhisperTimestamp: number;
private voicePlayer: RtpVoicePlayer;
private voicePlayer: VoicePlayer;
constructor(track: RemoteRTPAudioTrack, info: TrackClientInfo) {
this.events = new Registry<WhisperSessionEvents>();
@ -78,7 +78,7 @@ export class RtpWhisperSession implements WhisperSession {
this.sessionBlocked = data.blocked;
this.sessionTimeout = data.sessionTimeout;
this.voicePlayer = new RtpVoicePlayer();
this.voicePlayer = new VoicePlayer();
this.voicePlayer.setRtpTrack(this.track);
this.voicePlayer.setGloballyMuted(this.globallyMuted);
this.voicePlayer.setVolume(data.volume);