547 lines
20 KiB
TypeScript
547 lines
20 KiB
TypeScript
import * as log from "tc-shared/log";
|
|
import {LogCategory, logDebug, logError, logInfo, logTrace, logWarn} from "tc-shared/log";
|
|
import * as aplayer from "../../audio/player";
|
|
import {ServerConnection} from "../../connection/ServerConnection";
|
|
import {RecorderProfile} from "tc-shared/voice/RecorderProfile";
|
|
import {VoiceClientController} from "./VoiceClient";
|
|
import {tr} from "tc-shared/i18n/localize";
|
|
import {
|
|
AbstractVoiceConnection,
|
|
VoiceConnectionStatus,
|
|
WhisperSessionInitializer
|
|
} from "tc-shared/connection/VoiceConnection";
|
|
import {createErrorModal} from "tc-shared/ui/elements/Modal";
|
|
import {ConnectionStatistics, ServerConnectionEvents} from "tc-shared/connection/ConnectionBase";
|
|
import {ConnectionState} from "tc-shared/ConnectionHandler";
|
|
import {VoiceBridge, VoicePacket, VoiceWhisperPacket} from "./bridge/VoiceBridge";
|
|
import {NativeWebRTCVoiceBridge} from "./bridge/NativeWebRTCVoiceBridge";
|
|
import {
|
|
kUnknownWhisperClientUniqueId,
|
|
WhisperSession,
|
|
WhisperSessionState,
|
|
WhisperTarget
|
|
} from "tc-shared/voice/VoiceWhisper";
|
|
import {VoiceClient} from "tc-shared/voice/VoiceClient";
|
|
import {WebWhisperSession} from "./VoiceWhisper";
|
|
import {VoicePlayerState} from "tc-shared/voice/VoicePlayer";
|
|
|
|
type CancelableWhisperTarget = WhisperTarget & { canceled: boolean };
|
|
|
|
export class VoiceConnection extends AbstractVoiceConnection {
|
|
readonly connection: ServerConnection;
|
|
|
|
private readonly serverConnectionStateListener;
|
|
private connectionState: VoiceConnectionStatus;
|
|
private failedConnectionMessage: string;
|
|
|
|
private localAudioStarted = false;
|
|
private connectionLostModalOpen = false;
|
|
|
|
private connectAttemptCounter = 0;
|
|
private awaitingAudioInitialize = false;
|
|
|
|
private currentAudioSource: RecorderProfile;
|
|
private voiceClients: {[key: number]: VoiceClientController} = {};
|
|
|
|
private whisperSessionInitializer: WhisperSessionInitializer;
|
|
private whisperSessions: {[key: number]: WebWhisperSession} = {};
|
|
|
|
private whisperTarget: CancelableWhisperTarget | undefined;
|
|
private whisperTargetInitialize: Promise<void>;
|
|
|
|
private voiceBridge: VoiceBridge;
|
|
|
|
private encoderCodec: number = 5;
|
|
|
|
private lastConnectAttempt: number = 0;
|
|
|
|
private currentlyReplayingVoice: boolean = false;
|
|
private readonly voiceClientStateChangedEventListener;
|
|
private readonly whisperSessionStateChangedEventListener;
|
|
|
|
constructor(connection: ServerConnection) {
|
|
super(connection);
|
|
|
|
this.setWhisperSessionInitializer(undefined);
|
|
|
|
this.connectionState = VoiceConnectionStatus.Disconnected;
|
|
|
|
this.connection = connection;
|
|
this.connection.events.on("notify_connection_state_changed",
|
|
this.serverConnectionStateListener = this.handleServerConnectionStateChanged.bind(this));
|
|
|
|
this.voiceClientStateChangedEventListener = this.handleVoiceClientStateChange.bind(this);
|
|
this.whisperSessionStateChangedEventListener = this.handleWhisperSessionStateChange.bind(this);
|
|
}
|
|
|
|
getConnectionState(): VoiceConnectionStatus {
|
|
return this.connectionState;
|
|
}
|
|
|
|
getFailedMessage(): string {
|
|
return this.failedConnectionMessage;
|
|
}
|
|
|
|
getRetryTimestamp(): number | 0 {
|
|
return 0;
|
|
}
|
|
|
|
destroy() {
|
|
this.connection.events.off(this.serverConnectionStateListener);
|
|
this.dropVoiceBridge();
|
|
this.acquireVoiceRecorder(undefined, true).catch(error => {
|
|
logWarn(LogCategory.VOICE, tr("Failed to release voice recorder: %o"), error);
|
|
}).then(() => {
|
|
for(const client of Object.keys(this.voiceClients).map(clientId => this.voiceClients[clientId])) {
|
|
client.abortReplay();
|
|
}
|
|
this.voiceClients = undefined;
|
|
this.currentAudioSource = undefined;
|
|
});
|
|
if(Object.keys(this.voiceClients).length !== 0) {
|
|
logWarn(LogCategory.AUDIO, tr("Voice connection will be destroyed, but some voice clients are still left (%d)."), Object.keys(this.voiceClients).length);
|
|
}
|
|
const whisperSessions = Object.keys(this.whisperSessions);
|
|
whisperSessions.forEach(session => this.whisperSessions[session].destroy());
|
|
this.whisperSessions = {};
|
|
|
|
this.events.destroy();
|
|
}
|
|
|
|
reset() {
|
|
this.dropVoiceBridge();
|
|
}
|
|
|
|
async acquireVoiceRecorder(recorder: RecorderProfile | undefined, enforce?: boolean) {
|
|
if(this.currentAudioSource === recorder && !enforce) {
|
|
return;
|
|
}
|
|
|
|
if(this.currentAudioSource) {
|
|
await this.voiceBridge?.setInput(undefined);
|
|
this.currentAudioSource.callback_unmount = undefined;
|
|
await this.currentAudioSource.unmount();
|
|
}
|
|
|
|
/* unmount our target recorder */
|
|
await recorder?.unmount();
|
|
|
|
this.handleRecorderStop();
|
|
const oldRecorder = recorder;
|
|
this.currentAudioSource = recorder;
|
|
|
|
if(recorder) {
|
|
recorder.current_handler = this.connection.client;
|
|
|
|
recorder.callback_unmount = this.handleRecorderUnmount.bind(this);
|
|
recorder.callback_start = this.handleRecorderStart.bind(this);
|
|
recorder.callback_stop = this.handleRecorderStop.bind(this);
|
|
|
|
recorder.callback_input_initialized = async input => {
|
|
if(!this.voiceBridge)
|
|
return;
|
|
|
|
await this.voiceBridge.setInput(input);
|
|
};
|
|
|
|
if(recorder.input && this.voiceBridge) {
|
|
await this.voiceBridge.setInput(recorder.input);
|
|
}
|
|
|
|
if(!recorder.input || recorder.input.isFiltered()) {
|
|
this.handleRecorderStop();
|
|
} else {
|
|
this.handleRecorderStart();
|
|
}
|
|
} else {
|
|
await this.voiceBridge?.setInput(undefined);
|
|
}
|
|
|
|
this.events.fire("notify_recorder_changed", {
|
|
oldRecorder: oldRecorder,
|
|
newRecorder: recorder
|
|
});
|
|
}
|
|
|
|
public startVoiceBridge() {
|
|
if(!aplayer.initialized()) {
|
|
logDebug(LogCategory.VOICE, tr("Audio player isn't initialized yet. Waiting for it to initialize."));
|
|
if(!this.awaitingAudioInitialize) {
|
|
this.awaitingAudioInitialize = true;
|
|
aplayer.on_ready(() => this.startVoiceBridge());
|
|
}
|
|
return;
|
|
}
|
|
|
|
if(this.connection.getConnectionState() !== ConnectionState.CONNECTED) {
|
|
return;
|
|
}
|
|
|
|
this.lastConnectAttempt = Date.now();
|
|
this.connectAttemptCounter++;
|
|
if(this.voiceBridge) {
|
|
this.voiceBridge.callbackDisconnect = undefined;
|
|
this.voiceBridge.disconnect();
|
|
}
|
|
|
|
this.voiceBridge = new NativeWebRTCVoiceBridge();
|
|
this.voiceBridge.callback_incoming_voice = packet => this.handleVoicePacket(packet);
|
|
this.voiceBridge.callback_incoming_whisper = packet => this.handleWhisperPacket(packet);
|
|
this.voiceBridge.callback_send_control_data = (request, payload) => {
|
|
this.connection.sendData(JSON.stringify(Object.assign({
|
|
type: "WebRTC",
|
|
request: request
|
|
}, payload)))
|
|
};
|
|
this.voiceBridge.callbackDisconnect = () => {
|
|
this.connection.client.log.log("connection.voice.dropped", { });
|
|
if(!this.connectionLostModalOpen) {
|
|
this.connectionLostModalOpen = true;
|
|
const modal = createErrorModal(tr("Voice connection lost"), tr("Lost voice connection to the target server. Trying to reconnect..."));
|
|
modal.close_listener.push(() => this.connectionLostModalOpen = false);
|
|
modal.open();
|
|
}
|
|
logInfo(LogCategory.WEBRTC, tr("Lost voice connection to target server. Trying to reconnect."));
|
|
this.executeVoiceBridgeReconnect();
|
|
}
|
|
|
|
this.connection.client.log.log("connection.voice.connect", { attemptCount: this.connectAttemptCounter });
|
|
this.setConnectionState(VoiceConnectionStatus.Connecting);
|
|
this.voiceBridge.connect().then(result => {
|
|
if(result.type === "success") {
|
|
this.lastConnectAttempt = 0;
|
|
this.connectAttemptCounter = 0;
|
|
|
|
this.connection.client.log.log("connection.voice.connect.succeeded", { });
|
|
const currentInput = this.voiceRecorder()?.input;
|
|
if(currentInput) {
|
|
this.voiceBridge.setInput(currentInput).catch(error => {
|
|
createErrorModal(tr("Input recorder attechment failed"), tr("Failed to apply the current microphone recorder to the voice sender.")).open();
|
|
logWarn(LogCategory.VOICE, tr("Failed to apply the input to the voice bridge: %o"), error);
|
|
this.handleRecorderUnmount();
|
|
});
|
|
}
|
|
|
|
this.setConnectionState(VoiceConnectionStatus.Connected);
|
|
} else if(result.type === "canceled") {
|
|
/* we've to do nothing here */
|
|
} else if(result.type === "failed") {
|
|
let doReconnect = result.allowReconnect && this.connectAttemptCounter < 5;
|
|
logWarn(LogCategory.VOICE, tr("Failed to setup voice bridge: %s. Reconnect: %o"), result.message, doReconnect);
|
|
|
|
this.connection.client.log.log("connection.voice.connect.failed", {
|
|
reason: result.message,
|
|
reconnect_delay: doReconnect ? 1 : 0
|
|
});
|
|
|
|
if(doReconnect) {
|
|
this.executeVoiceBridgeReconnect();
|
|
} else {
|
|
this.failedConnectionMessage = result.message;
|
|
this.setConnectionState(VoiceConnectionStatus.Failed);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
private executeVoiceBridgeReconnect() {
|
|
/* TODO: May some kind of incremental timeout? */
|
|
this.startVoiceBridge();
|
|
}
|
|
|
|
private dropVoiceBridge() {
|
|
if(this.voiceBridge) {
|
|
this.voiceBridge.callbackDisconnect = undefined;
|
|
this.voiceBridge.disconnect();
|
|
this.voiceBridge = undefined;
|
|
}
|
|
this.setConnectionState(VoiceConnectionStatus.Disconnected);
|
|
}
|
|
|
|
handleControlPacket(json) {
|
|
this.voiceBridge.handleControlData(json["request"], json);
|
|
return;
|
|
}
|
|
|
|
protected handleVoicePacket(packet: VoicePacket) {
|
|
const chandler = this.connection.client;
|
|
if(chandler.isSpeakerMuted() || chandler.isSpeakerDisabled()) /* we dont need to do anything with sound playback when we're not listening to it */
|
|
return;
|
|
|
|
let client = this.findVoiceClient(packet.clientId);
|
|
if(!client) {
|
|
logError(LogCategory.VOICE, tr("Having voice from unknown audio client? (ClientID: %o)"), packet.clientId);
|
|
return;
|
|
}
|
|
|
|
client.enqueueAudioPacket(packet.voiceId, packet.codec, packet.head, packet.payload);
|
|
}
|
|
|
|
private handleRecorderStop() {
|
|
const chandler = this.connection.client;
|
|
const ch = chandler.getClient();
|
|
if(ch) ch.speaking = false;
|
|
|
|
if(!chandler.connected)
|
|
return false;
|
|
|
|
if(chandler.isMicrophoneMuted())
|
|
return false;
|
|
|
|
logInfo(LogCategory.VOICE, tr("Local voice ended"));
|
|
this.localAudioStarted = false;
|
|
|
|
this.voiceBridge?.sendStopSignal(this.encoderCodec);
|
|
}
|
|
|
|
private handleRecorderStart() {
|
|
const chandler = this.connection.client;
|
|
if(chandler.isMicrophoneMuted()) {
|
|
logWarn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted!"));
|
|
return;
|
|
}
|
|
|
|
this.localAudioStarted = true;
|
|
logInfo(LogCategory.VOICE, tr("Local voice started"));
|
|
|
|
const ch = chandler.getClient();
|
|
if(ch) ch.speaking = true;
|
|
}
|
|
|
|
private handleRecorderUnmount() {
|
|
logInfo(LogCategory.VOICE, "Lost recorder!");
|
|
this.currentAudioSource = undefined;
|
|
this.acquireVoiceRecorder(undefined, true); /* we can ignore the promise because we should finish this directly */
|
|
}
|
|
|
|
private setConnectionState(state: VoiceConnectionStatus) {
|
|
if(this.connectionState === state)
|
|
return;
|
|
|
|
const oldState = this.connectionState;
|
|
this.connectionState = state;
|
|
this.events.fire("notify_connection_status_changed", { newStatus: state, oldStatus: oldState });
|
|
}
|
|
|
|
private handleServerConnectionStateChanged(event: ServerConnectionEvents["notify_connection_state_changed"]) {
|
|
if(event.newState === ConnectionState.CONNECTED) {
|
|
/* startVoiceBridge() will be called by the server connection if we're using this old voice bridge */
|
|
/* this.startVoiceBridge(); */
|
|
} else {
|
|
this.connectAttemptCounter = 0;
|
|
this.lastConnectAttempt = 0;
|
|
this.dropVoiceBridge();
|
|
}
|
|
}
|
|
|
|
voiceRecorder(): RecorderProfile {
|
|
return this.currentAudioSource;
|
|
}
|
|
|
|
availableVoiceClients(): VoiceClient[] {
|
|
return Object.values(this.voiceClients);
|
|
}
|
|
|
|
findVoiceClient(clientId: number) : VoiceClientController | undefined {
|
|
return this.voiceClients[clientId];
|
|
}
|
|
|
|
unregisterVoiceClient(client: VoiceClient) {
|
|
if(!(client instanceof VoiceClientController))
|
|
throw "Invalid client type";
|
|
|
|
client.events.off("notify_state_changed", this.voiceClientStateChangedEventListener);
|
|
delete this.voiceClients[client.getClientId()];
|
|
client.destroy();
|
|
}
|
|
|
|
registerVoiceClient(clientId: number): VoiceClient {
|
|
if(typeof this.voiceClients[clientId] !== "undefined") {
|
|
throw tr("voice client already registered");
|
|
}
|
|
|
|
const client = new VoiceClientController(clientId);
|
|
this.voiceClients[clientId] = client;
|
|
client.events.on("notify_state_changed", this.voiceClientStateChangedEventListener);
|
|
return client;
|
|
}
|
|
|
|
decodingSupported(codec: number): boolean {
|
|
return codec >= 4 && codec <= 5;
|
|
}
|
|
|
|
encodingSupported(codec: number): boolean {
|
|
return codec >= 4 && codec <= 5;
|
|
}
|
|
|
|
getEncoderCodec(): number {
|
|
return this.encoderCodec;
|
|
}
|
|
|
|
setEncoderCodec(codec: number) {
|
|
this.encoderCodec = codec;
|
|
}
|
|
|
|
stopAllVoiceReplays() {
|
|
this.availableVoiceClients().forEach(e => e.abortReplay());
|
|
/* TODO: Whisper sessions as well */
|
|
}
|
|
|
|
isReplayingVoice(): boolean {
|
|
return this.currentlyReplayingVoice;
|
|
}
|
|
|
|
private handleVoiceClientStateChange(/* event: VoicePlayerEvents["notify_state_changed"] */) {
|
|
this.updateVoiceReplaying();
|
|
}
|
|
|
|
private handleWhisperSessionStateChange() {
|
|
this.updateVoiceReplaying();
|
|
}
|
|
|
|
private updateVoiceReplaying() {
|
|
let replaying = false;
|
|
if(this.connectionState === VoiceConnectionStatus.Connected) {
|
|
let index = this.availableVoiceClients().findIndex(client => client.getState() === VoicePlayerState.PLAYING || client.getState() === VoicePlayerState.BUFFERING);
|
|
replaying = index !== -1;
|
|
|
|
if(!replaying) {
|
|
index = this.getWhisperSessions().findIndex(session => session.getSessionState() === WhisperSessionState.PLAYING);
|
|
replaying = index !== -1;
|
|
}
|
|
}
|
|
|
|
if(this.currentlyReplayingVoice !== replaying) {
|
|
this.currentlyReplayingVoice = replaying;
|
|
this.events.fire_later("notify_voice_replay_state_change", { replaying: replaying });
|
|
}
|
|
}
|
|
|
|
protected handleWhisperPacket(packet: VoiceWhisperPacket) {
|
|
const clientId = packet.clientId;
|
|
|
|
let session = this.whisperSessions[clientId];
|
|
if(typeof session !== "object") {
|
|
logDebug(LogCategory.VOICE, tr("Received new whisper from %d (%s)"), packet.clientId, packet.clientNickname);
|
|
session = (this.whisperSessions[clientId] = new WebWhisperSession(packet));
|
|
session.events.on("notify_state_changed", this.whisperSessionStateChangedEventListener);
|
|
this.whisperSessionInitializer(session).then(result => {
|
|
session.initializeFromData(result).then(() => {
|
|
if(this.whisperSessions[clientId] !== session) {
|
|
/* seems to be an old session */
|
|
return;
|
|
}
|
|
this.events.fire("notify_whisper_initialized", { session });
|
|
}).catch(error => {
|
|
logError(LogCategory.VOICE, tr("Failed to internally initialize a voice whisper session: %o"), error);
|
|
session.setSessionState(WhisperSessionState.INITIALIZE_FAILED);
|
|
});
|
|
}).catch(error => {
|
|
logError(LogCategory.VOICE, tr("Failed to initialize whisper session: %o."), error);
|
|
session.initializeFailed();
|
|
});
|
|
|
|
session.events.on("notify_timed_out", () => {
|
|
logTrace(LogCategory.VOICE, tr("Whisper session %d timed out. Dropping session."), session.getClientId());
|
|
this.dropWhisperSession(session);
|
|
});
|
|
this.events.fire("notify_whisper_created", { session: session });
|
|
}
|
|
|
|
session.enqueueWhisperPacket(packet);
|
|
}
|
|
|
|
getWhisperSessions(): WhisperSession[] {
|
|
return Object.values(this.whisperSessions);
|
|
}
|
|
|
|
dropWhisperSession(session: WhisperSession) {
|
|
if(!(session instanceof WebWhisperSession)) {
|
|
throw tr("Session isn't an instance of the web whisper system");
|
|
}
|
|
|
|
session.events.off("notify_state_changed", this.whisperSessionStateChangedEventListener);
|
|
delete this.whisperSessions[session.getClientId()];
|
|
session.destroy();
|
|
}
|
|
|
|
setWhisperSessionInitializer(initializer: WhisperSessionInitializer | undefined) {
|
|
this.whisperSessionInitializer = initializer;
|
|
if(!this.whisperSessionInitializer) {
|
|
this.whisperSessionInitializer = async session => {
|
|
logWarn(LogCategory.VOICE, tr("Missing whisper session initializer. Blocking whisper from %d (%s)"), session.getClientId(), session.getClientUniqueId());
|
|
return {
|
|
clientName: session.getClientName() || tr("Unknown client"),
|
|
clientUniqueId: session.getClientUniqueId() || kUnknownWhisperClientUniqueId,
|
|
|
|
blocked: true,
|
|
volume: 1,
|
|
|
|
sessionTimeout: 60 * 1000
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
getWhisperSessionInitializer(): WhisperSessionInitializer | undefined {
|
|
return this.whisperSessionInitializer;
|
|
}
|
|
|
|
async startWhisper(target: WhisperTarget): Promise<void> {
|
|
while(this.whisperTargetInitialize) {
|
|
this.whisperTarget.canceled = true;
|
|
await this.whisperTargetInitialize;
|
|
}
|
|
|
|
this.whisperTarget = Object.assign({ canceled: false }, target);
|
|
try {
|
|
await (this.whisperTargetInitialize = this.doStartWhisper(this.whisperTarget));
|
|
} finally {
|
|
this.whisperTargetInitialize = undefined;
|
|
}
|
|
}
|
|
|
|
private async doStartWhisper(target: CancelableWhisperTarget) {
|
|
if(target.target === "echo") {
|
|
await this.connection.send_command("setwhispertarget", {
|
|
type: 0x10, /* self */
|
|
target: 0,
|
|
id: 0
|
|
}, { flagset: ["new"] });
|
|
} else if(target.target === "channel-clients") {
|
|
throw "target not yet supported";
|
|
} else if(target.target === "groups") {
|
|
throw "target not yet supported";
|
|
} else {
|
|
throw "target not yet supported";
|
|
}
|
|
|
|
if(target.canceled) {
|
|
return;
|
|
}
|
|
|
|
this.voiceBridge.startWhispering();
|
|
}
|
|
|
|
getWhisperTarget(): WhisperTarget | undefined {
|
|
return this.whisperTarget;
|
|
}
|
|
|
|
stopWhisper() {
|
|
if(this.whisperTarget) {
|
|
this.whisperTarget.canceled = true;
|
|
this.whisperTargetInitialize = undefined;
|
|
this.connection.send_command("clearwhispertarget").catch(error => {
|
|
logWarn(LogCategory.CLIENT, tr("Failed to clear the whisper target: %o"), error);
|
|
});
|
|
}
|
|
this.voiceBridge?.stopWhispering();
|
|
}
|
|
|
|
async getConnectionStats(): Promise<ConnectionStatistics> {
|
|
return {
|
|
bytesSend: 0,
|
|
bytesReceived: 0
|
|
};
|
|
}
|
|
} |