TeaWeb/web/js/voice/VoiceHandler.ts

719 lines
28 KiB
TypeScript
Raw Normal View History

2020-03-30 11:44:18 +00:00
import * as log from "tc-shared/log";
2020-04-03 13:59:32 +00:00
import {LogCategory} from "tc-shared/log";
2020-03-30 11:44:18 +00:00
import * as loader from "tc-loader";
import * as aplayer from "../audio/player";
import * as elog from "tc-shared/ui/frames/server_log";
import {BasicCodec} from "../codec/BasicCodec";
import {CodecType} from "../codec/Codec";
import {createErrorModal} from "tc-shared/ui/elements/Modal";
import {CodecWrapperWorker} from "../codec/CodecWrapperWorker";
import {ServerConnection} from "../connection/ServerConnection";
import {voice} from "tc-shared/connection/ConnectionBase";
import {RecorderProfile} from "tc-shared/voice/RecorderProfile";
import {VoiceClientController} from "./VoiceClient";
import {settings} from "tc-shared/settings";
import {CallbackInputConsumer, InputConsumerType, NodeInputConsumer} from "tc-shared/voice/RecorderBase";
2020-04-03 13:59:32 +00:00
import AbstractVoiceConnection = voice.AbstractVoiceConnection;
2020-03-30 11:44:18 +00:00
import VoiceClient = voice.VoiceClient;
export namespace codec {
class CacheEntry {
instance: BasicCodec;
owner: number;
last_access: number;
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
export function codec_supported(type: CodecType) {
return type == CodecType.OPUS_MUSIC || type == CodecType.OPUS_VOICE;
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
export class CodecPool {
codecIndex: number;
name: string;
type: CodecType;
entries: CacheEntry[] = [];
maxInstances: number = 2;
private _supported: boolean = true;
initialize(cached: number) {
/* test if we're able to use this codec */
const dummy_client_id = 0xFFEF;
this.ownCodec(dummy_client_id, _ => {}).then(codec => {
log.info(LogCategory.VOICE, tr("Release again! (%o)"), codec);
this.releaseCodec(dummy_client_id);
}).catch(error => {
if(this._supported) {
log.warn(LogCategory.VOICE, tr("Disabling codec support for "), this.name);
createErrorModal(tr("Could not load codec driver"), tr("Could not load or initialize codec ") + this.name + "<br>" +
"Error: <code>" + JSON.stringify(error) + "</code>").open();
log.error(LogCategory.VOICE, tr("Failed to initialize the opus codec. Error: %o"), error);
} else {
log.debug(LogCategory.VOICE, tr("Failed to initialize already disabled codec. Error: %o"), error);
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
this._supported = false;
});
}
2018-04-16 18:38:35 +00:00
2020-03-30 11:44:18 +00:00
supported() { return this._supported; }
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
ownCodec?(clientId: number, callback_encoded: (buffer: Uint8Array) => any, create: boolean = true) : Promise<BasicCodec | undefined> {
return new Promise<BasicCodec>((resolve, reject) => {
if(!this._supported) {
reject(tr("unsupported codec!"));
return;
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
let free_slot = 0;
for(let index = 0; index < this.entries.length; index++) {
if(this.entries[index].owner == clientId) {
this.entries[index].last_access = Date.now();
if(this.entries[index].instance.initialized())
resolve(this.entries[index].instance);
2019-04-04 19:47:52 +00:00
else {
2020-03-30 11:44:18 +00:00
this.entries[index].instance.initialise().then((flag) => {
//TODO test success flag
this.ownCodec(clientId, callback_encoded, false).then(resolve).catch(reject);
2020-04-18 18:25:58 +00:00
}).catch(reject);
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
return;
} else if(this.entries[index].owner == 0) {
free_slot = index;
}
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
if(!create) {
resolve(undefined);
return;
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
if(free_slot == 0){
free_slot = this.entries.length;
let entry = new CacheEntry();
entry.instance = new CodecWrapperWorker(this.type);
this.entries.push(entry);
2018-04-18 18:12:10 +00:00
}
2020-03-30 11:44:18 +00:00
this.entries[free_slot].owner = clientId;
this.entries[free_slot].last_access = new Date().getTime();
this.entries[free_slot].instance.on_encoded_data = callback_encoded;
if(this.entries[free_slot].instance.initialized())
this.entries[free_slot].instance.reset();
else {
this.ownCodec(clientId, callback_encoded, false).then(resolve).catch(reject);
return;
}
resolve(this.entries[free_slot].instance);
});
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
releaseCodec(clientId: number) {
for(let index = 0; index < this.entries.length; index++)
if(this.entries[index].owner == clientId) this.entries[index].owner = 0;
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
constructor(index: number, name: string, type: CodecType){
this.codecIndex = index;
this.name = name;
this.type = type;
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
this._supported = this.type !== undefined && codec_supported(this.type);
}
}
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
export enum VoiceEncodeType {
JS_ENCODE,
NATIVE_ENCODE
}
2019-10-13 19:33:07 +00:00
2020-03-30 11:44:18 +00:00
export class VoiceConnection extends AbstractVoiceConnection {
readonly connection: ServerConnection;
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
rtcPeerConnection: RTCPeerConnection;
dataChannel: RTCDataChannel;
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private _type: VoiceEncodeType = VoiceEncodeType.NATIVE_ENCODE;
2018-04-18 18:12:10 +00:00
2020-03-30 11:44:18 +00:00
/*
* To ensure we're not sending any audio because the settings activates the input,
* we self mute the audio stream
*/
local_audio_mute: GainNode;
local_audio_stream: MediaStreamAudioDestinationNode;
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
static codec_pool: codec.CodecPool[];
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
static codecSupported(type: number) : boolean {
return this.codec_pool && this.codec_pool.length > type && this.codec_pool[type].supported();
}
2019-09-12 21:59:35 +00:00
2020-03-30 11:44:18 +00:00
private voice_packet_id: number = 0;
private chunkVPacketId: number = 0;
private send_task: NodeJS.Timer;
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private _audio_source: RecorderProfile;
private _audio_clients: VoiceClientController[] = [];
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private _encoder_codec: number = 5;
2019-08-21 08:00:01 +00:00
2020-03-30 11:44:18 +00:00
constructor(connection: ServerConnection) {
super(connection);
this.connection = connection;
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
this._type = settings.static_global("voice_connection_type", this._type);
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
destroy() {
clearInterval(this.send_task);
this.drop_rtp_session();
this.acquire_voice_recorder(undefined, true).catch(error => {
log.warn(LogCategory.VOICE, tr("Failed to release voice recorder: %o"), error);
}).then(() => {
for(const client of this._audio_clients) {
client.abort_replay();
client.callback_playback = undefined;
client.callback_state_changed = undefined;
client.callback_stopped = undefined;
}
this._audio_clients = undefined;
this._audio_source = undefined;
});
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
static native_encoding_supported() : boolean {
const context = window.webkitAudioContext || window.AudioContext;
if(!context)
return false;
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
if(!context.prototype.createMediaStreamDestination)
return false; //Required, but not available within edge
2020-03-30 11:44:18 +00:00
return true;
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
static javascript_encoding_supported() : boolean {
if(!window.RTCPeerConnection)
return false;
if(!RTCPeerConnection.prototype.createDataChannel)
return false;
return true;
}
2018-09-21 21:25:03 +00:00
2020-03-30 11:44:18 +00:00
current_encoding_supported() : boolean {
switch (this._type) {
case VoiceEncodeType.JS_ENCODE:
return VoiceConnection.javascript_encoding_supported();
case VoiceEncodeType.NATIVE_ENCODE:
return VoiceConnection.native_encoding_supported();
}
return false;
}
2020-03-30 11:44:18 +00:00
private setup_native() {
log.info(LogCategory.VOICE, tr("Setting up native voice stream!"));
if(!VoiceConnection.native_encoding_supported()) {
log.warn(LogCategory.VOICE, tr("Native codec isn't supported!"));
return;
}
2018-04-16 18:38:35 +00:00
2020-03-30 11:44:18 +00:00
if(!this.local_audio_stream) {
this.local_audio_stream = aplayer.context().createMediaStreamDestination();
}
if(!this.local_audio_mute) {
this.local_audio_mute = aplayer.context().createGain();
this.local_audio_mute.connect(this.local_audio_stream);
this.local_audio_mute.gain.value = 1;
}
}
2018-09-23 13:24:07 +00:00
2020-03-30 11:44:18 +00:00
private setup_js() {
if(!VoiceConnection.javascript_encoding_supported()) return;
if(!this.send_task)
this.send_task = setInterval(this.send_next_voice_packet.bind(this), 20); /* send all 20ms out voice packets */
}
2018-09-23 13:24:07 +00:00
2020-03-30 11:44:18 +00:00
async acquire_voice_recorder(recorder: RecorderProfile | undefined, enforce?: boolean) {
if(this._audio_source === recorder && !enforce)
return;
2018-09-25 15:39:38 +00:00
2020-03-30 11:44:18 +00:00
if(recorder)
await recorder.unmount();
2018-09-23 13:24:07 +00:00
2020-03-30 11:44:18 +00:00
if(this._audio_source)
await this._audio_source.unmount();
2018-09-23 13:24:07 +00:00
2020-03-30 11:44:18 +00:00
this.handle_local_voice_ended();
this._audio_source = recorder;
2018-09-23 13:24:07 +00:00
2020-03-30 11:44:18 +00:00
if(recorder) {
recorder.current_handler = this.connection.client;
recorder.callback_unmount = this.on_recorder_yield.bind(this);
recorder.callback_start = this.handle_local_voice_started.bind(this);
recorder.callback_stop = this.handle_local_voice_ended.bind(this);
2018-08-08 17:32:12 +00:00
2020-04-03 13:59:32 +00:00
recorder.callback_input_change = async (old_input, new_input) => {
if(old_input) {
try {
await old_input.set_consumer(undefined);
} catch(error) {
log.warn(LogCategory.VOICE, tr("Failed to release own consumer from old input: %o"), error);
2019-04-04 19:47:52 +00:00
}
2020-04-03 13:59:32 +00:00
}
if(new_input) {
if(this._type == VoiceEncodeType.NATIVE_ENCODE) {
if(!this.local_audio_stream)
this.setup_native(); /* requires initialized audio */
try {
await new_input.set_consumer({
type: InputConsumerType.NODE,
callback_node: node => {
if(!this.local_audio_stream || !this.local_audio_mute)
return;
node.connect(this.local_audio_mute);
},
callback_disconnect: node => {
if(!this.local_audio_mute)
return;
node.disconnect(this.local_audio_mute);
}
} as NodeInputConsumer);
log.debug(LogCategory.VOICE, tr("Successfully set/updated to the new input for the recorder"));
} catch (e) {
log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e);
}
} else {
//TODO: Error handling?
await recorder.input.set_consumer({
type: InputConsumerType.CALLBACK,
callback_audio: buffer => this.handle_local_voice(buffer, false)
} as CallbackInputConsumer);
}
}
};
2020-03-30 11:44:18 +00:00
}
this.connection.client.update_voice_status(undefined);
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
get_encoder_type() : VoiceEncodeType { return this._type; }
set_encoder_type(target: VoiceEncodeType) {
if(target == this._type) return;
this._type = target;
2018-06-19 18:31:05 +00:00
2020-03-30 11:44:18 +00:00
if(this._type == VoiceEncodeType.NATIVE_ENCODE)
this.setup_native();
else
this.setup_js();
this.start_rtc_session();
}
2018-05-07 09:51:50 +00:00
2020-03-30 11:44:18 +00:00
voice_playback_support() : boolean {
return this.dataChannel && this.dataChannel.readyState == "open";
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
voice_send_support() : boolean {
if(this._type == VoiceEncodeType.NATIVE_ENCODE)
return VoiceConnection.native_encoding_supported() && this.rtcPeerConnection.getLocalStreams().length > 0;
else
return this.voice_playback_support();
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
private voice_send_queue: {data: Uint8Array, codec: number}[] = [];
handleEncodedVoicePacket(data: Uint8Array, codec: number){
this.voice_send_queue.push({data: data, codec: codec});
}
2018-09-25 15:39:38 +00:00
2020-03-30 11:44:18 +00:00
private send_next_voice_packet() {
const buffer = this.voice_send_queue.pop_front();
if(!buffer)
return;
this.send_voice_packet(buffer.data, buffer.codec);
}
2018-08-08 17:32:12 +00:00
2020-03-30 11:44:18 +00:00
send_voice_packet(encoded_data: Uint8Array, codec: number) {
if(this.dataChannel) {
this.voice_packet_id++;
if(this.voice_packet_id > 65535)
this.voice_packet_id = 0;
let packet = new Uint8Array(encoded_data.byteLength + 5);
packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header
packet[1] = 0; //Flag fragmented
packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID)
packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW (voiceID)
packet[4] = codec; //Codec
packet.set(encoded_data, 5);
try {
this.dataChannel.send(packet);
} catch (error) {
log.warn(LogCategory.VOICE, tr("Failed to send voice packet. Error: %o"), error);
}
} else {
log.warn(LogCategory.VOICE, tr("Could not transfer audio (not connected)"));
}
}
2018-09-21 21:25:03 +00:00
2020-03-30 11:44:18 +00:00
private _audio_player_waiting = false;
start_rtc_session() {
if(!aplayer.initialized()) {
log.info(LogCategory.VOICE, tr("Audio player isn't initialized yet. Waiting for gesture."));
if(!this._audio_player_waiting) {
this._audio_player_waiting = true;
aplayer.on_ready(() => this.start_rtc_session());
}
return;
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
if(!this.current_encoding_supported())
return false;
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
if(this._type == VoiceEncodeType.NATIVE_ENCODE)
this.setup_native();
else
this.setup_js();
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
this.drop_rtp_session();
this._ice_use_cache = true;
2018-09-21 21:25:03 +00:00
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
let config: RTCConfiguration = {};
config.iceServers = [];
config.iceServers.push({ urls: 'stun:stun.l.google.com:19302' });
//config.iceServers.push({ urls: "stun:stun.teaspeak.de:3478" });
2020-03-30 11:44:18 +00:00
this.rtcPeerConnection = new RTCPeerConnection(config);
const dataChannelConfig = { ordered: false, maxRetransmits: 0 };
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
this.dataChannel = this.rtcPeerConnection.createDataChannel('main', dataChannelConfig);
this.dataChannel.onmessage = this.on_data_channel_message.bind(this);
this.dataChannel.onopen = this.on_data_channel.bind(this);
this.dataChannel.binaryType = "arraybuffer";
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
let sdpConstraints : RTCOfferOptions = {};
sdpConstraints.offerToReceiveAudio = this._type == VoiceEncodeType.NATIVE_ENCODE;
sdpConstraints.offerToReceiveVideo = false;
sdpConstraints.voiceActivityDetection = true;
this.rtcPeerConnection.onicegatheringstatechange = () => console.log("ICE gathering state changed to %s", this.rtcPeerConnection.iceGatheringState);
this.rtcPeerConnection.oniceconnectionstatechange = () => console.log("ICE connection state changed to %s", this.rtcPeerConnection.iceConnectionState);
2020-03-30 11:44:18 +00:00
this.rtcPeerConnection.onicecandidate = this.on_local_ice_candidate.bind(this);
if(this.local_audio_stream) { //May a typecheck?
this.rtcPeerConnection.addStream(this.local_audio_stream.stream);
log.info(LogCategory.VOICE, tr("Adding native audio stream (%o)!"), this.local_audio_stream.stream);
}
this.rtcPeerConnection.createOffer(sdpConstraints)
.then(offer => this.on_local_offer_created(offer))
.catch(error => {
log.error(LogCategory.VOICE, tr("Could not create ice offer! error: %o"), error);
});
}
drop_rtp_session() {
if(this.dataChannel) {
this.dataChannel.close();
this.dataChannel = undefined;
}
if(this.rtcPeerConnection) {
this.rtcPeerConnection.close();
this.rtcPeerConnection = undefined;
}
this._ice_use_cache = true;
this._ice_cache = [];
this.connection.client.update_voice_status(undefined);
}
private registerRemoteICECandidate(candidate: RTCIceCandidate) {
if(candidate.candidate === "") {
console.log("Adding end candidate");
this.rtcPeerConnection.addIceCandidate(null).catch(error => {
log.info(LogCategory.VOICE, tr("Failed to add remote cached ice candidate finish: %o"), error);
});
return;
}
const pcandidate = new RTCIceCandidate(candidate);
if(pcandidate.protocol !== "tcp") return; /* UDP does not work currently */
log.info(LogCategory.VOICE, tr("Add remote ice! (%o)"), pcandidate);
this.rtcPeerConnection.addIceCandidate(pcandidate).catch(error => {
log.info(LogCategory.VOICE, tr("Failed to add remote cached ice candidate %o: %o"), candidate, error);
});
}
2020-03-30 11:44:18 +00:00
private _ice_use_cache: boolean = true;
private _ice_cache: RTCIceCandidate[] = [];
2020-03-30 11:44:18 +00:00
handleControlPacket(json) {
if(json["request"] === "answer") {
const session_description = new RTCSessionDescription(json["msg"]);
log.info(LogCategory.VOICE, tr("Received answer to our offer. Answer: %o"), session_description);
this.rtcPeerConnection.setRemoteDescription(session_description).then(() => {
log.info(LogCategory.VOICE, tr("Answer applied successfully. Applying ICE candidates (%d)."), this._ice_cache.length);
this._ice_use_cache = false;
for(let candidate of this._ice_cache)
this.registerRemoteICECandidate(candidate);
2020-03-30 11:44:18 +00:00
this._ice_cache = [];
}).catch(error => {
log.info(LogCategory.VOICE, tr("Failed to apply remote description: %o"), error); //FIXME error handling!
});
} else if(json["request"] === "ice" || json["request"] === "ice_finish") {
const candidate = new RTCIceCandidate(json["msg"]);
2020-03-30 11:44:18 +00:00
if(!this._ice_use_cache) {
this.registerRemoteICECandidate(candidate);
2020-03-30 11:44:18 +00:00
} else {
log.info(LogCategory.VOICE, tr("Cache remote ice! (%o)"), json["msg"]);
this._ice_cache.push(candidate);
}
2020-03-30 11:44:18 +00:00
} else if(json["request"] == "status") {
if(json["state"] == "failed") {
const chandler = this.connection.client;
chandler.log.log(elog.Type.CONNECTION_VOICE_SETUP_FAILED, {
reason: json["reason"],
reconnect_delay: json["allow_reconnect"] ? 1 : 0
});
log.error(LogCategory.NETWORKING, tr("Failed to setup voice bridge (%s). Allow reconnect: %s"), json["reason"], json["allow_reconnect"]);
if(json["allow_reconnect"] == true) {
this.start_rtc_session();
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
//TODO handle fail specially when its not allowed to reconnect
2019-04-04 19:47:52 +00:00
}
} else {
log.warn(LogCategory.NETWORKING, tr("Received unknown web client control packet: %s"), json["request"]);
2020-03-30 11:44:18 +00:00
}
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private on_local_ice_candidate(event: RTCPeerConnectionIceEvent) {
if (event) {
if(event.candidate && event.candidate.protocol !== "tcp")
return;
2020-03-30 11:44:18 +00:00
if(event.candidate) {
log.info(LogCategory.VOICE, tr("Gathered local ice candidate for stream %d: %s"), event.candidate.sdpMLineIndex, event.candidate.candidate);
2020-03-30 11:44:18 +00:00
this.connection.sendData(JSON.stringify({
type: 'WebRTC',
request: "ice",
msg: event.candidate,
}));
} else {
log.info(LogCategory.VOICE, tr("Local ICE candidate gathering finish."));
2020-03-30 11:44:18 +00:00
this.connection.sendData(JSON.stringify({
type: 'WebRTC',
request: "ice_finish"
}));
2019-04-04 19:47:52 +00:00
}
2020-03-30 11:44:18 +00:00
}
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private on_local_offer_created(localSession) {
log.info(LogCategory.VOICE, tr("Local offer created. Setting up local description. (%o)"), localSession);
this.rtcPeerConnection.setLocalDescription(localSession).then(() => {
log.info(LogCategory.VOICE, tr("Offer applied successfully. Sending offer to server."));
this.connection.sendData(JSON.stringify({type: 'WebRTC', request: "create", msg: localSession}));
}).catch(error => {
log.info(LogCategory.VOICE, tr("Failed to apply local description: %o"), error);
//FIXME error handling
});
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private on_data_channel(channel) {
log.info(LogCategory.VOICE, tr("Got new data channel! (%s)"), this.dataChannel.readyState);
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
this.connection.client.update_voice_status();
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
private on_data_channel_message(message: MessageEvent) {
const chandler = this.connection.client;
if(chandler.client_status.output_muted) /* we dont need to do anything with sound playback when we're not listening to it */
return;
let bin = new Uint8Array(message.data);
let clientId = bin[2] << 8 | bin[3];
let packetId = bin[0] << 8 | bin[1];
let codec = bin[4];
//log.info(LogCategory.VOICE, "Client id " + clientId + " PacketID " + packetId + " Codec: " + codec);
let client = this.find_client(clientId);
if(!client) {
log.error(LogCategory.VOICE, tr("Having voice from unknown audio client? (ClientID: %o)"), clientId);
return;
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
let codec_pool = VoiceConnection.codec_pool[codec];
if(!codec_pool) {
log.error(LogCategory.VOICE, tr("Could not playback codec %o"), codec);
return;
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
let encodedData;
if(message.data.subarray)
encodedData = message.data.subarray(5);
else encodedData = new Uint8Array(message.data, 5);
if(encodedData.length == 0) {
client.stopAudio();
codec_pool.releaseCodec(clientId);
} else {
codec_pool.ownCodec(clientId, e => this.handleEncodedVoicePacket(e, codec), true)
.then(decoder => decoder.decodeSamples(client.get_codec_cache(codec), encodedData))
.then(buffer => client.playback_buffer(buffer)).catch(error => {
log.error(LogCategory.VOICE, tr("Could not playback client's (%o) audio (%o)"), clientId, error);
if(error instanceof Error)
log.error(LogCategory.VOICE, error.stack);
});
}
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
private handle_local_voice(data: AudioBuffer, head: boolean) {
const chandler = this.connection.client;
if(!chandler.connected)
return false;
2019-10-13 19:33:07 +00:00
2020-03-30 11:44:18 +00:00
if(chandler.client_status.input_muted)
return false;
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
if(head)
this.chunkVPacketId = 0;
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
let client = this.find_client(chandler.clientId);
if(!client) {
log.error(LogCategory.VOICE, tr("Tried to send voice data, but local client hasn't a voice client handle"));
return;
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
const codec = this._encoder_codec;
VoiceConnection.codec_pool[codec]
.ownCodec(chandler.getClientId(), e => this.handleEncodedVoicePacket(e, codec), true)
.then(encoder => encoder.encodeSamples(client.get_codec_cache(codec), data));
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
private handle_local_voice_ended() {
const chandler = this.connection.client;
const ch = chandler.getClient();
if(ch) ch.speaking = false;
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
if(!chandler.connected)
return false;
if(chandler.client_status.input_muted)
return false;
log.info(LogCategory.VOICE, tr("Local voice ended"));
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
if(this.dataChannel && this._encoder_codec >= 0)
this.send_voice_packet(new Uint8Array(0), this._encoder_codec);
}
2020-03-30 11:44:18 +00:00
private handle_local_voice_started() {
const chandler = this.connection.client;
if(chandler.client_status.input_muted) {
/* evail hack due to the settings :D */
log.warn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted! Do not send any voice."));
if(this.local_audio_mute)
this.local_audio_mute.gain.value = 0;
return;
}
if(this.local_audio_mute)
this.local_audio_mute.gain.value = 1;
log.info(LogCategory.VOICE, tr("Local voice started"));
2019-08-21 08:00:01 +00:00
2020-03-30 11:44:18 +00:00
const ch = chandler.getClient();
if(ch) ch.speaking = true;
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
private on_recorder_yield() {
log.info(LogCategory.VOICE, "Lost recorder!");
this._audio_source = undefined;
this.acquire_voice_recorder(undefined, true); /* we can ignore the promise because we should finish this directly */
}
2018-04-18 18:12:10 +00:00
2020-03-30 11:44:18 +00:00
connected(): boolean {
return typeof(this.dataChannel) !== "undefined" && this.dataChannel.readyState === "open";
}
2018-03-07 18:06:52 +00:00
2020-03-30 11:44:18 +00:00
voice_recorder(): RecorderProfile {
return this._audio_source;
}
2018-09-22 13:14:39 +00:00
2020-03-30 11:44:18 +00:00
available_clients(): VoiceClient[] {
return this._audio_clients;
}
2018-04-11 15:56:09 +00:00
2020-03-30 11:44:18 +00:00
find_client(client_id: number) : VoiceClientController | undefined {
for(const client of this._audio_clients)
if(client.client_id === client_id)
return client;
return undefined;
}
2019-03-07 14:30:53 +00:00
2020-03-30 11:44:18 +00:00
unregister_client(client: VoiceClient): Promise<void> {
if(!(client instanceof VoiceClientController))
throw "Invalid client type";
2018-09-22 13:14:39 +00:00
2020-03-30 11:44:18 +00:00
this._audio_clients.remove(client);
return Promise.resolve();
}
2018-09-22 13:14:39 +00:00
2020-03-30 11:44:18 +00:00
register_client(client_id: number): VoiceClient {
const client = new VoiceClientController(client_id);
this._audio_clients.push(client);
return client;
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
decoding_supported(codec: number): boolean {
return VoiceConnection.codecSupported(codec);
}
2019-04-04 19:47:52 +00:00
2020-03-30 11:44:18 +00:00
encoding_supported(codec: number): boolean {
return VoiceConnection.codecSupported(codec);
}
2019-09-12 21:59:35 +00:00
2020-03-30 11:44:18 +00:00
get_encoder_codec(): number {
return this._encoder_codec;
}
2019-09-12 21:59:35 +00:00
2020-03-30 11:44:18 +00:00
set_encoder_codec(codec: number) {
this._encoder_codec = codec;
2018-09-22 13:14:39 +00:00
}
2019-04-04 19:47:52 +00:00
}
/* funny fact that typescript dosn't find this */
2020-03-30 11:44:18 +00:00
declare global {
interface RTCPeerConnection {
addStream(stream: MediaStream): void;
getLocalStreams(): MediaStream[];
getStreamById(streamId: string): MediaStream | null;
removeStream(stream: MediaStream): void;
createOffer(successCallback?: RTCSessionDescriptionCallback, failureCallback?: RTCPeerConnectionErrorCallback, options?: RTCOfferOptions): Promise<RTCSessionDescription>;
}
2019-04-04 19:47:52 +00:00
}
loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, {
priority: 10,
function: async () => {
2020-03-30 11:44:18 +00:00
aplayer.on_ready(() => {
2019-04-04 19:47:52 +00:00
log.info(LogCategory.VOICE, tr("Initializing voice handler after AudioController has been initialized!"));
2020-03-30 11:44:18 +00:00
VoiceConnection.codec_pool = [
new codec.CodecPool(0, tr("Speex Narrowband"), CodecType.SPEEX_NARROWBAND),
new codec.CodecPool(1, tr("Speex Wideband"), CodecType.SPEEX_WIDEBAND),
new codec.CodecPool(2, tr("Speex Ultra Wideband"), CodecType.SPEEX_ULTRA_WIDEBAND),
new codec.CodecPool(3, tr("CELT Mono"), CodecType.CELT_MONO),
new codec.CodecPool(4, tr("Opus Voice"), CodecType.OPUS_VOICE),
new codec.CodecPool(5, tr("Opus Music"), CodecType.OPUS_MUSIC)
2019-04-04 19:47:52 +00:00
];
2020-03-30 11:44:18 +00:00
VoiceConnection.codec_pool[4].initialize(2);
VoiceConnection.codec_pool[5].initialize(2);
2019-04-04 19:47:52 +00:00
});
},
name: "registering codec initialisation"
});