From d3df4f09765ea824ff6792da49edbc75e92de382 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 12 Jun 2020 19:33:05 +0200 Subject: [PATCH] Fixed script node decoding --- ChangeLog.md | 1 + shared/js/ConnectionHandler.ts | 11 ++- shared/js/connection/CommandHandler.ts | 6 +- shared/js/ui/frames/chat_frame.ts | 2 +- web/js/audio/recorder.ts | 1 + web/js/codec/BasicCodec.ts | 7 +- web/js/codec/Codec.ts | 1 - web/js/voice/AudioResampler.ts | 4 +- web/js/voice/VoiceHandler.ts | 96 +++++++++++++++++--------- web/js/workers/codec/OpusCodec.ts | 30 ++++---- web/native-codec/src/opus.cpp | 2 +- 11 files changed, 95 insertions(+), 66 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index c64e2daf..d581173a 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -2,6 +2,7 @@ * **12.06.20** - Added a copy/paste menu for all HTML input elements - Heavily improved web client audio de/encoding + - Fixed script node voice encoding * **11.06.20** - Fixed channel tree deletions diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts index 0126f262..6d938033 100644 --- a/shared/js/ConnectionHandler.ts +++ b/shared/js/ConnectionHandler.ts @@ -337,13 +337,12 @@ export class ConnectionHandler { getClient() : LocalClientEntry { return this._local_client; } getClientId() { return this._clientId; } - set clientId(id: number) { - this._clientId = id; - this._local_client["_clientId"] = id; - } + initializeLocalClient(clientId: number, acceptedName: string) { + this._clientId = clientId; + this._local_client["_clientId"] = clientId; - get clientId() { - return this._clientId; + this.channelTree.registerClient(this._local_client); + this._local_client.updateVariables( { key: "client_nickname", value: acceptedName }); } getServerConnection() : AbstractServerConnection { return this.serverConnection; } diff --git a/shared/js/connection/CommandHandler.ts b/shared/js/connection/CommandHandler.ts index c8dc004c..417055d4 100644 --- a/shared/js/connection/CommandHandler.ts +++ b/shared/js/connection/CommandHandler.ts @@ -191,10 +191,8 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { json = json[0]; //Only one bulk - this.connection_handler.channelTree.registerClient(this.connection_handler.getClient()); this.connection.client.side_bar.channel_conversations().reset(); - this.connection.client.clientId = parseInt(json["aclid"]); - this.connection.client.getClient().updateVariables( {key: "client_nickname", value: json["acn"]}); + this.connection.client.initializeLocalClient(parseInt(json["aclid"]), json["acn"]); let updates: { key: string, @@ -825,7 +823,7 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { const channel_id = typeof(json["cid"]) !== "undefined" ? parseInt(json["cid"]) : own_channel_id; const channel = this.connection_handler.channelTree.findChannel(channel_id) || this.connection_handler.getClient().currentChannel(); - if(json["invokerid"] == this.connection.client.clientId) + if(json["invokerid"] == this.connection.client.getClientId()) this.connection_handler.sound.play(Sound.MESSAGE_SEND, {default_volume: .5}); else if(channel_id == own_channel_id) { this.connection_handler.sound.play(Sound.MESSAGE_RECEIVED, {default_volume: .5}); diff --git a/shared/js/ui/frames/chat_frame.ts b/shared/js/ui/frames/chat_frame.ts index a584719a..7ba3fc39 100644 --- a/shared/js/ui/frames/chat_frame.ts +++ b/shared/js/ui/frames/chat_frame.ts @@ -251,7 +251,7 @@ export class InfoFrame { client_id: selected_client.clientId() }, { create: false, attach: false }) : undefined; - const visibility = (selected_client && selected_client.clientId() !== this.handle.handle.clientId) ? "visible" : "hidden"; + const visibility = (selected_client && selected_client.clientId() !== this.handle.handle.getClientId()) ? "visible" : "hidden"; if(this._button_conversation.style.visibility !== visibility) this._button_conversation.style.visibility = visibility; if(conversation) { diff --git a/web/js/audio/recorder.ts b/web/js/audio/recorder.ts index 909dabfe..66375a9a 100644 --- a/web/js/audio/recorder.ts +++ b/web/js/audio/recorder.ts @@ -407,6 +407,7 @@ class JavascriptInput implements AbstractInput { const callback = this._current_consumer as CallbackInputConsumer; if(callback.callback_audio) callback.callback_audio(event.inputBuffer); + if(callback.callback_buffer) { log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!")); } diff --git a/web/js/codec/BasicCodec.ts b/web/js/codec/BasicCodec.ts index 6c772a0c..cc04d330 100644 --- a/web/js/codec/BasicCodec.ts +++ b/web/js/codec/BasicCodec.ts @@ -53,9 +53,10 @@ export abstract class BasicCodec implements Codec { encodeSamples(cache: CodecClientCache, pcm: AudioBuffer) { - this._encodeResampler.resample(pcm).catch(error => log.error(LogCategory.VOICE, tr("Could not resample PCM data for codec. Error: %o"), error)) - .then(buffer => this.encodeSamples0(cache, buffer as any)).catch(error => console.error(tr("Could not encode PCM data for codec. Error: %o"), error)) - + this._encodeResampler.resample(pcm) + .catch(error => log.error(LogCategory.VOICE, tr("Could not resample PCM data for codec. Error: %o"), error)) + .then(buffer => this.encodeSamples0(cache, buffer as any)) + .catch(error => console.error(tr("Could not encode PCM data for codec. Error: %o"), error)) } private encodeSamples0(cache: CodecClientCache, buffer: AudioBuffer) { diff --git a/web/js/codec/Codec.ts b/web/js/codec/Codec.ts index 9b74ae52..81ef7c86 100644 --- a/web/js/codec/Codec.ts +++ b/web/js/codec/Codec.ts @@ -35,7 +35,6 @@ export class BufferChunk { } export class CodecClientCache { - _last_access: number; _chunks: BufferChunk[] = []; bufferedSamples(max: number = 0) : number { diff --git a/web/js/voice/AudioResampler.ts b/web/js/voice/AudioResampler.ts index 5d833300..207d0203 100644 --- a/web/js/voice/AudioResampler.ts +++ b/web/js/voice/AudioResampler.ts @@ -2,7 +2,7 @@ import {LogCategory} from "tc-shared/log"; import * as log from "tc-shared/log"; export class AudioResampler { - targetSampleRate: number; + readonly targetSampleRate: number; private _use_promise: boolean; constructor(targetSampleRate: number){ @@ -15,7 +15,7 @@ export class AudioResampler { log.warn(LogCategory.AUDIO, tr("Received empty buffer as input! Returning empty output!")); return Promise.resolve(buffer); } - //console.log("Encode from %i to %i", buffer.sampleRate, this.targetSampleRate); + if(buffer.sampleRate == this.targetSampleRate) return Promise.resolve(buffer); diff --git a/web/js/voice/VoiceHandler.ts b/web/js/voice/VoiceHandler.ts index 0eef79c4..e06c9d67 100644 --- a/web/js/voice/VoiceHandler.ts +++ b/web/js/voice/VoiceHandler.ts @@ -137,6 +137,7 @@ export class VoiceConnection extends AbstractVoiceConnection { private _type: VoiceEncodeType = VoiceEncodeType.NATIVE_ENCODE; + private localAudioStarted = false; /* * To ensure we're not sending any audio because the settings activates the input, * we self mute the audio stream @@ -245,15 +246,15 @@ export class VoiceConnection extends AbstractVoiceConnection { if(this._audio_source) await this._audio_source.unmount(); - this.handle_local_voice_ended(); + this.handleLocalVoiceEnded(); this._audio_source = recorder; if(recorder) { recorder.current_handler = this.connection.client; recorder.callback_unmount = this.on_recorder_yield.bind(this); - recorder.callback_start = this.handle_local_voice_started.bind(this); - recorder.callback_stop = this.handle_local_voice_ended.bind(this); + recorder.callback_start = this.handleLocalVoiceStarted.bind(this); + recorder.callback_stop = this.handleLocalVoiceEnded.bind(this); recorder.callback_input_change = async (old_input, new_input) => { if(old_input) { @@ -289,11 +290,16 @@ export class VoiceConnection extends AbstractVoiceConnection { log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e); } } else { - //TODO: Error handling? - await recorder.input.set_consumer({ - type: InputConsumerType.CALLBACK, - callback_audio: buffer => this.handle_local_voice(buffer, false) - } as CallbackInputConsumer); + try { + await recorder.input.set_consumer({ + type: InputConsumerType.CALLBACK, + callback_audio: buffer => this.handleLocalVoiceBuffer(buffer, false) + } as CallbackInputConsumer); + + log.debug(LogCategory.VOICE, tr("Successfully set/updated to the new input for the recorder")); + } catch (e) { + log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e); + } } } }; @@ -333,22 +339,27 @@ export class VoiceConnection extends AbstractVoiceConnection { const buffer = this.voice_send_queue.pop_front(); if(!buffer) return; - this.send_voice_packet(buffer.data, buffer.codec); + this.sendVoicePacket(buffer.data, buffer.codec); } - send_voice_packet(encoded_data: Uint8Array, codec: number) { + private fillVoicePacketHeader(packet: Uint8Array, codec: number) { + packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header + packet[1] = 0; //Flag fragmented + packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID) + packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW (voiceID) + packet[4] = codec; //Codec + } + + sendVoicePacket(encoded_data: Uint8Array, codec: number) { if(this.dataChannel) { this.voice_packet_id++; if(this.voice_packet_id > 65535) this.voice_packet_id = 0; let packet = new Uint8Array(encoded_data.byteLength + 5); - packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header - packet[1] = 0; //Flag fragmented - packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID) - packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW (voiceID) - packet[4] = codec; //Codec + this.fillVoicePacketHeader(packet, codec); packet.set(encoded_data, 5); + try { this.dataChannel.send(packet); } catch (error) { @@ -359,6 +370,20 @@ export class VoiceConnection extends AbstractVoiceConnection { } } + sendVoiceStopPacket(codec: number) { + if(!this.dataChannel) + return; + + const packet = new Uint8Array(5); + this.fillVoicePacketHeader(packet, codec); + + try { + this.dataChannel.send(packet); + } catch (error) { + log.warn(LogCategory.VOICE, tr("Failed to send voice packet. Error: %o"), error); + } + } + private _audio_player_waiting = false; start_rtc_session() { if(!aplayer.initialized()) { @@ -390,8 +415,8 @@ export class VoiceConnection extends AbstractVoiceConnection { const dataChannelConfig = { ordered: false, maxRetransmits: 0 }; this.dataChannel = this.rtcPeerConnection.createDataChannel('main', dataChannelConfig); - this.dataChannel.onmessage = this.on_data_channel_message.bind(this); - this.dataChannel.onopen = this.on_data_channel.bind(this); + this.dataChannel.onmessage = this.onMainDataChannelMessage.bind(this); + this.dataChannel.onopen = this.onMainDataChannelOpen.bind(this); this.dataChannel.binaryType = "arraybuffer"; let sdpConstraints : RTCOfferOptions = {}; @@ -524,15 +549,15 @@ export class VoiceConnection extends AbstractVoiceConnection { }); } - private on_data_channel(channel) { + private onMainDataChannelOpen(channel) { log.info(LogCategory.VOICE, tr("Got new data channel! (%s)"), this.dataChannel.readyState); this.connection.client.update_voice_status(); } - private on_data_channel_message(message: MessageEvent) { + private onMainDataChannelMessage(message: MessageEvent) { const chandler = this.connection.client; - if(chandler.client_status.output_muted) /* we dont need to do anything with sound playback when we're not listening to it */ + if(chandler.isSpeakerMuted() || chandler.isSpeakerDisabled()) /* we dont need to do anything with sound playback when we're not listening to it */ return; let bin = new Uint8Array(message.data); @@ -571,18 +596,18 @@ export class VoiceConnection extends AbstractVoiceConnection { } } - private handle_local_voice(data: AudioBuffer, head: boolean) { + private handleLocalVoiceBuffer(data: AudioBuffer, head: boolean) { const chandler = this.connection.client; - if(!chandler.connected) + if(!this.localAudioStarted || !chandler.connected) return false; - if(chandler.client_status.input_muted) + if(chandler.isMicrophoneMuted()) return false; if(head) this.chunkVPacketId = 0; - let client = this.find_client(chandler.clientId); + let client = this.find_client(chandler.getClientId()); if(!client) { log.error(LogCategory.VOICE, tr("Tried to send voice data, but local client hasn't a voice client handle")); return; @@ -594,25 +619,31 @@ export class VoiceConnection extends AbstractVoiceConnection { .then(encoder => encoder.encodeSamples(client.get_codec_cache(codec), data)); } - private handle_local_voice_ended() { + private handleLocalVoiceEnded() { const chandler = this.connection.client; const ch = chandler.getClient(); if(ch) ch.speaking = false; if(!chandler.connected) return false; - if(chandler.client_status.input_muted) + if(chandler.isMicrophoneMuted()) return false; log.info(LogCategory.VOICE, tr("Local voice ended")); + this.localAudioStarted = false; - if(this.dataChannel && this._encoder_codec >= 0) - this.send_voice_packet(new Uint8Array(0), this._encoder_codec); + if(this._type === VoiceEncodeType.NATIVE_ENCODE) { + setTimeout(() => { + /* first send all data, than send the stop signal */ + this.sendVoiceStopPacket(this._encoder_codec); + }, 150); + } else { + this.sendVoiceStopPacket(this._encoder_codec); + } } - private handle_local_voice_started() { + private handleLocalVoiceStarted() { const chandler = this.connection.client; - if(chandler.client_status.input_muted) { - /* evail hack due to the settings :D */ + if(chandler.isMicrophoneMuted()) { log.warn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted! Do not send any voice.")); if(this.local_audio_mute) this.local_audio_mute.gain.value = 0; @@ -620,6 +651,8 @@ export class VoiceConnection extends AbstractVoiceConnection { } if(this.local_audio_mute) this.local_audio_mute.gain.value = 1; + + this.localAudioStarted = true; log.info(LogCategory.VOICE, tr("Local voice started")); const ch = chandler.getClient(); @@ -683,7 +716,6 @@ export class VoiceConnection extends AbstractVoiceConnection { } - /* funny fact that typescript dosn't find this */ declare global { interface RTCPeerConnection { diff --git a/web/js/workers/codec/OpusCodec.ts b/web/js/workers/codec/OpusCodec.ts index f892ac3f..48c36cf6 100644 --- a/web/js/workers/codec/OpusCodec.ts +++ b/web/js/workers/codec/OpusCodec.ts @@ -121,6 +121,8 @@ const OPUS_ERROR_CODES = [ ]; class OpusWorker implements CodecWorker { + private static readonly kProcessBufferSize = 4096 * 2; + private readonly channelCount: number; private readonly type: OpusType; private nativeHandle: any; @@ -130,11 +132,8 @@ class OpusWorker implements CodecWorker { private fn_encode: any; private fn_reset: any; - private buffer_size = 4096 * 2; - private buffer: any; - - private encode_buffer: Float32Array; - private decode_buffer: Uint8Array; + private nativeBufferPtr: number; + private processBuffer: Uint8Array; constructor(channelCount: number, type: OpusType) { this.channelCount = channelCount; @@ -153,40 +152,39 @@ class OpusWorker implements CodecWorker { this.nativeHandle = this.fn_newHandle(this.channelCount, this.type); - this.buffer = Module._malloc(this.buffer_size); - this.encode_buffer = new Float32Array(Module.HEAPF32.buffer, this.buffer, Math.floor(this.buffer_size / 4)); - this.decode_buffer = new Uint8Array(Module.HEAPU8.buffer, this.buffer, this.buffer_size); + this.nativeBufferPtr = Module._malloc(OpusWorker.kProcessBufferSize); + this.processBuffer = new Uint8Array(Module.HEAPU8.buffer, this.nativeBufferPtr, OpusWorker.kProcessBufferSize); return undefined; } deinitialise() { } //TODO decode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string { - if (buffer.byteLength > this.decode_buffer.byteLength) + if (buffer.byteLength > this.processBuffer.byteLength) return "supplied data exceeds internal buffer"; - this.decode_buffer.set(buffer); + this.processBuffer.set(buffer); - let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, buffer.byteLength, this.decode_buffer.byteLength); + let result = this.fn_decode(this.nativeHandle, this.processBuffer.byteOffset, buffer.byteLength, this.processBuffer.byteLength); if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown decode error " + result; const resultByteLength = result * this.channelCount * 4; const resultBuffer = responseBuffer(resultByteLength); - resultBuffer.set(this.decode_buffer.subarray(0, resultByteLength), 0); + resultBuffer.set(this.processBuffer.subarray(0, resultByteLength), 0); return resultByteLength; } encode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string { - if (buffer.byteLength > this.decode_buffer.byteLength) + if (buffer.byteLength > this.processBuffer.byteLength) return "supplied data exceeds internal buffer"; - this.encode_buffer.set(buffer); + this.processBuffer.set(buffer); - let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, buffer.byteLength, this.encode_buffer.byteLength); + let result = this.fn_encode(this.nativeHandle, this.processBuffer.byteOffset, buffer.byteLength, this.processBuffer.byteLength); if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown encode error " + result; const resultBuffer = responseBuffer(result); - resultBuffer.set(Module.HEAP8.subarray(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result)); + resultBuffer.set(this.processBuffer.subarray(0, result), 0); return result; } diff --git a/web/native-codec/src/opus.cpp b/web/native-codec/src/opus.cpp index a3384f30..bd89f882 100644 --- a/web/native-codec/src/opus.cpp +++ b/web/native-codec/src/opus.cpp @@ -87,7 +87,7 @@ int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t byte_length, s if(handle->channelCount == 2) sequenced2interleaved_intersecting<2>((float *) buffer, byte_length / (sizeof(float) * 2)); - auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / handle->channelCount, buffer, maxLength); + auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / (handle->channelCount * sizeof(float)), buffer, maxLength); if (result < 0) return result; return result; }