Fixed script node decoding

ChangeLog.md

@@ -2,6 +2,7 @@
 * **12.06.20**
     - Added a copy/paste menu for all HTML input elements
     - Heavily improved web client audio de/encoding
+    - Fixed script node voice encoding
     
 * **11.06.20**
     - Fixed channel tree deletions

shared/js/ConnectionHandler.ts

@@ -337,13 +337,12 @@ export class ConnectionHandler {
     getClient() : LocalClientEntry { return this._local_client; }
     getClientId() { return this._clientId; }
 
-    set clientId(id: number) {
+    initializeLocalClient(clientId: number, acceptedName: string) {
-        this._clientId = id;
+        this._clientId = clientId;
-        this._local_client["_clientId"] = id;
+        this._local_client["_clientId"] = clientId;
-    }
 
-    get clientId() {
+        this.channelTree.registerClient(this._local_client);
-        return this._clientId;
+        this._local_client.updateVariables( { key: "client_nickname", value: acceptedName });
     }
 
     getServerConnection() : AbstractServerConnection { return this.serverConnection; }

shared/js/connection/CommandHandler.ts

@@ -191,10 +191,8 @@ export class ConnectionCommandHandler extends AbstractCommandHandler {
 
         json = json[0]; //Only one bulk
 
-        this.connection_handler.channelTree.registerClient(this.connection_handler.getClient());
         this.connection.client.side_bar.channel_conversations().reset();
-        this.connection.client.clientId = parseInt(json["aclid"]);
+        this.connection.client.initializeLocalClient(parseInt(json["aclid"]), json["acn"]);
-        this.connection.client.getClient().updateVariables( {key: "client_nickname", value: json["acn"]});
 
         let updates: {
             key: string,
@@ -825,7 +823,7 @@ export class ConnectionCommandHandler extends AbstractCommandHandler {
             const channel_id = typeof(json["cid"]) !== "undefined" ? parseInt(json["cid"]) : own_channel_id;
             const channel = this.connection_handler.channelTree.findChannel(channel_id) || this.connection_handler.getClient().currentChannel();
 
-            if(json["invokerid"] == this.connection.client.clientId)
+            if(json["invokerid"] == this.connection.client.getClientId())
                 this.connection_handler.sound.play(Sound.MESSAGE_SEND, {default_volume: .5});
             else if(channel_id == own_channel_id) {
                 this.connection_handler.sound.play(Sound.MESSAGE_RECEIVED, {default_volume: .5});

shared/js/ui/frames/chat_frame.ts

@@ -251,7 +251,7 @@ export class InfoFrame {
                 client_id: selected_client.clientId()
             }, { create: false, attach: false }) : undefined;
 
-            const visibility = (selected_client && selected_client.clientId() !== this.handle.handle.clientId) ? "visible" : "hidden";
+            const visibility = (selected_client && selected_client.clientId() !== this.handle.handle.getClientId()) ? "visible" : "hidden";
             if(this._button_conversation.style.visibility !== visibility)
                 this._button_conversation.style.visibility = visibility;
             if(conversation) {

web/js/audio/recorder.ts

@@ -407,6 +407,7 @@ class JavascriptInput implements AbstractInput {
         const callback = this._current_consumer as CallbackInputConsumer;
         if(callback.callback_audio)
             callback.callback_audio(event.inputBuffer);
+
         if(callback.callback_buffer) {
             log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!"));
         }

web/js/codec/BasicCodec.ts

@@ -53,9 +53,10 @@ export abstract class BasicCodec implements Codec {
 
 
     encodeSamples(cache: CodecClientCache, pcm: AudioBuffer) {
-        this._encodeResampler.resample(pcm).catch(error => log.error(LogCategory.VOICE, tr("Could not resample PCM data for codec. Error: %o"), error))
+        this._encodeResampler.resample(pcm)
-            .then(buffer => this.encodeSamples0(cache, buffer as any)).catch(error => console.error(tr("Could not encode PCM data for codec. Error: %o"), error))
+            .catch(error => log.error(LogCategory.VOICE, tr("Could not resample PCM data for codec. Error: %o"), error))
-
+            .then(buffer => this.encodeSamples0(cache, buffer as any))
+            .catch(error => console.error(tr("Could not encode PCM data for codec. Error: %o"), error))
     }
 
     private encodeSamples0(cache: CodecClientCache, buffer: AudioBuffer) {

web/js/codec/Codec.ts

@@ -35,7 +35,6 @@ export class BufferChunk {
 }
 
 export class CodecClientCache {
-    _last_access: number;
     _chunks: BufferChunk[] = [];
 
     bufferedSamples(max: number = 0) : number {

web/js/voice/AudioResampler.ts

@@ -2,7 +2,7 @@ import {LogCategory} from "tc-shared/log";
 import * as log from "tc-shared/log";
 
 export class AudioResampler {
-    targetSampleRate: number;
+    readonly targetSampleRate: number;
     private _use_promise: boolean;
 
     constructor(targetSampleRate: number){
@@ -15,7 +15,7 @@ export class AudioResampler {
             log.warn(LogCategory.AUDIO, tr("Received empty buffer as input! Returning empty output!"));
             return Promise.resolve(buffer);
         }
-        //console.log("Encode from %i to %i", buffer.sampleRate, this.targetSampleRate);
+
         if(buffer.sampleRate == this.targetSampleRate)
             return Promise.resolve(buffer);
 

web/js/voice/VoiceHandler.ts

@@ -137,6 +137,7 @@ export class VoiceConnection extends AbstractVoiceConnection {
 
     private _type: VoiceEncodeType = VoiceEncodeType.NATIVE_ENCODE;
 
+    private localAudioStarted = false;
     /*
      * To ensure we're not sending any audio because the settings activates the input,
      * we self mute the audio stream
@@ -245,15 +246,15 @@ export class VoiceConnection extends AbstractVoiceConnection {
         if(this._audio_source)
             await this._audio_source.unmount();
 
-        this.handle_local_voice_ended();
+        this.handleLocalVoiceEnded();
         this._audio_source = recorder;
 
         if(recorder) {
             recorder.current_handler = this.connection.client;
 
             recorder.callback_unmount = this.on_recorder_yield.bind(this);
-            recorder.callback_start = this.handle_local_voice_started.bind(this);
+            recorder.callback_start = this.handleLocalVoiceStarted.bind(this);
-            recorder.callback_stop = this.handle_local_voice_ended.bind(this);
+            recorder.callback_stop = this.handleLocalVoiceEnded.bind(this);
 
             recorder.callback_input_change = async (old_input, new_input) => {
                 if(old_input) {
@@ -289,11 +290,16 @@ export class VoiceConnection extends AbstractVoiceConnection {
                             log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e);
                         }
                     } else {
-                        //TODO: Error handling?
+                        try {
-                        await recorder.input.set_consumer({
+                            await recorder.input.set_consumer({
-                            type: InputConsumerType.CALLBACK,
+                                type: InputConsumerType.CALLBACK,
-                            callback_audio: buffer => this.handle_local_voice(buffer, false)
+                                callback_audio: buffer => this.handleLocalVoiceBuffer(buffer, false)
-                        } as CallbackInputConsumer);
+                            } as CallbackInputConsumer);
+
+                            log.debug(LogCategory.VOICE, tr("Successfully set/updated to the new input for the recorder"));
+                        } catch (e) {
+                            log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e);
+                        }
                     }
                 }
             };
@@ -333,22 +339,27 @@ export class VoiceConnection extends AbstractVoiceConnection {
         const buffer = this.voice_send_queue.pop_front();
         if(!buffer)
             return;
-        this.send_voice_packet(buffer.data, buffer.codec);
+        this.sendVoicePacket(buffer.data, buffer.codec);
     }
 
-    send_voice_packet(encoded_data: Uint8Array, codec: number) {
+    private fillVoicePacketHeader(packet: Uint8Array, codec: number) {
+        packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header
+        packet[1] = 0; //Flag fragmented
+        packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID)
+        packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW   (voiceID)
+        packet[4] = codec; //Codec
+    }
+
+    sendVoicePacket(encoded_data: Uint8Array, codec: number) {
         if(this.dataChannel) {
             this.voice_packet_id++;
             if(this.voice_packet_id > 65535)
                 this.voice_packet_id = 0;
 
             let packet = new Uint8Array(encoded_data.byteLength + 5);
-            packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header
+            this.fillVoicePacketHeader(packet, codec);
-            packet[1] = 0; //Flag fragmented
-            packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID)
-            packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW   (voiceID)
-            packet[4] = codec; //Codec
             packet.set(encoded_data, 5);
+
             try {
                 this.dataChannel.send(packet);
             } catch (error) {
@@ -359,6 +370,20 @@ export class VoiceConnection extends AbstractVoiceConnection {
         }
     }
 
+    sendVoiceStopPacket(codec: number) {
+        if(!this.dataChannel)
+            return;
+
+        const packet = new Uint8Array(5);
+        this.fillVoicePacketHeader(packet, codec);
+
+        try {
+            this.dataChannel.send(packet);
+        } catch (error) {
+            log.warn(LogCategory.VOICE, tr("Failed to send voice packet. Error: %o"), error);
+        }
+    }
+
     private _audio_player_waiting = false;
     start_rtc_session() {
         if(!aplayer.initialized()) {
@@ -390,8 +415,8 @@ export class VoiceConnection extends AbstractVoiceConnection {
         const dataChannelConfig = { ordered: false, maxRetransmits: 0 };
 
         this.dataChannel = this.rtcPeerConnection.createDataChannel('main', dataChannelConfig);
-        this.dataChannel.onmessage = this.on_data_channel_message.bind(this);
+        this.dataChannel.onmessage = this.onMainDataChannelMessage.bind(this);
-        this.dataChannel.onopen = this.on_data_channel.bind(this);
+        this.dataChannel.onopen = this.onMainDataChannelOpen.bind(this);
         this.dataChannel.binaryType = "arraybuffer";
 
         let sdpConstraints : RTCOfferOptions = {};
@@ -524,15 +549,15 @@ export class VoiceConnection extends AbstractVoiceConnection {
         });
     }
 
-    private on_data_channel(channel) {
+    private onMainDataChannelOpen(channel) {
         log.info(LogCategory.VOICE, tr("Got new data channel! (%s)"), this.dataChannel.readyState);
 
         this.connection.client.update_voice_status();
     }
 
-    private on_data_channel_message(message: MessageEvent) {
+    private onMainDataChannelMessage(message: MessageEvent) {
         const chandler = this.connection.client;
-        if(chandler.client_status.output_muted) /* we dont need to do anything with sound playback when we're not listening to it */
+        if(chandler.isSpeakerMuted() || chandler.isSpeakerDisabled()) /* we dont need to do anything with sound playback when we're not listening to it */
             return;
 
         let bin = new Uint8Array(message.data);
@@ -571,18 +596,18 @@ export class VoiceConnection extends AbstractVoiceConnection {
         }
     }
 
-    private handle_local_voice(data: AudioBuffer, head: boolean) {
+    private handleLocalVoiceBuffer(data: AudioBuffer, head: boolean) {
         const chandler = this.connection.client;
-        if(!chandler.connected)
+        if(!this.localAudioStarted || !chandler.connected)
             return false;
 
-        if(chandler.client_status.input_muted)
+        if(chandler.isMicrophoneMuted())
             return false;
 
         if(head)
             this.chunkVPacketId = 0;
 
-        let client = this.find_client(chandler.clientId);
+        let client = this.find_client(chandler.getClientId());
         if(!client) {
             log.error(LogCategory.VOICE, tr("Tried to send voice data, but local client hasn't a voice client handle"));
             return;
@@ -594,25 +619,31 @@ export class VoiceConnection extends AbstractVoiceConnection {
             .then(encoder => encoder.encodeSamples(client.get_codec_cache(codec), data));
     }
 
-    private handle_local_voice_ended() {
+    private handleLocalVoiceEnded() {
         const chandler = this.connection.client;
         const ch = chandler.getClient();
         if(ch) ch.speaking = false;
 
         if(!chandler.connected)
             return false;
-        if(chandler.client_status.input_muted)
+        if(chandler.isMicrophoneMuted())
             return false;
         log.info(LogCategory.VOICE, tr("Local voice ended"));
+        this.localAudioStarted = false;
 
-        if(this.dataChannel && this._encoder_codec >= 0)
+        if(this._type === VoiceEncodeType.NATIVE_ENCODE) {
-            this.send_voice_packet(new Uint8Array(0), this._encoder_codec);
+            setTimeout(() => {
+                /* first send all data, than send the stop signal */
+                this.sendVoiceStopPacket(this._encoder_codec);
+            }, 150);
+        } else {
+            this.sendVoiceStopPacket(this._encoder_codec);
+        }
     }
 
-    private handle_local_voice_started() {
+    private handleLocalVoiceStarted() {
         const chandler = this.connection.client;
-        if(chandler.client_status.input_muted) {
+        if(chandler.isMicrophoneMuted()) {
-            /* evail hack due to the settings :D */
             log.warn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted! Do not send any voice."));
             if(this.local_audio_mute)
                 this.local_audio_mute.gain.value = 0;
@@ -620,6 +651,8 @@ export class VoiceConnection extends AbstractVoiceConnection {
         }
         if(this.local_audio_mute)
             this.local_audio_mute.gain.value = 1;
+
+        this.localAudioStarted = true;
         log.info(LogCategory.VOICE, tr("Local voice started"));
 
         const ch = chandler.getClient();
@@ -683,7 +716,6 @@ export class VoiceConnection extends AbstractVoiceConnection {
 }
 
 
-
 /* funny fact that typescript dosn't find this */
 declare global {
     interface RTCPeerConnection {

web/js/workers/codec/OpusCodec.ts

@@ -121,6 +121,8 @@ const OPUS_ERROR_CODES = [
 ];
 
 class OpusWorker implements CodecWorker {
+    private static readonly kProcessBufferSize = 4096 * 2;
+
     private readonly channelCount: number;
     private readonly type: OpusType;
     private nativeHandle: any;
@@ -130,11 +132,8 @@ class OpusWorker implements CodecWorker {
     private fn_encode: any;
     private fn_reset: any;
 
-    private buffer_size = 4096 * 2;
+    private nativeBufferPtr: number;
-    private buffer: any;
+    private processBuffer: Uint8Array;
-
-    private encode_buffer: Float32Array;
-    private decode_buffer: Uint8Array;
 
     constructor(channelCount: number, type: OpusType) {
         this.channelCount = channelCount;
@@ -153,40 +152,39 @@ class OpusWorker implements CodecWorker {
 
         this.nativeHandle = this.fn_newHandle(this.channelCount, this.type);
 
-        this.buffer = Module._malloc(this.buffer_size);
+        this.nativeBufferPtr = Module._malloc(OpusWorker.kProcessBufferSize);
-        this.encode_buffer = new Float32Array(Module.HEAPF32.buffer, this.buffer, Math.floor(this.buffer_size / 4));
+        this.processBuffer = new Uint8Array(Module.HEAPU8.buffer, this.nativeBufferPtr, OpusWorker.kProcessBufferSize);
-        this.decode_buffer = new Uint8Array(Module.HEAPU8.buffer, this.buffer, this.buffer_size);
         return undefined;
     }
 
     deinitialise() { } //TODO
 
     decode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string {
-        if (buffer.byteLength > this.decode_buffer.byteLength)
+        if (buffer.byteLength > this.processBuffer.byteLength)
             return "supplied data exceeds internal buffer";
 
-        this.decode_buffer.set(buffer);
+        this.processBuffer.set(buffer);
 
-        let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, buffer.byteLength, this.decode_buffer.byteLength);
+        let result = this.fn_decode(this.nativeHandle, this.processBuffer.byteOffset, buffer.byteLength, this.processBuffer.byteLength);
         if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown decode error " + result;
 
         const resultByteLength = result * this.channelCount * 4;
         const resultBuffer = responseBuffer(resultByteLength);
-        resultBuffer.set(this.decode_buffer.subarray(0, resultByteLength), 0);
+        resultBuffer.set(this.processBuffer.subarray(0, resultByteLength), 0);
         return resultByteLength;
     }
 
     encode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string {
-        if (buffer.byteLength > this.decode_buffer.byteLength)
+        if (buffer.byteLength > this.processBuffer.byteLength)
             return "supplied data exceeds internal buffer";
 
-        this.encode_buffer.set(buffer);
+        this.processBuffer.set(buffer);
 
-        let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, buffer.byteLength, this.encode_buffer.byteLength);
+        let result = this.fn_encode(this.nativeHandle, this.processBuffer.byteOffset, buffer.byteLength, this.processBuffer.byteLength);
         if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown encode error " + result;
 
         const resultBuffer = responseBuffer(result);
-        resultBuffer.set(Module.HEAP8.subarray(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result));
+        resultBuffer.set(this.processBuffer.subarray(0, result), 0);
         return result;
     }
 

web/native-codec/src/opus.cpp

@@ -87,7 +87,7 @@ int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t byte_length, s
     if(handle->channelCount == 2)
         sequenced2interleaved_intersecting<2>((float *) buffer, byte_length / (sizeof(float) * 2));
 
-    auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / handle->channelCount, buffer, maxLength);
+    auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / (handle->channelCount * sizeof(float)), buffer, maxLength);
     if (result < 0) return result;
     return result;
 }