Merge remote-tracking branch 'origin/master'

asm/libraries/opus

@@ -1 +1 @@
-Subproject commit 19966ccd4b743026d895c179ede04d436f65eca0
+Subproject commit 655cc54c564b84ef2827f0b2152ce3811046201e

asm/make_opus.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 OPUS_FN="'_free','_malloc','_opus_strerror','_opus_get_version_string','_opus_encoder_get_size','_opus_encoder_init','_opus_encode','_opus_encode_float','_opus_encoder_ctl','_opus_decoder_get_size','_opus_decoder_init','_opus_decode','_opus_decode_float','_opus_decoder_ctl','_opus_packet_get_nb_samples'"
-cd libs/opus/
+cd libraries/opus/
 
 git checkout v1.1.2
 ./autogen.sh
 emconfigure ./configure --disable-extra-programs --disable-doc --disable-rtcd
 emmake make
 cd ../../
-emcc -o generated/libopus.js -O3 --memory-init-file 0 --closure 1 -s NO_FILESYSTEM=1 -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="[$OPUS_FN]" libs/opus/.libs/libopus.a
+emcc -o generated/libopus.js -O3 --memory-init-file 0 --closure 1 -s NO_FILESYSTEM=1 -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="[$OPUS_FN]" libraries/opus/.libs/libopus.a
 

asm/src/opus.cpp

@@ -9,17 +9,72 @@ extern "C" {
         OpusDecoder* decoder = nullptr;
 
         size_t channelCount = 1;
+	    size_t sampleRate = 48000;
         int opusType = OPUS_APPLICATION_AUDIO;
     };
 
+	const char* opus_errors[7] = {
+			"One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG)
+			"Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL)
+			"An internal error was detected", //-3 (OPUS_INTERNAL_ERROR)
+			"The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET)
+			"Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED)
+			"An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE)
+			"Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL)
+	};
+
+	inline const char* opus_error_message(int error) {
+		error = abs(error);
+		if(error > 0 && error <= 7) return opus_errors[error - 1];
+		return "undefined error";
+	}
+
+	inline int currentMillies() {
+		return EM_ASM_INT({ return Date.now(); });
+	}
+
+	#define _S(x) #x
+	#define INVOKE_OPUS(result, method, ...)                                                                                        \
+	result = method( __VA_ARGS__ );                                                                                                 \
+	if(error != 0){                                                                                                                 \
+		printf("Got opus error while invoking %s. Code: %d Message: %s\n", _S(method), error, opus_error_message(error));           \
+		return false;                                                                                                               \
+	}
+
+	inline bool reinitialize_decoder(OpusHandle *handle) {
+		if (handle->decoder)
+			opus_decoder_destroy(handle->decoder);
+
+		int error = 0;
+		INVOKE_OPUS(handle->decoder, opus_decoder_create, 48000, handle->channelCount, &error);
+		return true;
+	}
+
+    inline bool reinitialize_encoder(OpusHandle *handle) {
+	    if (handle->encoder)
+		    opus_encoder_destroy(handle->encoder);
+
+	    int error = 0;
+	    INVOKE_OPUS(handle->encoder, opus_encoder_create, 48000, handle->channelCount, handle->opusType, &error);
+	    INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_COMPLEXITY(1));
+	    //INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_BITRATE(4740));
+
+	    EM_ASM(
+		    printMessageToServerTab('Encoder initialized!');
+		    printMessageToServerTab('  Comprexity: 1');
+		    printMessageToServerTab('  Bitrate:    4740');
+	    );
+
+        return true;
+    }
+
     EMSCRIPTEN_KEEPALIVE
     OpusHandle* codec_opus_createNativeHandle(size_t channelCount, int type) {
-        printf("Inizalisize opus. (Channel count: %d Sample rate: %d Type: %d)!\n", channelCount, 48000, type);
+        printf("Initialize opus. (Channel count: %d Sample rate: %d Type: %d)!\n", channelCount, 48000, type);
         auto codec = new OpusHandle{};
-        int error = 0;
+	    codec->opusType = type;
-        codec->decoder = opus_decoder_create(48000, channelCount, &error);
+	    if(!reinitialize_decoder(codec)) return nullptr;
-        codec->encoder = opus_encoder_create(48000, channelCount, type, &error);
+	    if(!reinitialize_encoder(codec)) return nullptr;
-        codec->opusType = type;
         return codec;
     }
 
@@ -38,8 +93,13 @@ extern "C" {
 
     EMSCRIPTEN_KEEPALIVE
     int codec_opus_encode(OpusHandle* handle, uint8_t* buffer, size_t length, size_t maxLength) {
+	    auto begin = currentMillies();
         auto result = opus_encode_float(handle->encoder, (float*) buffer, length / handle->channelCount, buffer, maxLength);
         if(result < 0) return result;
+	    auto end = currentMillies();
+	    EM_ASM({
+		           printMessageToServerTab("codec_opus_encode(...) tooks " + $0 + "ms to execute!");
+	           }, end - begin);
         return result;
     }
 
@@ -60,12 +120,8 @@ extern "C" {
 
     EMSCRIPTEN_KEEPALIVE
     int codec_opus_reset(OpusHandle* handle) {
-        if(handle->encoder) opus_encoder_destroy(handle->encoder);
+	    if(!reinitialize_decoder(handle)) return 0;
-	    if(handle->decoder) opus_decoder_destroy(handle->decoder);
+	    if(!reinitialize_encoder(handle)) return 0;
-
-        int error = 0;
-        handle->decoder = opus_decoder_create(48000, handle->channelCount, &error);
-        handle->encoder = opus_encoder_create(48000, handle->channelCount, handle->opusType, &error);
         return 1;
     }
 /*

js/FileManager.ts

@@ -225,8 +225,8 @@ class FileManager {
         transfer.totalSize = json["size"];
 
         transfer.remotePort = json["port"];
-        transfer.remoteHost = json["ip"].replace(/,/g, "");
+        transfer.remoteHost = (json["ip"] ? json["ip"] : "").replace(/,/g, "");
-        if(transfer.remoteHost == '0.0.0.0' || transfer.remoteHost == '127.168.0.0')
+        if(!transfer.remoteHost || transfer.remoteHost == '0.0.0.0' || transfer.remoteHost == '127.168.0.0')
             transfer.remoteHost = this.handle.serverConnection._remoteHost;
 
         (transfer["_promiseCallback"] as (val: DownloadFileTransfer) => void)(transfer);

js/codec/BasicCodec.ts

@@ -1,10 +1,29 @@
 /// <reference path="Codec.ts"/>
 
+class AVGCalculator {
+    history_size: number = 100;
+    history: number[] = [];
+
+    push(entry: number) {
+        while(this.history.length > this.history_size)
+            this.history.pop();
+        this.history.unshift(entry);
+    }
+
+    avg() : number {
+        let count = 0;
+        for(let entry of this.history)
+            count += entry;
+        return count / this.history.length;
+    }
+}
+
 abstract class BasicCodec implements Codec {
     protected _audioContext: OfflineAudioContext;
     protected _decodeResampler: AudioResampler;
     protected _encodeResampler: AudioResampler;
     protected _codecSampleRate: number;
+    protected _latenz: AVGCalculator = new AVGCalculator();
 
     on_encoded_data: (Uint8Array) => void = $ => {};
     channelCount: number = 1;
@@ -13,7 +32,7 @@ abstract class BasicCodec implements Codec {
     constructor(codecSampleRate: number) {
         this.channelCount = 1;
         this.samplesPerUnit = 960;
-        this._audioContext = new OfflineAudioContext(1, 1024,44100 );
+        this._audioContext = new OfflineAudioContext(AudioController.globalContext.destination.channelCount, 1024,AudioController.globalContext.sampleRate );
         this._codecSampleRate = codecSampleRate;
         this._decodeResampler = new AudioResampler(AudioController.globalContext.sampleRate);
         this._encodeResampler = new AudioResampler(codecSampleRate);
@@ -50,11 +69,14 @@ abstract class BasicCodec implements Codec {
                     cache._chunks.pop_front();
             }
 
-            let encodeBegin = new Date().getTime();
+            let encodeBegin = Date.now();
             this.encode(buffer).then(result => {
                 if(result instanceof Uint8Array) {
-                    if(new Date().getTime() - 20 > encodeBegin)
+                    let time = Date.now() - encodeBegin;
-                        console.error("Required time: %d", new Date().getTime() - encodeBegin);
+                    if(time > 20)
+                        console.error("Required time: %d", time);
+                    if(time > 20)
+                        chat.serverChat().appendError("Required decode time: " + time);
                     this.on_encoded_data(result);
                 }
                 else console.error("[Codec][" + this.name() + "] Could not encode buffer. Result: " + result);

js/codec/CodecWrapper.ts

@@ -143,11 +143,12 @@ class CodecWrapper extends BasicCodec {
 
     private sendWorkerMessage(message: any, transfare?: any[]) {
         //console.log("Send worker: %o", message);
+        message["timestamp"] = Date.now();
         this._worker.postMessage(JSON.stringify(message), transfare);
     }
 
     private onWorkerMessage(message: any) {
-        //console.log("Worker message: %o", message);
+        console.log("Worker message stock time: %d", Date.now() - message["timestamp"]);
         if(!message["token"]) {
             console.error("Invalid worker token!");
             return;
@@ -166,6 +167,9 @@ class CodecWrapper extends BasicCodec {
                 this._workerCallbackReject = undefined;
                 this._workerCallbackResolve = undefined;
                 return;
+            } else if(message["type"] == "chatmessage_server") {
+                chat.serverChat().appendMessage(message["message"]);
+                return;
             }
             console.log("Costume callback! (%o)", message);
             return;

js/codec/RawCodec.ts

@@ -40,4 +40,8 @@ class RawCodec extends BasicCodec {
     }
 
     reset() : boolean { return true; }
+
+    processLatency(): number {
+        return 0;
+    }
 }

js/voice/VoiceHandler.ts

@@ -129,12 +129,29 @@ class VoiceConnection {
 
         this.codecPool[4].initialize(2);
         this.codecPool[5].initialize(2);
+
+        setTimeout(() => {
+            //if(Date.now() - this.last != 20)
+            //    chat.serverChat().appendError("INVALID LAST: " + (Date.now() - this.last));
+            this.last = Date.now();
+            if(this.encodedCache.length == 0){
+                //console.log("MISSING VOICE!");
+                //chat.serverChat().appendError("MISSING VOICE!");
+            } else this.sendVoicePacket(this.encodedCache[0].data, this.encodedCache[0].codec);
+            this.encodedCache.pop_front();
+        }, 20);
     }
 
     codecSupported(type: number) : boolean {
         return this.codecPool.length > type && this.codecPool[type].supported();
     }
 
+    encodedCache: {data: Uint8Array, codec: number}[] = [];
+    last: number;
+    handleEncodedVoicePacket(data: Uint8Array, codec: number){
+        this.encodedCache.push({data: data, codec: codec});
+    }
+
     sendVoicePacket(data: Uint8Array, codec: number) {
         if(this.dataChannel) {
             this.vpacketId++;

js/voice/VoiceRecorder.ts

@@ -17,10 +17,15 @@ abstract class VoiceActivityDetector {
     }
 }
 
+//A small class extention
+interface MediaStreamConstraints {
+    deviceId?: string;
+}
+
 class VoiceRecorder {
     private static readonly CHANNEL = 0;
     private static readonly CHANNELS = 1;
-    private static readonly BUFFER_SIZE = 1024;
+    private static readonly BUFFER_SIZE = 1024 * 4;
 
     handle: VoiceConnection;
     on_data: (data: AudioBuffer, head: boolean) => void = (data) => {};
@@ -180,10 +185,13 @@ class VoiceRecorder {
         const oldStream = this.microphoneStream;
         this.microphoneStream = this.audioContext.createMediaStreamSource(stream);
         this.microphoneStream.connect(this.processor);
+        chat.serverChat().appendMessage("Mic channels " + this.microphoneStream.channelCount);
+        chat.serverChat().appendMessage("Mic channel mode " + this.microphoneStream.channelCountMode);
+        chat.serverChat().appendMessage("Max channel count " + this.audioContext.destination.maxChannelCount);
+        chat.serverChat().appendMessage("Sample rate " + this.audioContext.sampleRate);
         this.vadHandler.initialiseNewStream(oldStream, this.microphoneStream);
     }
 }
-
 class MuteVAD extends VoiceActivityDetector {
     shouldRecord(buffer: AudioBuffer): boolean {
         return false;

js/workers/codec/CodecWorker.ts

@@ -19,6 +19,7 @@ let codecInstance: CodecWorker;
 
 onmessage = function(e) {
     let data = JSON.parse(e.data);
+    console.log(prefix + "Pipeline timestamp: %d");
 
     let res: any = {};
     res.token = data.token;
@@ -85,8 +86,19 @@ onmessage = function(e) {
     if(res.token && res.token.length > 0) sendMessage(res, e.origin);
 };
 
+function printMessageToServerTab(message: string) {
+    /*
+    sendMessage({
+        token: workerCallbackToken,
+        type: "chatmessage_server",
+        message: message
+    });
+    */
+}
+
 declare function postMessage(message: any): void;
 function sendMessage(message: any, origin?: string){
     //console.log(prefix + " Send to main: %o", message);
+    message["timestamp"] = Date.now();
     postMessage(JSON.stringify(message));
 }