diff --git a/asm/libraries/opus b/asm/libraries/opus index 19966ccd..655cc54c 160000 --- a/asm/libraries/opus +++ b/asm/libraries/opus @@ -1 +1 @@ -Subproject commit 19966ccd4b743026d895c179ede04d436f65eca0 +Subproject commit 655cc54c564b84ef2827f0b2152ce3811046201e diff --git a/asm/make_opus.sh b/asm/make_opus.sh index 31563648..98543428 100755 --- a/asm/make_opus.sh +++ b/asm/make_opus.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash OPUS_FN="'_free','_malloc','_opus_strerror','_opus_get_version_string','_opus_encoder_get_size','_opus_encoder_init','_opus_encode','_opus_encode_float','_opus_encoder_ctl','_opus_decoder_get_size','_opus_decoder_init','_opus_decode','_opus_decode_float','_opus_decoder_ctl','_opus_packet_get_nb_samples'" -cd libs/opus/ +cd libraries/opus/ git checkout v1.1.2 ./autogen.sh emconfigure ./configure --disable-extra-programs --disable-doc --disable-rtcd emmake make cd ../../ -emcc -o generated/libopus.js -O3 --memory-init-file 0 --closure 1 -s NO_FILESYSTEM=1 -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="[$OPUS_FN]" libs/opus/.libs/libopus.a +emcc -o generated/libopus.js -O3 --memory-init-file 0 --closure 1 -s NO_FILESYSTEM=1 -s MODULARIZE=1 -s EXPORTED_FUNCTIONS="[$OPUS_FN]" libraries/opus/.libs/libopus.a diff --git a/asm/src/opus.cpp b/asm/src/opus.cpp index b27f836c..a86e8b2f 100644 --- a/asm/src/opus.cpp +++ b/asm/src/opus.cpp @@ -9,17 +9,72 @@ extern "C" { OpusDecoder* decoder = nullptr; size_t channelCount = 1; + size_t sampleRate = 48000; int opusType = OPUS_APPLICATION_AUDIO; }; + const char* opus_errors[7] = { + "One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG) + "Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL) + "An internal error was detected", //-3 (OPUS_INTERNAL_ERROR) + "The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET) + "Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED) + "An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE) + "Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL) + }; + + inline const char* opus_error_message(int error) { + error = abs(error); + if(error > 0 && error <= 7) return opus_errors[error - 1]; + return "undefined error"; + } + + inline int currentMillies() { + return EM_ASM_INT({ return Date.now(); }); + } + + #define _S(x) #x + #define INVOKE_OPUS(result, method, ...) \ + result = method( __VA_ARGS__ ); \ + if(error != 0){ \ + printf("Got opus error while invoking %s. Code: %d Message: %s\n", _S(method), error, opus_error_message(error)); \ + return false; \ + } + + inline bool reinitialize_decoder(OpusHandle *handle) { + if (handle->decoder) + opus_decoder_destroy(handle->decoder); + + int error = 0; + INVOKE_OPUS(handle->decoder, opus_decoder_create, 48000, handle->channelCount, &error); + return true; + } + + inline bool reinitialize_encoder(OpusHandle *handle) { + if (handle->encoder) + opus_encoder_destroy(handle->encoder); + + int error = 0; + INVOKE_OPUS(handle->encoder, opus_encoder_create, 48000, handle->channelCount, handle->opusType, &error); + INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_COMPLEXITY(1)); + //INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_BITRATE(4740)); + + EM_ASM( + printMessageToServerTab('Encoder initialized!'); + printMessageToServerTab(' Comprexity: 1'); + printMessageToServerTab(' Bitrate: 4740'); + ); + + return true; + } + EMSCRIPTEN_KEEPALIVE OpusHandle* codec_opus_createNativeHandle(size_t channelCount, int type) { - printf("Inizalisize opus. (Channel count: %d Sample rate: %d Type: %d)!\n", channelCount, 48000, type); + printf("Initialize opus. (Channel count: %d Sample rate: %d Type: %d)!\n", channelCount, 48000, type); auto codec = new OpusHandle{}; - int error = 0; - codec->decoder = opus_decoder_create(48000, channelCount, &error); - codec->encoder = opus_encoder_create(48000, channelCount, type, &error); - codec->opusType = type; + codec->opusType = type; + if(!reinitialize_decoder(codec)) return nullptr; + if(!reinitialize_encoder(codec)) return nullptr; return codec; } @@ -38,8 +93,13 @@ extern "C" { EMSCRIPTEN_KEEPALIVE int codec_opus_encode(OpusHandle* handle, uint8_t* buffer, size_t length, size_t maxLength) { + auto begin = currentMillies(); auto result = opus_encode_float(handle->encoder, (float*) buffer, length / handle->channelCount, buffer, maxLength); if(result < 0) return result; + auto end = currentMillies(); + EM_ASM({ + printMessageToServerTab("codec_opus_encode(...) tooks " + $0 + "ms to execute!"); + }, end - begin); return result; } @@ -60,12 +120,8 @@ extern "C" { EMSCRIPTEN_KEEPALIVE int codec_opus_reset(OpusHandle* handle) { - if(handle->encoder) opus_encoder_destroy(handle->encoder); - if(handle->decoder) opus_decoder_destroy(handle->decoder); - - int error = 0; - handle->decoder = opus_decoder_create(48000, handle->channelCount, &error); - handle->encoder = opus_encoder_create(48000, handle->channelCount, handle->opusType, &error); + if(!reinitialize_decoder(handle)) return 0; + if(!reinitialize_encoder(handle)) return 0; return 1; } /* diff --git a/js/FileManager.ts b/js/FileManager.ts index d7b0f91d..6a58f041 100644 --- a/js/FileManager.ts +++ b/js/FileManager.ts @@ -225,8 +225,8 @@ class FileManager { transfer.totalSize = json["size"]; transfer.remotePort = json["port"]; - transfer.remoteHost = json["ip"].replace(/,/g, ""); - if(transfer.remoteHost == '0.0.0.0' || transfer.remoteHost == '127.168.0.0') + transfer.remoteHost = (json["ip"] ? json["ip"] : "").replace(/,/g, ""); + if(!transfer.remoteHost || transfer.remoteHost == '0.0.0.0' || transfer.remoteHost == '127.168.0.0') transfer.remoteHost = this.handle.serverConnection._remoteHost; (transfer["_promiseCallback"] as (val: DownloadFileTransfer) => void)(transfer); diff --git a/js/codec/BasicCodec.ts b/js/codec/BasicCodec.ts index 801f55cd..ec391a76 100644 --- a/js/codec/BasicCodec.ts +++ b/js/codec/BasicCodec.ts @@ -1,10 +1,29 @@ /// +class AVGCalculator { + history_size: number = 100; + history: number[] = []; + + push(entry: number) { + while(this.history.length > this.history_size) + this.history.pop(); + this.history.unshift(entry); + } + + avg() : number { + let count = 0; + for(let entry of this.history) + count += entry; + return count / this.history.length; + } +} + abstract class BasicCodec implements Codec { protected _audioContext: OfflineAudioContext; protected _decodeResampler: AudioResampler; protected _encodeResampler: AudioResampler; protected _codecSampleRate: number; + protected _latenz: AVGCalculator = new AVGCalculator(); on_encoded_data: (Uint8Array) => void = $ => {}; channelCount: number = 1; @@ -13,7 +32,7 @@ abstract class BasicCodec implements Codec { constructor(codecSampleRate: number) { this.channelCount = 1; this.samplesPerUnit = 960; - this._audioContext = new OfflineAudioContext(1, 1024,44100 ); + this._audioContext = new OfflineAudioContext(AudioController.globalContext.destination.channelCount, 1024,AudioController.globalContext.sampleRate ); this._codecSampleRate = codecSampleRate; this._decodeResampler = new AudioResampler(AudioController.globalContext.sampleRate); this._encodeResampler = new AudioResampler(codecSampleRate); @@ -50,11 +69,14 @@ abstract class BasicCodec implements Codec { cache._chunks.pop_front(); } - let encodeBegin = new Date().getTime(); + let encodeBegin = Date.now(); this.encode(buffer).then(result => { if(result instanceof Uint8Array) { - if(new Date().getTime() - 20 > encodeBegin) - console.error("Required time: %d", new Date().getTime() - encodeBegin); + let time = Date.now() - encodeBegin; + if(time > 20) + console.error("Required time: %d", time); + if(time > 20) + chat.serverChat().appendError("Required decode time: " + time); this.on_encoded_data(result); } else console.error("[Codec][" + this.name() + "] Could not encode buffer. Result: " + result); diff --git a/js/codec/CodecWrapper.ts b/js/codec/CodecWrapper.ts index 77120a46..8bc9e02d 100644 --- a/js/codec/CodecWrapper.ts +++ b/js/codec/CodecWrapper.ts @@ -143,11 +143,12 @@ class CodecWrapper extends BasicCodec { private sendWorkerMessage(message: any, transfare?: any[]) { //console.log("Send worker: %o", message); + message["timestamp"] = Date.now(); this._worker.postMessage(JSON.stringify(message), transfare); } private onWorkerMessage(message: any) { - //console.log("Worker message: %o", message); + console.log("Worker message stock time: %d", Date.now() - message["timestamp"]); if(!message["token"]) { console.error("Invalid worker token!"); return; @@ -166,6 +167,9 @@ class CodecWrapper extends BasicCodec { this._workerCallbackReject = undefined; this._workerCallbackResolve = undefined; return; + } else if(message["type"] == "chatmessage_server") { + chat.serverChat().appendMessage(message["message"]); + return; } console.log("Costume callback! (%o)", message); return; diff --git a/js/codec/RawCodec.ts b/js/codec/RawCodec.ts index 9c90c688..bd8796cc 100644 --- a/js/codec/RawCodec.ts +++ b/js/codec/RawCodec.ts @@ -40,4 +40,8 @@ class RawCodec extends BasicCodec { } reset() : boolean { return true; } + + processLatency(): number { + return 0; + } } \ No newline at end of file diff --git a/js/voice/VoiceHandler.ts b/js/voice/VoiceHandler.ts index 7d072179..7bce9eaa 100644 --- a/js/voice/VoiceHandler.ts +++ b/js/voice/VoiceHandler.ts @@ -129,12 +129,29 @@ class VoiceConnection { this.codecPool[4].initialize(2); this.codecPool[5].initialize(2); + + setTimeout(() => { + //if(Date.now() - this.last != 20) + // chat.serverChat().appendError("INVALID LAST: " + (Date.now() - this.last)); + this.last = Date.now(); + if(this.encodedCache.length == 0){ + //console.log("MISSING VOICE!"); + //chat.serverChat().appendError("MISSING VOICE!"); + } else this.sendVoicePacket(this.encodedCache[0].data, this.encodedCache[0].codec); + this.encodedCache.pop_front(); + }, 20); } codecSupported(type: number) : boolean { return this.codecPool.length > type && this.codecPool[type].supported(); } + encodedCache: {data: Uint8Array, codec: number}[] = []; + last: number; + handleEncodedVoicePacket(data: Uint8Array, codec: number){ + this.encodedCache.push({data: data, codec: codec}); + } + sendVoicePacket(data: Uint8Array, codec: number) { if(this.dataChannel) { this.vpacketId++; diff --git a/js/voice/VoiceRecorder.ts b/js/voice/VoiceRecorder.ts index 5c0e66ec..212a76f3 100644 --- a/js/voice/VoiceRecorder.ts +++ b/js/voice/VoiceRecorder.ts @@ -17,10 +17,15 @@ abstract class VoiceActivityDetector { } } +//A small class extention +interface MediaStreamConstraints { + deviceId?: string; +} + class VoiceRecorder { private static readonly CHANNEL = 0; private static readonly CHANNELS = 1; - private static readonly BUFFER_SIZE = 1024; + private static readonly BUFFER_SIZE = 1024 * 4; handle: VoiceConnection; on_data: (data: AudioBuffer, head: boolean) => void = (data) => {}; @@ -180,10 +185,13 @@ class VoiceRecorder { const oldStream = this.microphoneStream; this.microphoneStream = this.audioContext.createMediaStreamSource(stream); this.microphoneStream.connect(this.processor); + chat.serverChat().appendMessage("Mic channels " + this.microphoneStream.channelCount); + chat.serverChat().appendMessage("Mic channel mode " + this.microphoneStream.channelCountMode); + chat.serverChat().appendMessage("Max channel count " + this.audioContext.destination.maxChannelCount); + chat.serverChat().appendMessage("Sample rate " + this.audioContext.sampleRate); this.vadHandler.initialiseNewStream(oldStream, this.microphoneStream); } } - class MuteVAD extends VoiceActivityDetector { shouldRecord(buffer: AudioBuffer): boolean { return false; diff --git a/js/workers/codec/CodecWorker.ts b/js/workers/codec/CodecWorker.ts index 4bba3b15..5d97f098 100644 --- a/js/workers/codec/CodecWorker.ts +++ b/js/workers/codec/CodecWorker.ts @@ -19,6 +19,7 @@ let codecInstance: CodecWorker; onmessage = function(e) { let data = JSON.parse(e.data); + console.log(prefix + "Pipeline timestamp: %d"); let res: any = {}; res.token = data.token; @@ -85,8 +86,19 @@ onmessage = function(e) { if(res.token && res.token.length > 0) sendMessage(res, e.origin); }; +function printMessageToServerTab(message: string) { + /* + sendMessage({ + token: workerCallbackToken, + type: "chatmessage_server", + message: message + }); + */ +} + declare function postMessage(message: any): void; function sendMessage(message: any, origin?: string){ //console.log(prefix + " Send to main: %o", message); + message["timestamp"] = Date.now(); postMessage(JSON.stringify(message)); } \ No newline at end of file