diff --git a/asm/CMakeLists.txt b/asm/CMakeLists.txt index f82daa65..ead1f8c6 100644 --- a/asm/CMakeLists.txt +++ b/asm/CMakeLists.txt @@ -1,21 +1,25 @@ cmake_minimum_required(VERSION 3.9) project(TeaWeb-Native) -set (CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_FLAGS_DEBUG "") #Override some config values from the parent project -set(CMAKE_CXX_COMPILER "emcc") -set(CMAKE_C_COMPILER "emcc") -set(CMAKE_C_LINK_EXECUTABLE "emcc") +set (CMAKE_CXX_STANDARD 17) + + +function(import_opus) + # Native SIMD isn't supported yet by most browsers (only experimental) + # But since opus already detects if emscripten is able to handle SIMD we have no need to disable this explicitly + + # Disable the math.h warning spam: + # #warning "Don't have the functions lrint() and lrintf ()." + # #warning "Replacing these functions with a standard C cast." + set(CMAKE_C_FLAGS "-Wno-#warnings") + set(OPUS_STACK_PROTECTOR OFF CACHE BOOL "" FORCE) + add_subdirectory(libraries/opus/) +endfunction() +import_opus() + set(CMAKE_CXX_FLAGS "-O3 --llvm-lto 1 --memory-init-file 0 -s WASM=1 -s ASSERTIONS=1") # -s ALLOW_MEMORY_GROWTH=1 -O3 -set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_EXE_LINKER_FLAGS "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]' -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -s ENVIRONMENT='worker' --pre-js ${CMAKE_SOURCE_DIR}/init.js") # -#add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/generated/") -include_directories(libraries/tommath/) -include_directories(libraries/tomcrypt/src/headers) -include_directories(libraries/opus/include/) -add_definitions(-DLTM_DESC) - add_executable(TeaWeb-Worker-Codec-Opus src/opus.cpp) -target_link_libraries(TeaWeb-Worker-Codec-Opus ${CMAKE_CURRENT_SOURCE_DIR}/libraries/opus/out/lib/libopus.a) +target_link_libraries(TeaWeb-Worker-Codec-Opus opus) diff --git a/asm/src/opus.cpp b/asm/src/opus.cpp index 854ad3e7..bdc1030a 100644 --- a/asm/src/opus.cpp +++ b/asm/src/opus.cpp @@ -1,138 +1,106 @@ #include +#include +#include #include #include -using namespace std; +typedef std::unique_ptr opus_encoder_t; +typedef std::unique_ptr opus_decoder_t; +struct OpusHandle { + opus_encoder_t encoder{nullptr, opus_encoder_destroy}; + opus_decoder_t decoder{nullptr, opus_decoder_destroy}; + + size_t channelCount{1}; + size_t sampleRate{48000}; + int opusType{OPUS_APPLICATION_AUDIO}; +}; + +constexpr std::array opus_errors = { + "One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG) + "Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL) + "An internal error was detected", //-3 (OPUS_INTERNAL_ERROR) + "The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET) + "Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED) + "An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE) + "Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL) +}; + +inline std::string_view opus_error_message(int error) { + error = abs(error); + if (error > 0 && error <= 7) return opus_errors[error - 1]; + return "undefined error"; +} + +inline bool reinitialize_decoder(OpusHandle *handle) { + int error; + handle->decoder.reset(opus_decoder_create(handle->sampleRate, handle->channelCount, &error)); + if(error != OPUS_OK) { + printf("Failed to create decoder (%s)\n", opus_error_message(error).data()); + return false; + } + return true; +} + +inline bool reinitialize_encoder(OpusHandle *handle) { + int error; + handle->encoder.reset(opus_encoder_create(handle->sampleRate, handle->channelCount, handle->opusType, &error)); + if (error != OPUS_OK) { + printf("Failed to create encoder (%s)\n", opus_error_message(error).data()); + return false; + } + + if(error = opus_encoder_ctl(&*handle->encoder, OPUS_SET_COMPLEXITY(1)); error != OPUS_OK) { + printf("Failed to setup encoder (%s)\n", opus_error_message(error).data()); + return false; + } + //TODO: May set OPUS_SET_BITRATE(4740)? + //TODO: Is the encoder event needed anymore? Or is it just overhead + return true; +} + +#ifdef __cplusplus extern "C" { - struct OpusHandle { - OpusEncoder* encoder = nullptr; - OpusDecoder* decoder = nullptr; +#endif +EMSCRIPTEN_KEEPALIVE +OpusHandle *codec_opus_createNativeHandle(size_t channelCount, int type) { + auto codec = new OpusHandle{}; + codec->opusType = type; + codec->channelCount = channelCount; + codec->sampleRate = 48000; + if (!reinitialize_decoder(codec)) return nullptr; + if (!reinitialize_encoder(codec)) return nullptr; + return codec; +} - size_t channelCount = 1; - size_t sampleRate = 48000; - int opusType = OPUS_APPLICATION_AUDIO; - }; +EMSCRIPTEN_KEEPALIVE +void codec_opus_deleteNativeHandle(OpusHandle *codec) { + if (!codec) return; - const char* opus_errors[7] = { - "One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG) - "Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL) - "An internal error was detected", //-3 (OPUS_INTERNAL_ERROR) - "The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET) - "Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED) - "An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE) - "Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL) - }; + codec->decoder.reset(); + codec->encoder.reset(); + delete codec; +} - EMSCRIPTEN_KEEPALIVE - inline const char* opus_error_message(int error) { - error = abs(error); - if(error > 0 && error <= 7) return opus_errors[error - 1]; - return "undefined error"; - } +EMSCRIPTEN_KEEPALIVE +int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t length, size_t maxLength) { + auto result = opus_encode_float(&*handle->encoder, (float *) buffer, length / handle->channelCount, buffer, maxLength); + if (result < 0) return result; + return result; +} - inline int currentMillies() { - return EM_ASM_INT({ return Date.now(); }); - } +EMSCRIPTEN_KEEPALIVE +int codec_opus_decode(OpusHandle *handle, uint8_t *buffer, size_t length, size_t maxLength) { + auto result = opus_decode_float(&*handle->decoder, buffer, length, (float *) buffer, maxLength / sizeof(float) / handle->channelCount, false); + if (result < 0) return result; + return result; +} - #define _S(x) #x - #define INVOKE_OPUS(result, method, ...) \ - result = method( __VA_ARGS__ ); \ - if(error != 0){ \ - printf("Got opus error while invoking %s. Code: %d Message: %s\n", _S(method), error, opus_error_message(error)); \ - return false; \ - } - - inline bool reinitialize_decoder(OpusHandle *handle) { - if (handle->decoder) - opus_decoder_destroy(handle->decoder); - - int error = 0; - INVOKE_OPUS(handle->decoder, opus_decoder_create, 48000, handle->channelCount, &error); - return true; - } - - inline bool reinitialize_encoder(OpusHandle *handle) { - if (handle->encoder) - opus_encoder_destroy(handle->encoder); - - int error = 0; - INVOKE_OPUS(handle->encoder, opus_encoder_create, 48000, handle->channelCount, handle->opusType, &error); - INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_COMPLEXITY(1)); - //INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_BITRATE(4740)); - - /* //This method is obsolete! - EM_ASM( - printMessageToServerTab('Encoder initialized!'); - printMessageToServerTab(' Comprexity: 1'); - printMessageToServerTab(' Bitrate: 4740'); - ); - */ - - return true; - } - - EMSCRIPTEN_KEEPALIVE - OpusHandle* codec_opus_createNativeHandle(size_t channelCount, int type) { - printf("Initialize opus. (Channel count: %d Sample rate: %d Type: %d)!\n", channelCount, 48000, type); - auto codec = new OpusHandle{}; - codec->opusType = type; - codec->channelCount = channelCount; - if(!reinitialize_decoder(codec)) return nullptr; - if(!reinitialize_encoder(codec)) return nullptr; - return codec; - } - - EMSCRIPTEN_KEEPALIVE - void codec_opus_deleteNativeHandle(OpusHandle* codec) { - if(!codec) return; - - if(codec->decoder) opus_decoder_destroy(codec->decoder); - codec->decoder = nullptr; - - if(codec->encoder) opus_encoder_destroy(codec->encoder); - codec->encoder = nullptr; - - delete codec; - } - - EMSCRIPTEN_KEEPALIVE - int codec_opus_encode(OpusHandle* handle, uint8_t* buffer, size_t length, size_t maxLength) { - auto begin = currentMillies(); - auto result = opus_encode_float(handle->encoder, (float*) buffer, length / handle->channelCount, buffer, maxLength); - if(result < 0) return result; - auto end = currentMillies(); - /* //This message is obsolete - EM_ASM({ - printMessageToServerTab("codec_opus_encode(...) tooks " + $0 + "ms to execute!"); - }, end - begin); - */ - return result; - } - - EMSCRIPTEN_KEEPALIVE - int codec_opus_decode(OpusHandle* handle, uint8_t* buffer, size_t length, size_t maxLength) { - auto result = opus_decode_float(handle->decoder, buffer, length, (float*) buffer, maxLength / sizeof(float) / handle->channelCount, false); - if(result < 0) return result; //Failed - return result; - } - - EMSCRIPTEN_KEEPALIVE - int codec_opus_changeApplication(OpusHandle* handle, int type) { - handle->opusType = type; - if(type != OPUS_APPLICATION_VOIP && type != OPUS_APPLICATION_AUDIO && type != OPUS_APPLICATION_RESTRICTED_LOWDELAY) - return 1; - return opus_encoder_ctl(handle->encoder, OPUS_SET_APPLICATION(type)); - } - - EMSCRIPTEN_KEEPALIVE - int codec_opus_reset(OpusHandle* handle) { - if(!reinitialize_decoder(handle)) return 0; - if(!reinitialize_encoder(handle)) return 0; - return 1; - } -/* -opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate)); -opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(complexity)); -opus_encoder_ctl(enc, OPUS_SET_SIGNAL(signal_type)); - */ -} \ No newline at end of file +EMSCRIPTEN_KEEPALIVE +int codec_opus_reset(OpusHandle *handle) { + if (!reinitialize_decoder(handle)) return 0; + if (!reinitialize_encoder(handle)) return 0; + return 1; +} +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/web/js/workers/codec/CodecWorker.ts b/web/js/workers/codec/CodecWorker.ts index 2b88482f..2184d58c 100644 --- a/web/js/workers/codec/CodecWorker.ts +++ b/web/js/workers/codec/CodecWorker.ts @@ -22,24 +22,7 @@ export function set_initialize_callback(callback: () => Promise) initialize_callback = callback; } -export let codecInstance: CodecWorker; - -function printMessageToServerTab(message: string) { - /* - sendMessage({ - token: workerCallbackToken, - type: "chatmessage_server", - message: message - }); - */ -} - -declare function postMessage(message: any): void; -function sendMessage(message: any, origin?: string) { - message["timestamp"] = Date.now(); - postMessage(message); -} - +export let codec_instance: CodecWorker; let globally_initialized = false; let global_initialize_result; @@ -59,18 +42,18 @@ async function handle_message(command: string, data: any) : Promise { data["timestamp_received"] = received; data["timestamp_send"] = Date.now(); - sendMessage(data, e.origin); + postMessage(data, undefined); }; handle_message(e.data.command, e.data.data).then(res => { if(token) { diff --git a/web/js/workers/codec/OpusCodec.ts b/web/js/workers/codec/OpusCodec.ts index b89fbc5c..91eca0e1 100644 --- a/web/js/workers/codec/OpusCodec.ts +++ b/web/js/workers/codec/OpusCodec.ts @@ -2,8 +2,6 @@ import * as cworker from "./CodecWorker"; import {CodecType} from "tc-backend/web/codec/Codec"; import {CodecWorker} from "./CodecWorker"; -const prefix = "OpusWorker"; - declare global { interface Window { __init_em_module: ((Module: any) => void)[]; @@ -60,7 +58,7 @@ self.__init_em_module.push(Module => { if(arguments.length == 1 && arguments[0] == abort_message) return; /* we don't need to reprint the abort message! */ - console.log("Print: ", ...arguments); + console.log(...arguments); }; Module['printErr'] = function() { @@ -72,7 +70,7 @@ self.__init_em_module.push(Module => { if((arguments[0] as string).indexOf(suppress) != -1) return; - console.error("Error: ",...arguments); + console.error(...arguments); }; Module['locateFile'] = file => "../../wasm/" + file; @@ -84,22 +82,31 @@ enum OpusType { RESTRICTED_LOWDELAY = 2051 } +const OPUS_ERROR_CODES = [ + "One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG) + "Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL) + "An internal error was detected", //-3 (OPUS_INTERNAL_ERROR) + "The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET) + "Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED) + "An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE) + "Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL) +]; + class OpusWorker implements CodecWorker { - private channelCount: number; + private readonly channelCount: number; + private readonly type: OpusType; private nativeHandle: any; - private type: OpusType; private fn_newHandle: any; private fn_decode: any; private fn_encode: any; private fn_reset: any; - private fn_error_message: any; - private bufferSize = 4096 * 2; - private encodeBufferRaw: any; - private encodeBuffer: Float32Array; - private decodeBufferRaw: any; - private decodeBuffer: Uint8Array; + private buffer_size = 4096 * 2; + private buffer: any; + + private encode_buffer: Float32Array; + private decode_buffer: Uint8Array; constructor(channelCount: number, type: OpusType) { this.channelCount = channelCount; @@ -113,42 +120,39 @@ class OpusWorker implements CodecWorker { initialise?() : string { this.fn_newHandle = Module.cwrap("codec_opus_createNativeHandle", "number", ["number", "number"]); this.fn_decode = Module.cwrap("codec_opus_decode", "number", ["number", "number", "number", "number"]); - /* codec_opus_decode(handle, buffer, length, maxlength) */ this.fn_encode = Module.cwrap("codec_opus_encode", "number", ["number", "number", "number", "number"]); this.fn_reset = Module.cwrap("codec_opus_reset", "number", ["number"]); - this.fn_error_message = Module.cwrap("opus_error_message", "string", ["number"]); this.nativeHandle = this.fn_newHandle(this.channelCount, this.type); - this.encodeBufferRaw = Module._malloc(this.bufferSize); - this.encodeBuffer = new Float32Array(Module.HEAPF32.buffer, this.encodeBufferRaw, this.bufferSize / 4); - - this.decodeBufferRaw = Module._malloc(this.bufferSize); - this.decodeBuffer = new Uint8Array(Module.HEAPU8.buffer, this.decodeBufferRaw, this.bufferSize); + this.buffer = Module._malloc(this.buffer_size); + this.encode_buffer = new Float32Array(Module.HEAPF32.buffer, this.buffer, Math.floor(this.buffer_size / 4)); + this.decode_buffer = new Uint8Array(Module.HEAPU8.buffer, this.buffer, this.buffer_size); return undefined; } deinitialise() { } //TODO decode(data: Uint8Array): Float32Array | string { - if (data.byteLength > this.decodeBuffer.byteLength) return "Data to long!"; - this.decodeBuffer.set(data); - let result = this.fn_decode(this.nativeHandle, this.decodeBuffer.byteOffset, data.byteLength, this.decodeBuffer.byteLength); - if (result < 0) return this.fn_error_message(result); - return Module.HEAPF32.slice(this.decodeBuffer.byteOffset / 4, (this.decodeBuffer.byteOffset / 4) + (result * this.channelCount)); + if (data.byteLength > this.decode_buffer.byteLength) return "supplied data exceeds internal buffer"; + this.decode_buffer.set(data); + + let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, data.byteLength, this.decode_buffer.byteLength); + if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown decode error " + result; + + return Module.HEAPF32.slice(this.decode_buffer.byteOffset / 4, (this.decode_buffer.byteOffset / 4) + (result * this.channelCount)); } encode(data: Float32Array): Uint8Array | string { - this.encodeBuffer.set(data); + this.encode_buffer.set(data); - let result = this.fn_encode(this.nativeHandle, this.encodeBuffer.byteOffset, data.length, this.encodeBuffer.byteLength); - if (result < 0) return this.fn_error_message(result); - let buf = Module.HEAP8.slice(this.encodeBuffer.byteOffset, this.encodeBuffer.byteOffset + result); - return Uint8Array.from(buf); + let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, data.length, this.encode_buffer.byteLength); + if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown encode error " + result; + + return Module.HEAP8.slice(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result); } reset() { - console.log(prefix + " Reseting opus codec!"); this.fn_reset(this.nativeHandle); } } diff --git a/webpack.config.js b/webpack.config.js index 55222cbb..e2811529 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -33,12 +33,10 @@ module.exports = { cwd: process.cwd(), }) */ - /* new webpack.optimize.AggressiveSplittingPlugin({ minSize: 1024 * 128, maxSize: 1024 * 1024 }) - */ ], module: { rules: [