TeaSpeak/TeaWeb
TeaSpeak/TeaWeb
1
0
Fork 0
Code Issues 1 Pull requests Activity

Cleaned up the opus worker a bit

Browse source
This commit is contained in:
WolverinDEV 2020-03-31 02:11:55 +02:00
parent 956f778c01
commit bc9375afa1
5 changed files with 155 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
asm/CMakeLists.txt
View file

@ -1,21 +1,25 @@
cmake_minimum_required(VERSION 3.9) cmake_minimum_required(VERSION 3.9)
project(TeaWeb-Native) project(TeaWeb-Native)
set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS_DEBUG "") #Override some config values from the parent project
set(CMAKE_CXX_COMPILER "emcc")
set(CMAKE_C_COMPILER "emcc") function(import_opus)
set(CMAKE_C_LINK_EXECUTABLE "emcc") # Native SIMD isn't supported yet by most browsers (only experimental)
# But since opus already detects if emscripten is able to handle SIMD we have no need to disable this explicitly
# Disable the math.h warning spam:
# #warning "Don't have the functions lrint() and lrintf ()."
# #warning "Replacing these functions with a standard C cast."
set(CMAKE_C_FLAGS "-Wno-#warnings")
set(OPUS_STACK_PROTECTOR OFF CACHE BOOL "" FORCE)
add_subdirectory(libraries/opus/)
endfunction()
import_opus()
set(CMAKE_CXX_FLAGS "-O3 --llvm-lto 1 --memory-init-file 0 -s WASM=1 -s ASSERTIONS=1") # -s ALLOW_MEMORY_GROWTH=1 -O3 set(CMAKE_CXX_FLAGS "-O3 --llvm-lto 1 --memory-init-file 0 -s WASM=1 -s ASSERTIONS=1") # -s ALLOW_MEMORY_GROWTH=1 -O3
set(CMAKE_VERBOSE_MAKEFILE ON)
set(CMAKE_EXE_LINKER_FLAGS "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]' -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -s ENVIRONMENT='worker' --pre-js ${CMAKE_SOURCE_DIR}/init.js") # set(CMAKE_EXE_LINKER_FLAGS "-s EXPORTED_FUNCTIONS='[\"_malloc\", \"_free\"]' -s EXTRA_EXPORTED_RUNTIME_METHODS='[\"ccall\", \"cwrap\"]' -s ENVIRONMENT='worker' --pre-js ${CMAKE_SOURCE_DIR}/init.js") #
#add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/generated/") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/generated/")
include_directories(libraries/tommath/)
include_directories(libraries/tomcrypt/src/headers)
include_directories(libraries/opus/include/)
add_definitions(-DLTM_DESC)
add_executable(TeaWeb-Worker-Codec-Opus src/opus.cpp) add_executable(TeaWeb-Worker-Codec-Opus src/opus.cpp)
target_link_libraries(TeaWeb-Worker-Codec-Opus ${CMAKE_CURRENT_SOURCE_DIR}/libraries/opus/out/lib/libopus.a) target_link_libraries(TeaWeb-Worker-Codec-Opus opus)

228
asm/src/opus.cpp
View file

@ -1,138 +1,106 @@
#include <opus.h> #include <opus.h>
#include <array>
#include <string_view>
#include <emscripten.h> #include <emscripten.h>
#include <string> #include <string>
using namespace std; typedef std::unique_ptr<OpusEncoder, decltype(opus_encoder_destroy)*> opus_encoder_t;
extern "C" { typedef std::unique_ptr<OpusDecoder, decltype(opus_decoder_destroy)*> opus_decoder_t;
struct OpusHandle { struct OpusHandle {
OpusEncoder* encoder = nullptr; opus_encoder_t encoder{nullptr, opus_encoder_destroy};
OpusDecoder* decoder = nullptr; opus_decoder_t decoder{nullptr, opus_decoder_destroy};
size_t channelCount = 1; size_t channelCount{1};
size_t sampleRate = 48000; size_t sampleRate{48000};
int opusType = OPUS_APPLICATION_AUDIO; int opusType{OPUS_APPLICATION_AUDIO};
}; };
const char* opus_errors[7] = { constexpr std::array<std::string_view, 7> opus_errors = {
"One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG) "One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG)
"Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL) "Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL)
"An internal error was detected", //-3 (OPUS_INTERNAL_ERROR) "An internal error was detected", //-3 (OPUS_INTERNAL_ERROR)
"The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET) "The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET)
"Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED) "Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED)
"An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE) "An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE)
"Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL) "Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL)
}; };
EMSCRIPTEN_KEEPALIVE inline std::string_view opus_error_message(int error) {
inline const char* opus_error_message(int error) { error = abs(error);
error = abs(error); if (error > 0 && error <= 7) return opus_errors[error - 1];
if(error > 0 && error <= 7) return opus_errors[error - 1]; return "undefined error";
return "undefined error";
}
inline int currentMillies() {
return EM_ASM_INT({ return Date.now(); });
}
#define _S(x) #x
#define INVOKE_OPUS(result, method, ...) \
result = method( __VA_ARGS__ ); \
if(error != 0){ \
printf("Got opus error while invoking %s. Code: %d Message: %s\n", _S(method), error, opus_error_message(error)); \
return false; \
}
inline bool reinitialize_decoder(OpusHandle *handle) {
if (handle->decoder)
opus_decoder_destroy(handle->decoder);
int error = 0;
INVOKE_OPUS(handle->decoder, opus_decoder_create, 48000, handle->channelCount, &error);
return true;
}
inline bool reinitialize_encoder(OpusHandle *handle) {
if (handle->encoder)
opus_encoder_destroy(handle->encoder);
int error = 0;
INVOKE_OPUS(handle->encoder, opus_encoder_create, 48000, handle->channelCount, handle->opusType, &error);
INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_COMPLEXITY(1));
//INVOKE_OPUS(error, opus_encoder_ctl, handle->encoder, OPUS_SET_BITRATE(4740));
/* //This method is obsolete!
EM_ASM(
printMessageToServerTab('Encoder initialized!');
printMessageToServerTab(' Comprexity: 1');
printMessageToServerTab(' Bitrate: 4740');
);
*/
return true;
}
EMSCRIPTEN_KEEPALIVE
OpusHandle* codec_opus_createNativeHandle(size_t channelCount, int type) {
printf("Initialize opus. (Channel count: %d Sample rate: %d Type: %d)!\n", channelCount, 48000, type);
auto codec = new OpusHandle{};
codec->opusType = type;
codec->channelCount = channelCount;
if(!reinitialize_decoder(codec)) return nullptr;
if(!reinitialize_encoder(codec)) return nullptr;
return codec;
}
EMSCRIPTEN_KEEPALIVE
void codec_opus_deleteNativeHandle(OpusHandle* codec) {
if(!codec) return;
if(codec->decoder) opus_decoder_destroy(codec->decoder);
codec->decoder = nullptr;
if(codec->encoder) opus_encoder_destroy(codec->encoder);
codec->encoder = nullptr;
delete codec;
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_encode(OpusHandle* handle, uint8_t* buffer, size_t length, size_t maxLength) {
auto begin = currentMillies();
auto result = opus_encode_float(handle->encoder, (float*) buffer, length / handle->channelCount, buffer, maxLength);
if(result < 0) return result;
auto end = currentMillies();
/* //This message is obsolete
EM_ASM({
printMessageToServerTab("codec_opus_encode(...) tooks " + $0 + "ms to execute!");
}, end - begin);
*/
return result;
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_decode(OpusHandle* handle, uint8_t* buffer, size_t length, size_t maxLength) {
auto result = opus_decode_float(handle->decoder, buffer, length, (float*) buffer, maxLength / sizeof(float) / handle->channelCount, false);
if(result < 0) return result; //Failed
return result;
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_changeApplication(OpusHandle* handle, int type) {
handle->opusType = type;
if(type != OPUS_APPLICATION_VOIP && type != OPUS_APPLICATION_AUDIO && type != OPUS_APPLICATION_RESTRICTED_LOWDELAY)
return 1;
return opus_encoder_ctl(handle->encoder, OPUS_SET_APPLICATION(type));
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_reset(OpusHandle* handle) {
if(!reinitialize_decoder(handle)) return 0;
if(!reinitialize_encoder(handle)) return 0;
return 1;
}
/*
opus_encoder_ctl(enc, OPUS_SET_BITRATE(bitrate));
opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(complexity));
opus_encoder_ctl(enc, OPUS_SET_SIGNAL(signal_type));
*/
} }
inline bool reinitialize_decoder(OpusHandle *handle) {
int error;
handle->decoder.reset(opus_decoder_create(handle->sampleRate, handle->channelCount, &error));
if(error != OPUS_OK) {
printf("Failed to create decoder (%s)\n", opus_error_message(error).data());
return false;
}
return true;
}
inline bool reinitialize_encoder(OpusHandle *handle) {
int error;
handle->encoder.reset(opus_encoder_create(handle->sampleRate, handle->channelCount, handle->opusType, &error));
if (error != OPUS_OK) {
printf("Failed to create encoder (%s)\n", opus_error_message(error).data());
return false;
}
if(error = opus_encoder_ctl(&*handle->encoder, OPUS_SET_COMPLEXITY(1)); error != OPUS_OK) {
printf("Failed to setup encoder (%s)\n", opus_error_message(error).data());
return false;
}
//TODO: May set OPUS_SET_BITRATE(4740)?
//TODO: Is the encoder event needed anymore? Or is it just overhead
return true;
}
#ifdef __cplusplus
extern "C" {
#endif
EMSCRIPTEN_KEEPALIVE
OpusHandle *codec_opus_createNativeHandle(size_t channelCount, int type) {
auto codec = new OpusHandle{};
codec->opusType = type;
codec->channelCount = channelCount;
codec->sampleRate = 48000;
if (!reinitialize_decoder(codec)) return nullptr;
if (!reinitialize_encoder(codec)) return nullptr;
return codec;
}
EMSCRIPTEN_KEEPALIVE
void codec_opus_deleteNativeHandle(OpusHandle *codec) {
if (!codec) return;
codec->decoder.reset();
codec->encoder.reset();
delete codec;
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t length, size_t maxLength) {
auto result = opus_encode_float(&*handle->encoder, (float *) buffer, length / handle->channelCount, buffer, maxLength);
if (result < 0) return result;
return result;
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_decode(OpusHandle *handle, uint8_t *buffer, size_t length, size_t maxLength) {
auto result = opus_decode_float(&*handle->decoder, buffer, length, (float *) buffer, maxLength / sizeof(float) / handle->channelCount, false);
if (result < 0) return result;
return result;
}
EMSCRIPTEN_KEEPALIVE
int codec_opus_reset(OpusHandle *handle) {
if (!reinitialize_decoder(handle)) return 0;
if (!reinitialize_encoder(handle)) return 0;
return 1;
}
#ifdef __cplusplus
}
#endif

33
web/js/workers/codec/CodecWorker.ts
View file

@ -22,24 +22,7 @@ export function set_initialize_callback(callback: () => Promise<true | string>)
initialize_callback = callback; initialize_callback = callback;
} }
export let codecInstance: CodecWorker; export let codec_instance: CodecWorker;
function printMessageToServerTab(message: string) {
/*
sendMessage({
token: workerCallbackToken,
type: "chatmessage_server",
message: message
});
*/
}
declare function postMessage(message: any): void;
function sendMessage(message: any, origin?: string) {
message["timestamp"] = Date.now();
postMessage(message);
}
let globally_initialized = false; let globally_initialized = false;
let global_initialize_result; let global_initialize_result;
@ -59,18 +42,18 @@ async function handle_message(command: string, data: any) : Promise<string | obj
return {}; return {};
case "initialise": case "initialise":
console.log(prefix + "Got initialize for type " + CodecType[data.type as CodecType]); console.log(prefix + "Initialize for codec %s", CodecType[data.type as CodecType]);
if(!supported_types[data.type]) if(!supported_types[data.type])
return "type unsupported"; return "type unsupported";
try { try {
codecInstance = await supported_types[data.type](data.options); codec_instance = await supported_types[data.type](data.options);
} catch(ex) { } catch(ex) {
console.error(prefix + "Failed to allocate codec: %o", ex); console.error(prefix + "Failed to allocate codec: %o", ex);
return typeof ex === "string" ? ex : "failed to allocate codec"; return typeof ex === "string" ? ex : "failed to allocate codec";
} }
const error = codecInstance.initialise(); const error = codec_instance.initialise();
if(error) return error; if(error) return error;
return {}; return {};
@ -79,7 +62,7 @@ async function handle_message(command: string, data: any) : Promise<string | obj
for(let index = 0; index < encodeArray.length; index++) for(let index = 0; index < encodeArray.length; index++)
encodeArray[index] = data.data[index]; encodeArray[index] = data.data[index];
let encodeResult = codecInstance.encode(encodeArray); let encodeResult = codec_instance.encode(encodeArray);
if(typeof encodeResult === "string") if(typeof encodeResult === "string")
return encodeResult; return encodeResult;
else else
@ -89,13 +72,13 @@ async function handle_message(command: string, data: any) : Promise<string | obj
for(let index = 0; index < decodeArray.length; index++) for(let index = 0; index < decodeArray.length; index++)
decodeArray[index] = data.data[index]; decodeArray[index] = data.data[index];
let decodeResult = codecInstance.decode(decodeArray); let decodeResult = codec_instance.decode(decodeArray);
if(typeof decodeResult === "string") if(typeof decodeResult === "string")
return decodeResult; return decodeResult;
else else
return { data: decodeResult, length: decodeResult.length }; return { data: decodeResult, length: decodeResult.length };
case "reset": case "reset":
codecInstance.reset(); codec_instance.reset();
break; break;
default: default:
return "unknown command"; return "unknown command";
@ -123,7 +106,7 @@ const handle_message_event = (e: MessageEvent) => {
data["timestamp_received"] = received; data["timestamp_received"] = received;
data["timestamp_send"] = Date.now(); data["timestamp_send"] = Date.now();
sendMessage(data, e.origin); postMessage(data, undefined);
}; };
handle_message(e.data.command, e.data.data).then(res => { handle_message(e.data.command, e.data.data).then(res => {
if(token) { if(token) {

64
web/js/workers/codec/OpusCodec.ts
View file

@ -2,8 +2,6 @@ import * as cworker from "./CodecWorker";
import {CodecType} from "tc-backend/web/codec/Codec"; import {CodecType} from "tc-backend/web/codec/Codec";
import {CodecWorker} from "./CodecWorker"; import {CodecWorker} from "./CodecWorker";
const prefix = "OpusWorker";
declare global { declare global {
interface Window { interface Window {
__init_em_module: ((Module: any) => void)[]; __init_em_module: ((Module: any) => void)[];
@ -60,7 +58,7 @@ self.__init_em_module.push(Module => {
if(arguments.length == 1 && arguments[0] == abort_message) if(arguments.length == 1 && arguments[0] == abort_message)
return; /* we don't need to reprint the abort message! */ return; /* we don't need to reprint the abort message! */
console.log("Print: ", ...arguments); console.log(...arguments);
}; };
Module['printErr'] = function() { Module['printErr'] = function() {
@ -72,7 +70,7 @@ self.__init_em_module.push(Module => {
if((arguments[0] as string).indexOf(suppress) != -1) if((arguments[0] as string).indexOf(suppress) != -1)
return; return;
console.error("Error: ",...arguments); console.error(...arguments);
}; };
Module['locateFile'] = file => "../../wasm/" + file; Module['locateFile'] = file => "../../wasm/" + file;
@ -84,22 +82,31 @@ enum OpusType {
RESTRICTED_LOWDELAY = 2051 RESTRICTED_LOWDELAY = 2051
} }
const OPUS_ERROR_CODES = [
"One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG)
"Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL)
"An internal error was detected", //-3 (OPUS_INTERNAL_ERROR)
"The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET)
"Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED)
"An encoder or decoder structure is invalid or already freed", //-6 (OPUS_INVALID_STATE)
"Memory allocation has failed" //-7 (OPUS_ALLOC_FAIL)
];
class OpusWorker implements CodecWorker { class OpusWorker implements CodecWorker {
private channelCount: number; private readonly channelCount: number;
private readonly type: OpusType;
private nativeHandle: any; private nativeHandle: any;
private type: OpusType;
private fn_newHandle: any; private fn_newHandle: any;
private fn_decode: any; private fn_decode: any;
private fn_encode: any; private fn_encode: any;
private fn_reset: any; private fn_reset: any;
private fn_error_message: any;
private bufferSize = 4096 * 2; private buffer_size = 4096 * 2;
private encodeBufferRaw: any; private buffer: any;
private encodeBuffer: Float32Array;
private decodeBufferRaw: any; private encode_buffer: Float32Array;
private decodeBuffer: Uint8Array; private decode_buffer: Uint8Array;
constructor(channelCount: number, type: OpusType) { constructor(channelCount: number, type: OpusType) {
this.channelCount = channelCount; this.channelCount = channelCount;
@ -113,42 +120,39 @@ class OpusWorker implements CodecWorker {
initialise?() : string { initialise?() : string {
this.fn_newHandle = Module.cwrap("codec_opus_createNativeHandle", "number", ["number", "number"]); this.fn_newHandle = Module.cwrap("codec_opus_createNativeHandle", "number", ["number", "number"]);
this.fn_decode = Module.cwrap("codec_opus_decode", "number", ["number", "number", "number", "number"]); this.fn_decode = Module.cwrap("codec_opus_decode", "number", ["number", "number", "number", "number"]);
/* codec_opus_decode(handle, buffer, length, maxlength) */
this.fn_encode = Module.cwrap("codec_opus_encode", "number", ["number", "number", "number", "number"]); this.fn_encode = Module.cwrap("codec_opus_encode", "number", ["number", "number", "number", "number"]);
this.fn_reset = Module.cwrap("codec_opus_reset", "number", ["number"]); this.fn_reset = Module.cwrap("codec_opus_reset", "number", ["number"]);
this.fn_error_message = Module.cwrap("opus_error_message", "string", ["number"]);
this.nativeHandle = this.fn_newHandle(this.channelCount, this.type); this.nativeHandle = this.fn_newHandle(this.channelCount, this.type);
this.encodeBufferRaw = Module._malloc(this.bufferSize); this.buffer = Module._malloc(this.buffer_size);
this.encodeBuffer = new Float32Array(Module.HEAPF32.buffer, this.encodeBufferRaw, this.bufferSize / 4); this.encode_buffer = new Float32Array(Module.HEAPF32.buffer, this.buffer, Math.floor(this.buffer_size / 4));
this.decode_buffer = new Uint8Array(Module.HEAPU8.buffer, this.buffer, this.buffer_size);
this.decodeBufferRaw = Module._malloc(this.bufferSize);
this.decodeBuffer = new Uint8Array(Module.HEAPU8.buffer, this.decodeBufferRaw, this.bufferSize);
return undefined; return undefined;
} }
deinitialise() { } //TODO deinitialise() { } //TODO
decode(data: Uint8Array): Float32Array | string { decode(data: Uint8Array): Float32Array | string {
if (data.byteLength > this.decodeBuffer.byteLength) return "Data to long!"; if (data.byteLength > this.decode_buffer.byteLength) return "supplied data exceeds internal buffer";
this.decodeBuffer.set(data); this.decode_buffer.set(data);
let result = this.fn_decode(this.nativeHandle, this.decodeBuffer.byteOffset, data.byteLength, this.decodeBuffer.byteLength);
if (result < 0) return this.fn_error_message(result); let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, data.byteLength, this.decode_buffer.byteLength);
return Module.HEAPF32.slice(this.decodeBuffer.byteOffset / 4, (this.decodeBuffer.byteOffset / 4) + (result * this.channelCount)); if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown decode error " + result;
return Module.HEAPF32.slice(this.decode_buffer.byteOffset / 4, (this.decode_buffer.byteOffset / 4) + (result * this.channelCount));
} }
encode(data: Float32Array): Uint8Array | string { encode(data: Float32Array): Uint8Array | string {
this.encodeBuffer.set(data); this.encode_buffer.set(data);
let result = this.fn_encode(this.nativeHandle, this.encodeBuffer.byteOffset, data.length, this.encodeBuffer.byteLength); let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, data.length, this.encode_buffer.byteLength);
if (result < 0) return this.fn_error_message(result); if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown encode error " + result;
let buf = Module.HEAP8.slice(this.encodeBuffer.byteOffset, this.encodeBuffer.byteOffset + result);
return Uint8Array.from(buf); return Module.HEAP8.slice(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result);
} }
reset() { reset() {
console.log(prefix + " Reseting opus codec!");
this.fn_reset(this.nativeHandle); this.fn_reset(this.nativeHandle);
} }
} }

2
webpack.config.js
View file

@ -33,12 +33,10 @@ module.exports = {
cwd: process.cwd(), cwd: process.cwd(),
}) })
*/ */
/*
new webpack.optimize.AggressiveSplittingPlugin({ new webpack.optimize.AggressiveSplittingPlugin({
minSize: 1024 * 128, minSize: 1024 * 128,
maxSize: 1024 * 1024 maxSize: 1024 * 1024
}) })
*/
], ],
module: { module: {
rules: [ rules: [