Improved audio de/encoding

ChangeLog.md

@@ -1,6 +1,7 @@
 # Changelog:
 * **12.06.20**
     - Added a copy/paste menu for all HTML input elements
+    - Heavily improved web client audio de/encoding
     
 * **11.06.20**
     - Fixed channel tree deletions

web/js/codec/CodecWorkerMessages.ts

@@ -0,0 +1,92 @@
+import {CodecType} from "tc-backend/web/codec/Codec";
+
+export type CWMessageResponse = {
+    type: "success";
+    token: string;
+
+    response: any;
+
+    timestampReceived: number;
+    timestampSend: number;
+};
+
+export type CWMessageErrorResponse = {
+    type: "error";
+    token: string;
+
+    error: string;
+
+    timestampReceived: number;
+    timestampSend: number;
+}
+
+export type CWMessageCommand<T = CWCommand | CWCommandResponse> = {
+    type: "command";
+    token: string;
+
+    command: keyof T;
+    payload: any;
+}
+
+export type CWMessageNotify = {
+    type: "notify";
+}
+
+export type CWMessage = CWMessageCommand | CWMessageErrorResponse | CWMessageResponse | CWMessageNotify;
+
+/* from handle to worker */
+export interface CWCommand {
+    "global-initialize": {},
+
+
+    "initialise": {
+        type: CodecType,
+        channelCount: number
+    },
+    "reset": {}
+    "finalize": {},
+
+    "decode-payload": {
+        buffer: ArrayBuffer;
+        byteLength: number;
+        byteOffset: number;
+        maxByteLength: number;
+    },
+
+    "encode-payload": {
+        buffer: ArrayBuffer;
+        byteLength: number;
+        byteOffset: number;
+        maxByteLength: number;
+    },
+}
+
+/* from worker to handle */
+export interface CWCommandResponse {
+    "decode-payload-result": {
+        buffer: ArrayBuffer;
+        byteLength: number;
+        byteOffset: number;
+    },
+
+    "encode-payload-result": {
+        buffer: ArrayBuffer;
+        byteLength: number;
+        byteOffset: number;
+    }
+}
+
+export interface CWMessageRelations {
+    "decode-payload": "decode-payload-result",
+    "decode-payload-result": never,
+
+    "encode-payload": "encode-payload-result",
+    "encode-payload-result": never,
+
+    "global-initialize": void,
+    "initialise": void,
+    "reset": void,
+    "finalize": void
+}
+
+export type CWCommandResponseType<T extends keyof CWCommand | keyof CWCommandResponse> = CWMessageRelations[T] extends string ? CWCommandResponse[CWMessageRelations[T]] : CWMessageRelations[T];

web/js/codec/CodecWrapperWorker.ts

@@ -2,18 +2,52 @@ import {BasicCodec} from "./BasicCodec";
 import {CodecType} from "./Codec";
 import * as log from "tc-shared/log";
 import {LogCategory} from "tc-shared/log";
+import {
+    CWCommand,
+    CWCommandResponseType,
+    CWMessage, CWMessageCommand,
+    CWMessageErrorResponse,
+    CWMessageResponse
+} from "tc-backend/web/codec/CodecWorkerMessages";
 
-interface ExecuteResult {
+type MessageTimings = {
-    result?: any;
+    upstream: number;
-    error?: string;
+    downstream: number;
+    handle: number;
+};
 
+interface ExecuteResultBase {
     success: boolean;
 
-    timings: {
+    timings: MessageTimings
-        upstream: number;
+}
-        downstream: number;
+
-        handle: number;
+interface SuccessExecuteResult<T> extends ExecuteResultBase {
+    success: true;
+    result: T;
+}
+
+interface ErrorExecuteResult extends ExecuteResultBase {
+    success: false;
+    error: string;
+}
+type ExecuteResult<T = any> = SuccessExecuteResult<T> | ErrorExecuteResult;
+
+const cachedBufferSize = 1024 * 8;
+let cachedBuffers: ArrayBuffer[] = [];
+function nextCachedBuffer() : ArrayBuffer {
+    if(cachedBuffers.length === 0) {
+        return new ArrayBuffer(cachedBufferSize);
     }
+    return cachedBuffers.pop();
+}
+
+function freeCachedBuffer(buffer: ArrayBuffer) {
+    if(cachedBuffers.length > 32)
+        return;
+    else if(buffer.byteLength < cachedBufferSize)
+        return;
+    cachedBuffers.push(buffer);
 }
 
 export class CodecWrapperWorker extends BasicCodec {
@@ -27,10 +61,8 @@ export class CodecWrapperWorker extends BasicCodec {
     private pending_executes: {[key: string]: {
         timeout?: any;
 
-        timestamp_send: number,
+        timestampSend: number,
-
         resolve: (_: ExecuteResult) => void;
-        reject: (_: any) => void;
     }} = {};
 
     constructor(type: CodecType) {
@@ -61,7 +93,7 @@ export class CodecWrapperWorker extends BasicCodec {
             type: this.type,
             channelCount: this.channelCount,
         })).then(result => {
-            if(result.success) {
+            if(result.success === true) {
                 this._initialized = true;
                 return Promise.resolve(true);
             }
@@ -78,7 +110,7 @@ export class CodecWrapperWorker extends BasicCodec {
     }
 
     deinitialise() {
-        this.execute("deinitialise", {});
+        this.execute("finalize", {});
         this._initialized = false;
         this._initialize_promise = undefined;
     }
@@ -86,44 +118,52 @@ export class CodecWrapperWorker extends BasicCodec {
     async decode(data: Uint8Array): Promise<AudioBuffer> {
         if(!this.initialized()) throw "codec not initialized/initialize failed";
 
-        const result = await this.execute("decodeSamples", { data: data, length: data.length });
+        const cachedBuffer = nextCachedBuffer();
+        new Uint8Array(cachedBuffer).set(data);
+
+        const result = await this.execute("decode-payload", {
+            byteLength: data.byteLength,
+            buffer: cachedBuffer,
+            byteOffset: 0,
+            maxByteLength: cachedBuffer.byteLength
+        }, 5000, [ cachedBuffer ]);
         if(result.timings.downstream > 5 || result.timings.upstream > 5 || result.timings.handle > 5)
             log.warn(LogCategory.VOICE, tr("Worker message stock time: {downstream: %dms, handle: %dms, upstream: %dms}"), result.timings.downstream, result.timings.handle, result.timings.upstream);
 
-        if(!result.success) throw result.error || tr("unknown decode error");
+        if(result.success === false)
+            throw result.error;
 
-        let array = new Float32Array(result.result.length);
+        const chunkLength = result.result.byteLength / this.channelCount;
-        for(let index = 0; index < array.length; index++)
+        const audioBuffer = this._audioContext.createBuffer(this.channelCount, chunkLength / 4, this._codecSampleRate);
-            array[index] = result.result.data[index];
 
-        let audioBuf = this._audioContext.createBuffer(this.channelCount, array.length / this.channelCount, this._codecSampleRate);
+        for(let channel = 0; channel < this.channelCount; channel++) {
-        for (let channel = 0; channel < this.channelCount; channel++) {
+            const buffer = new Float32Array(result.result.buffer, result.result.byteOffset + chunkLength * channel, chunkLength / 4);
-            for (let offset = 0; offset < audioBuf.length; offset++) {
+            audioBuffer.copyToChannel(buffer, channel, 0);
-                audioBuf.getChannelData(channel)[offset] = array[channel + offset * this.channelCount];
-            }
         }
 
-        return audioBuf;
+        freeCachedBuffer(result.result.buffer);
+        return audioBuffer;
     }
 
     async encode(data: AudioBuffer) : Promise<Uint8Array> {
         if(!this.initialized()) throw "codec not initialized/initialize failed";
 
-        let buffer = new Float32Array(this.channelCount * data.length);
+        const buffer = nextCachedBuffer();
-        for (let offset = 0; offset < data.length; offset++) {
+        const f32Buffer = new Float32Array(buffer);
-            for (let channel = 0; channel < this.channelCount; channel++)
+        for(let channel = 0; channel < this.channelCount; channel++)
-                buffer[offset * this.channelCount + channel] = data.getChannelData(channel)[offset];
+            data.copyFromChannel(f32Buffer, channel, data.length * channel);
-        }
+
+        const result = await this.execute("encode-payload", { byteLength: data.length * this.channelCount * 4, buffer: buffer, byteOffset: 0, maxByteLength: buffer.byteLength });
 
-        const result = await this.execute("encodeSamples", { data: buffer, length: buffer.length });
         if(result.timings.downstream > 5 || result.timings.upstream > 5)
             log.warn(LogCategory.VOICE, tr("Worker message stock time: {downstream: %dms, handle: %dms, upstream: %dms}"), result.timings.downstream, result.timings.handle, result.timings.upstream);
-        if(!result.success) throw result.error || tr("unknown encode error");
 
-        let array = new Uint8Array(result.result.length);
+        if(result.success === false)
-        for(let index = 0; index < array.length; index++)
+            throw result.error;
-            array[index] = result.result.data[index];
+
-        return array;
+        const encodedResult = new Uint8Array(result.result.buffer, result.result.byteOffset, result.result.byteLength).slice(0);
+        freeCachedBuffer(result.result.buffer);
+        return encodedResult;
     }
 
     reset() : boolean {
@@ -132,85 +172,118 @@ export class CodecWrapperWorker extends BasicCodec {
         return true;
     }
 
-    private handle_worker_message(message: any) {
+    private handleWorkerMessage(message: CWMessage) {
-        if(!message["token"]) {
+        if(message.type === "notify") {
-            log.error(LogCategory.VOICE, tr("Invalid worker token!"));
+            log.warn(LogCategory.VOICE, tr("Received unknown notify from worker."));
             return;
-        }
+        } else if(message.type === "error") {
-
+            const request = this.pending_executes[message.token];
-        if(message["token"] === "notify") {
+            if(typeof request !== "object") {
-            /* currently not really used */
+                log.warn(LogCategory.VOICE, tr("Received worker execute error for unknown token (%s)"), message.token);
-             if(message["type"] == "chatmessage_server") {
-                //FIXME?
                 return;
             }
-            log.debug(LogCategory.VOICE, tr("Costume callback! (%o)"), message);
+            delete this.pending_executes[message.token];
-            return;
+            clearTimeout(request.timeout);
-        }
 
-        const request = this.pending_executes[message["token"]];
+            const eresponse = message as CWMessageErrorResponse;
-        if(typeof request !== "object") {
+            request.resolve({
-            log.error(LogCategory.VOICE, tr("Received worker execute result for unknown token (%s)"), message["token"]);
+                success: false,
-            return;
+                timings: {
-        }
+                    downstream: eresponse.timestampReceived - request.timestampSend,
-        delete this.pending_executes[message["token"]];
+                    handle: eresponse.timestampSend - eresponse.timestampReceived,
-
+                    upstream: Date.now() - eresponse.timestampSend
-        const result: ExecuteResult = {
+                },
-            success: message["success"],
+                error: eresponse.error
-            error: message["error"],
+            });
-            result: message["result"],
+        } else if(message.type === "success") {
-            timings: {
+            const request = this.pending_executes[message.token];
-                downstream: message["timestamp_received"] - request.timestamp_send,
+            if(typeof request !== "object") {
-                handle: message["timestamp_send"] - message["timestamp_received"],
+                log.warn(LogCategory.VOICE, tr("Received worker execute result for unknown token (%s)"), message.token);
-                upstream: Date.now() - message["timestamp_send"]
+                return;
             }
-        };
+            delete this.pending_executes[message.token];
-        clearTimeout(request.timeout);
+            clearTimeout(request.timeout);
-        request.resolve(result);
+
+            const response = message as CWMessageResponse;
+            request.resolve({
+                success: true,
+                timings: {
+                    downstream: response.timestampReceived - request.timestampSend,
+                    handle: response.timestampSend - response.timestampReceived,
+                    upstream: Date.now() - response.timestampSend
+                },
+                result: response.response
+            });
+        } else if(message.type === "command") {
+            log.warn(LogCategory.VOICE, tr("Received command %s from voice worker. This should never happen!"), (message as CWMessageCommand).command);
+            return;
+        } else {
+            log.warn(LogCategory.VOICE, tr("Received unknown message of type %s from voice worker. This should never happen!"), (message as any).type);
+            return;
+        }
     }
 
-    private handle_worker_error(error: any) {
+    private handleWorkerError() {
         log.error(LogCategory.VOICE, tr("Received error from codec worker. Closing worker."));
         for(const token of Object.keys(this.pending_executes)) {
-            this.pending_executes[token].reject(error);
+            this.pending_executes[token].resolve({
+                success: false,
+                error: tr("worker terminated with an error"),
+                timings: { downstream: 0, handle: 0, upstream: 0}
+            });
             delete this.pending_executes[token];
         }
 
         this._worker = undefined;
     }
 
-    private execute(command: string, data: any, timeout?: number) : Promise<ExecuteResult> {
+    private execute<T extends keyof CWCommand>(command: T, data: CWCommand[T], timeout?: number, transfer?: Transferable[]) : Promise<ExecuteResult<CWCommandResponseType<T>>> {
-        return new Promise<any>((resolve, reject) => {
+        return new Promise<ExecuteResult>(resolve => {
             if(!this._worker) {
-                reject(tr("worker does not exists"));
+                resolve({
+                    success: false,
+                    error: tr("worker does not exists"),
+                    timings: {
+                        downstream: 0,
+                        handle: 0,
+                        upstream: 0
+                    }
+                });
                 return;
             }
 
             const token = this._token_index++ + "_token";
 
-            const payload = {
-                token: token,
-                command: command,
-                data: data,
-            };
-
             this.pending_executes[token] = {
-                timeout: typeof timeout === "number" ? setTimeout(() => reject(tr("timeout for command ") + command), timeout) : undefined,
+                timeout: typeof timeout === "number" ? setTimeout(() => {
+                    delete this.pending_executes[token];
+                    resolve({
+                        success: false,
+                        error: tr("command timed out"),
+                        timings: { upstream: 0, handle: 0, downstream: 0 }
+                    })
+                }, timeout) : undefined,
                 resolve: resolve,
-                reject: reject,
+                timestampSend: Date.now()
-                timestamp_send: Date.now()
             };
 
-            this._worker.postMessage(payload);
+            this._worker.postMessage({
+                command: command,
+                type: "command",
+
+                payload: data,
+                token: token
+            } as CWMessageCommand, transfer);
         });
     }
 
     private async spawn_worker() : Promise<void> {
         this._worker = new Worker("tc-backend/web/workers/codec", { type: "module" });
-        this._worker.onmessage = event => this.handle_worker_message(event.data);
+        this._worker.onmessage = event => this.handleWorkerMessage(event.data);
-        this._worker.onerror = event => this.handle_worker_error(event.error);
+        this._worker.onerror = () => this.handleWorkerError();
 
         const result = await this.execute("global-initialize", {}, 15000);
-        if(!result.success) throw result.error;
+        if(result.success === false)
+            throw result.error;
     }
 }

web/js/workers/codec/CodecWorker.ts

@@ -1,4 +1,11 @@
 import {CodecType} from "tc-backend/web/codec/Codec";
+import {
+    CWMessageCommand,
+    CWCommand,
+    CWMessage,
+    CWMessageResponse,
+    CWMessageErrorResponse, CWCommandResponseType
+} from "tc-backend/web/codec/CodecWorkerMessages";
 
 const prefix = "[CodecWorker] ";
 
@@ -6,8 +13,8 @@ export interface CodecWorker {
     name();
     initialise?() : string;
     deinitialise();
-    decode(data: Uint8Array);
+    decode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array) : number | string;
-    encode(data: Float32Array) : Uint8Array | string;
+    encode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array) : number | string;
 
     reset();
 }
@@ -25,110 +32,152 @@ export function set_initialize_callback(callback: () => Promise<true | string>)
 export let codec_instance: CodecWorker;
 let globally_initialized = false;
 let global_initialize_result;
+let commandTransferableResponse: Transferable[];
 
-/**
+let messageHandlers: { [T in keyof CWCommand]: (message: CWCommand[T]) => Promise<CWCommandResponseType<T>> } = {} as any;
- * @param command
- * @param data
- * @return string on error or object on success
- */
-async function handle_message(command: string, data: any) : Promise<string | object> {
-    switch (command) {
-        case "global-initialize":
-            try {
-                const init_result = globally_initialized ? global_initialize_result : await initialize_callback();
-                globally_initialized = true;
 
-                if(typeof init_result === "string")
+function registerCommandHandler<T extends keyof CWCommand>(command: T, callback: (message: CWCommand[T]) => Promise<CWCommandResponseType<T>>) {
-                    throw init_result;
+    messageHandlers[command as any] = callback;
-            } catch (e) {
-                if(typeof e === "string")
-                    return e;
-                throw e;
-            }
-
-            return {};
-        case "initialise":
-            console.log(prefix + "Initialize for codec %s", CodecType[data.type as CodecType]);
-            if(!supported_types[data.type])
-                return "type unsupported";
-
-            try {
-                codec_instance = await supported_types[data.type](data.options);
-            } catch(ex) {
-                console.error(prefix + "Failed to allocate codec: %o", ex);
-                return typeof ex === "string" ? ex : "failed to allocate codec";
-            }
-
-            const error = codec_instance.initialise();
-            if(error) return error;
-
-            return {};
-        case "encodeSamples":
-            if(!codec_instance)
-                return "codec not initialized/initialize failed";
-
-            let encodeArray = new Float32Array(data.length);
-            for(let index = 0; index < encodeArray.length; index++)
-                encodeArray[index] = data.data[index];
-
-            let encodeResult = codec_instance.encode(encodeArray);
-            if(typeof encodeResult === "string")
-                return encodeResult;
-            else
-                return { data: encodeResult, length: encodeResult.length };
-        case "decodeSamples":
-            if(!codec_instance)
-                return "codec not initialized/initialize failed";
-
-            let decodeArray = new Uint8Array(data.length);
-            for(let index = 0; index < decodeArray.length; index++)
-                decodeArray[index] = data.data[index];
-
-            let decodeResult = codec_instance.decode(decodeArray);
-            if(typeof decodeResult === "string")
-                return decodeResult;
-            else
-                return { data: decodeResult, length: decodeResult.length };
-        case "reset":
-            codec_instance.reset();
-            break;
-        default:
-            return "unknown command";
-    }
 }
 
+const handleOwnerMessage = (e: MessageEvent) => {
+    const timestampReceived = Date.now();
+    const message = e.data as CWMessage;
 
-const handle_message_event = (e: MessageEvent) => {
+    if(message.type === "error" || message.type === "success") {
-    const token = e.data.token;
+        console.warn("%sReceived a command response within the worker. We're not sending any commands so this should not happen!", prefix);
-    const received = Date.now();
+        return;
+    } else if(message.type === "notify") {
+        console.warn("%sReceived a notify within the worker. This should not happen!", prefix);
+        return;
+    } else if(message.type === "command") {
+        const command = message as CWMessageCommand;
 
-    const send_result = result => {
+        const sendExecuteError = error => {
-        const data = {};
+            let errorMessage;
-        if(typeof result === "object") {
+            if(typeof error === "string") {
-            data["result"] = result;
+                errorMessage = error;
-            data["success"] = true;
+            } else if(error instanceof Error) {
-        } else if(typeof result === "string") {
+                console.error("%sMessage handle error: %o", prefix, error);
-            data["error"] = result;
+                errorMessage = error.message;
-            data["success"] = false;
+            } else {
-        } else {
+                console.error("%sMessage handle error: %o", prefix, error);
-            data["error"] = "invalid result";
+                errorMessage = "lookup the console";
-            data["success"] = false;
+            }
-        }
-        data["token"] = token;
-        data["timestamp_received"] = received;
-        data["timestamp_send"] = Date.now();
 
-        postMessage(data, undefined);
+            postMessage({
-    };
+                type: "error",
-    handle_message(e.data.command, e.data.data).then(res => {
+                error: errorMessage,
-        if(token) {
+
-            send_result(res);
+                timestampReceived: timestampReceived,
+                timestampSend: Date.now(),
+
+                token: command.token
+            } as CWMessageErrorResponse, undefined, commandTransferableResponse);
+        };
+
+        const sendExecuteResult = result => {
+            postMessage({
+                type: "success",
+                response: result,
+
+                timestampReceived: timestampReceived,
+                timestampSend: Date.now(),
+
+                token: command.token
+            } as CWMessageResponse, undefined);
+        };
+
+        const handler = messageHandlers[message.command as any];
+        if(!handler) {
+            sendExecuteError("unknown command");
+            return;
         }
-    }).catch(error => {
+
-        console.warn("An error has been thrown while handing command %s: %o", e.data.command, error);
+        handler(command.payload).then(sendExecuteResult).catch(sendExecuteError);
-        if(token) {
+    }
-            send_result(typeof error === "string" ? error : "unexpected exception has been thrown");
-        }
-    });
 };
-addEventListener("message", handle_message_event);
+addEventListener("message", handleOwnerMessage);
+
+
+/* command handlers */
+registerCommandHandler("global-initialize", async () => {
+    const init_result = globally_initialized ? global_initialize_result : await initialize_callback();
+    globally_initialized = true;
+
+    if(typeof init_result === "string")
+        throw init_result;
+});
+
+registerCommandHandler("initialise", async data => {
+    console.log(prefix + "Initialize for codec %s", CodecType[data.type as CodecType]);
+    if(!supported_types[data.type])
+        throw "type unsupported";
+
+    try {
+        codec_instance = await supported_types[data.type](data);
+    } catch(ex) {
+        console.error("%sFailed to allocate codec: %o", prefix, ex);
+        throw typeof ex === "string" ? ex : "failed to allocate codec";
+    }
+
+    const error = codec_instance.initialise();
+    if(error)
+        throw error;
+});
+
+registerCommandHandler("reset", async () => {
+    codec_instance.reset();
+});
+
+registerCommandHandler("finalize", async () => {
+    /* memory will be cleaned up by its own */
+});
+
+let responseBuffer: Uint8Array;
+const popResponseBuffer = () => { const temp = responseBuffer; responseBuffer = undefined; return temp; }
+registerCommandHandler("decode-payload", async data => {
+    if(!codec_instance)
+        throw "codec not initialized/initialize failed";
+
+    const byteLength = codec_instance.decode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), length => {
+        if(length > data.maxByteLength)
+            throw "source buffer too small to hold the result";
+            //return responseBuffer = new Uint8Array(length);
+        return responseBuffer = new Uint8Array(data.buffer, 0, data.maxByteLength);
+    });
+    const buffer = popResponseBuffer();
+    if(typeof byteLength === "string") {
+        throw byteLength;
+    }
+
+    commandTransferableResponse = [buffer.buffer];
+    return {
+        buffer: buffer.buffer,
+        byteLength: byteLength,
+        byteOffset: 0,
+    };
+});
+
+registerCommandHandler("encode-payload", async data => {
+    if(!codec_instance)
+        throw "codec not initialized/initialize failed";
+
+    const byteLength = codec_instance.encode(new Uint8Array(data.buffer, data.byteOffset, data.byteLength), length => {
+        if(length > data.maxByteLength)
+            throw "source buffer too small to hold the result";
+            //return responseBuffer = new Uint8Array(length);
+        return responseBuffer = new Uint8Array(data.buffer, 0, data.maxByteLength);
+    });
+    const buffer = popResponseBuffer();
+    if(typeof byteLength === "string") {
+        throw byteLength;
+    }
+
+    commandTransferableResponse = [buffer.buffer];
+    return {
+        buffer: buffer.buffer,
+        byteLength: byteLength,
+        byteOffset: 0
+    };
+});

web/js/workers/codec/OpusCodec.ts

@@ -1,7 +1,6 @@
 import * as cworker from "./CodecWorker";
 import {CodecType} from "tc-backend/web/codec/Codec";
 import {CodecWorker} from "./CodecWorker";
-import {type} from "os";
 
 declare global {
     interface Window {
@@ -113,7 +112,7 @@ enum OpusType {
 
 const OPUS_ERROR_CODES = [
     "One or more invalid/out of range arguments", //-1 (OPUS_BAD_ARG)
-    "Not enough bytes allocated in the buffer", //-2 (OPUS_BUFFER_TOO_SMALL)
+    "Not enough bytes allocated in the target buffer", //-2 (OPUS_BUFFER_TOO_SMALL)
     "An internal error was detected", //-3 (OPUS_INTERNAL_ERROR)
     "The compressed data passed is corrupted", //-4 (OPUS_INVALID_PACKET)
     "Invalid/unsupported request number", //-5 (OPUS_UNIMPLEMENTED)
@@ -162,23 +161,33 @@ class OpusWorker implements CodecWorker {
 
     deinitialise() { } //TODO
 
-    decode(data: Uint8Array): Float32Array | string {
+    decode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string {
-        if (data.byteLength > this.decode_buffer.byteLength) return "supplied data exceeds internal buffer";
+        if (buffer.byteLength > this.decode_buffer.byteLength)
-        this.decode_buffer.set(data);
+            return "supplied data exceeds internal buffer";
 
-        let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, data.byteLength, this.decode_buffer.byteLength);
+        this.decode_buffer.set(buffer);
+
+        let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, buffer.byteLength, this.decode_buffer.byteLength);
         if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown decode error " + result;
 
-        return Module.HEAPF32.slice(this.decode_buffer.byteOffset / 4, (this.decode_buffer.byteOffset / 4) + (result * this.channelCount));
+        const resultByteLength = result * this.channelCount * 4;
+        const resultBuffer = responseBuffer(resultByteLength);
+        resultBuffer.set(this.decode_buffer.subarray(0, resultByteLength), 0);
+        return resultByteLength;
     }
 
-    encode(data: Float32Array): Uint8Array | string {
+    encode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string {
-        this.encode_buffer.set(data);
+        if (buffer.byteLength > this.decode_buffer.byteLength)
+            return "supplied data exceeds internal buffer";
 
-        let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, data.length, this.encode_buffer.byteLength);
+        this.encode_buffer.set(buffer);
+
+        let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, buffer.byteLength, this.encode_buffer.byteLength);
         if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown encode error " + result;
 
-        return Module.HEAP8.slice(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result);
+        const resultBuffer = responseBuffer(result);
+        resultBuffer.set(Module.HEAP8.subarray(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result));
+        return result;
     }
 
     reset() {

web/native-codec/src/ilarvecon.cpp

@@ -0,0 +1,68 @@
+//
+// Created by WolverinDEV on 12/06/2020.
+//
+
+/* source and target should not be intersecting! */
+template <size_t kChannelCount>
+void sequenced2interleaved(float* source, float* target, size_t sample_count) {
+    #pragma unroll
+    for(size_t channel = 0; channel < kChannelCount; channel++) {
+        auto src_ptr = source + channel * sample_count;
+        auto dest_ptr = target + channel;
+
+        auto samples_left = sample_count;
+        while(samples_left--) {
+            *dest_ptr = *src_ptr;
+
+            src_ptr++;
+            dest_ptr += kChannelCount;
+        }
+    }
+}
+
+/* source and target should not be intersecting! */
+template <size_t kChannelCount>
+void interleaved2sequenced(float* source, float* target, size_t sample_count) {
+    #pragma unroll
+    for(size_t channel = 0; channel < kChannelCount; channel++) {
+        auto src_ptr = source + channel;
+        auto dest_ptr = target + channel * sample_count;
+
+        auto samples_left = sample_count;
+        while(samples_left--) {
+            *dest_ptr = *src_ptr;
+
+            src_ptr += kChannelCount;
+            dest_ptr++;
+        }
+    }
+}
+
+#define kTempBufferSize (1024 * 8)
+
+/* since js is single threaded we need no lock here */
+float temp_buffer[kTempBufferSize];
+
+template <size_t kChannelCount>
+void interleaved2sequenced_intersecting(float* buffer, size_t sample_count) {
+    auto temp = temp_buffer;
+    if(sample_count * kChannelCount > kTempBufferSize)
+        temp = (float*) malloc(sample_count * sizeof(float) * kChannelCount);
+
+    memcpy(temp, buffer, sample_count * sizeof(float) * kChannelCount);
+    interleaved2sequenced<kChannelCount>(temp, buffer, sample_count);
+    if(temp != temp_buffer)
+        free(temp);
+}
+
+template <size_t kChannelCount>
+void sequenced2interleaved_intersecting(float* buffer, size_t sample_count) {
+    auto temp = temp_buffer;
+    if(sample_count * kChannelCount > kTempBufferSize)
+        temp = (float*) malloc(sample_count * sizeof(float) * kChannelCount);
+
+    memcpy(temp, buffer, sample_count * sizeof(float) * kChannelCount);
+    sequenced2interleaved<kChannelCount>(temp, buffer, sample_count);
+    if(temp != temp_buffer)
+        free(temp);
+}

web/native-codec/src/opus.cpp

@@ -3,6 +3,7 @@
 #include <string_view>
 #include <emscripten.h>
 #include <string>
+#include "./ilarvecon.cpp"
 
 typedef std::unique_ptr<OpusEncoder, decltype(opus_encoder_destroy)*> opus_encoder_t;
 typedef std::unique_ptr<OpusDecoder, decltype(opus_decoder_destroy)*> opus_decoder_t;
@@ -82,16 +83,22 @@ void codec_opus_deleteNativeHandle(OpusHandle *codec) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t length, size_t maxLength) {
+int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t byte_length, size_t maxLength) {
-    auto result = opus_encode_float(&*handle->encoder, (float *) buffer, length / handle->channelCount, buffer, maxLength);
+    if(handle->channelCount == 2)
+        sequenced2interleaved_intersecting<2>((float *) buffer, byte_length / (sizeof(float) * 2));
+
+    auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / handle->channelCount, buffer, maxLength);
     if (result < 0) return result;
     return result;
 }
 
 EMSCRIPTEN_KEEPALIVE
-int codec_opus_decode(OpusHandle *handle, uint8_t *buffer, size_t length, size_t maxLength) {
+int codec_opus_decode(OpusHandle *handle, uint8_t *buffer, size_t byte_length, size_t buffer_max_byte_length) {
-    auto result = opus_decode_float(&*handle->decoder, buffer, length, (float *) buffer, maxLength / sizeof(float) / handle->channelCount, false);
+    auto result = opus_decode_float(&*handle->decoder, buffer, byte_length, (float *) buffer, buffer_max_byte_length / sizeof(float) / handle->channelCount, false);
     if (result < 0) return result;
+
+    if(handle->channelCount == 2)
+        interleaved2sequenced_intersecting<2>((float *) buffer, result);
     return result;
 }