TeaSpeak
/
TeaWeb
1
0
Fork 0
Code Issues 1 Pull Requests Activity

Audio recive works perfect for opus voice, send fails (laggy) 

TODO:
- Opus music
- Client volumes
- Some lagecy settings
canary
WolverinDEV 2018-03-02 23:39:12 +01:00
parent debab3baaf
commit 96c3aabec9
7 changed files with 300 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
.idea/workspace.xml
View File

@ -3,11 +3,6 @@
<component name="ChangeListManager">
<list default="true" id="978d055d-27d3-431a-bd34-e5e79bb273b3" name="Default" comment="">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change beforePath="$PROJECT_DIR$/asm/generated/TeaWeb-Native.js" afterPath="$PROJECT_DIR$/asm/generated/TeaWeb-Native.js" />
<change beforePath="$PROJECT_DIR$/asm/generated/libopus.js" afterPath="$PROJECT_DIR$/asm/generated/libopus.js" />
<change beforePath="$PROJECT_DIR$/asm/libs/opus" afterPath="$PROJECT_DIR$/asm/libs/opus" />
<change beforePath="$PROJECT_DIR$/asm/make_opus.sh" afterPath="$PROJECT_DIR$/asm/make_opus.sh" />
<change beforePath="$PROJECT_DIR$/asm/src/WebASMTest.cpp" afterPath="$PROJECT_DIR$/asm/src/WebASMTest.cpp" />
<change beforePath="$PROJECT_DIR$/js/codec/Codec.js" afterPath="$PROJECT_DIR$/js/codec/Codec.js" />
<change beforePath="$PROJECT_DIR$/js/codec/Codec.js.map" afterPath="$PROJECT_DIR$/js/codec/Codec.js.map" />
<change beforePath="$PROJECT_DIR$/js/codec/Codec.ts" afterPath="$PROJECT_DIR$/js/codec/Codec.ts" />
@ -24,21 +19,11 @@
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="index.html" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/index.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2898">
<caret line="161" column="19" lean-forward="false" selection-start-line="161" selection-start-column="19" selection-end-line="161" selection-end-column="19" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Codec.ts" pinned="false" current-in-tab="false">
<file leaf-file-name="Codec.ts" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/js/codec/Codec.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="751">
<caret line="99" column="102" lean-forward="false" selection-start-line="99" selection-start-column="102" selection-end-line="99" selection-end-column="102" />
<state relative-caret-position="198">
<caret line="14" column="35" lean-forward="true" selection-start-line="14" selection-start-column="35" selection-end-line="14" selection-end-column="35" />
<folding />
</state>
</provider>
@ -59,8 +44,8 @@
<file leaf-file-name="connection.ts" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/js/connection.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<state relative-caret-position="612">
<caret line="34" column="24" lean-forward="true" selection-start-line="34" selection-start-column="24" selection-end-line="34" selection-end-column="24" />
<folding>
<element signature="n#!!doc" expanded="true" />
</folding>
@ -81,8 +66,18 @@
<file leaf-file-name="lib.es6.d.ts" pinned="false" current-in-tab="false">
<entry file="file://$APPLICATION_HOME_DIR$/plugins/JavaScriptLanguage/jsLanguageServicesImpl/external/lib.es6.d.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="14435" column="40" lean-forward="false" selection-start-line="14435" selection-start-column="34" selection-end-line="14435" selection-end-column="40" />
<state relative-caret-position="647">
<caret line="7148" column="4" lean-forward="false" selection-start-line="7148" selection-start-column="4" selection-end-line="7148" selection-end-column="4" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="settings.ts" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/js/settings.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
@ -98,11 +93,11 @@
</provider>
</entry>
</file>
<file leaf-file-name="voice.ts" pinned="false" current-in-tab="true">
<file leaf-file-name="voice.ts" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/js/voice.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1201">
<caret line="320" column="33" lean-forward="false" selection-start-line="320" selection-start-column="33" selection-end-line="320" selection-end-column="33" />
<state relative-caret-position="594">
<caret line="51" column="10" lean-forward="false" selection-start-line="51" selection-start-column="10" selection-end-line="51" selection-end-column="10" />
<folding />
</state>
</provider>
@ -118,18 +113,26 @@
</provider>
</entry>
</file>
<file leaf-file-name="package.json" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/package.json">
<file leaf-file-name="chat.ts" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/js/chat.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
<state relative-caret-position="513">
<caret line="32" column="41" lean-forward="false" selection-start-line="32" selection-start-column="41" selection-end-line="32" selection-end-column="41" />
<folding>
<marker date="1519748729173" expanded="true" signature="1975:1985" ph="..." />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>createBuffer</find>
<find>on_data</find>
</findStrings>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
@ -155,10 +158,10 @@
</component>
<component name="PhpWorkspaceProjectConfiguration" backward_compatibility_performed="true" />
<component name="ProjectFrameBounds" extendedState="6">
<option name="x" value="2028" />
<option name="y" value="357" />
<option name="width" value="1855" />
<option name="height" value="1084" />
<option name="x" value="2918" />
<option name="y" value="-4" />
<option name="width" value="1853" />
<option name="height" value="1076" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
@ -252,28 +255,30 @@
<option name="presentableId" value="Default" />
<updated>1519749313999</updated>
<workItem from="1519749316100" duration="9698000" />
<workItem from="1520012895408" duration="6591000" />
<workItem from="1520012895408" duration="13484000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="16289000" />
<option name="totallyTimeSpent" value="23182000" />
</component>
<component name="ToolWindowManager">
<frame x="2028" y="357" width="3777" height="2172" extended-state="6" />
<frame x="1985" y="-4" width="3775" height="2164" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="npm" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="TypeScript" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32993197" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="TypeScript" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32993197" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24980132" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24980132" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
<window_info id="Docker" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
@ -281,7 +286,6 @@
<window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
<window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
@ -336,7 +340,9 @@
<option name="myLimit" value="2678400000" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager />
<breakpoint-manager>
<option name="time" value="1" />
</breakpoint-manager>
<watches-manager />
</component>
<component name="editorHistoryManager">
@ -460,16 +466,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/connection.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="n#!!doc" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/client.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
@ -488,6 +484,14 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/settings.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/ui/client.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="324">
@ -496,26 +500,46 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/codec/Codec.ts">
<entry file="file://$PROJECT_DIR$/js/chat.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="751">
<caret line="99" column="102" lean-forward="false" selection-start-line="99" selection-start-column="102" selection-end-line="99" selection-end-column="102" />
<folding />
<state relative-caret-position="513">
<caret line="32" column="41" lean-forward="false" selection-start-line="32" selection-start-column="41" selection-end-line="32" selection-end-column="41" />
<folding>
<marker date="1519748729173" expanded="true" signature="1975:1985" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/connection.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="612">
<caret line="34" column="24" lean-forward="true" selection-start-line="34" selection-start-column="24" selection-end-line="34" selection-end-column="24" />
<folding>
<element signature="n#!!doc" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$APPLICATION_HOME_DIR$/plugins/JavaScriptLanguage/jsLanguageServicesImpl/external/lib.es6.d.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="442">
<caret line="14435" column="40" lean-forward="false" selection-start-line="14435" selection-start-column="34" selection-end-line="14435" selection-end-column="40" />
<state relative-caret-position="647">
<caret line="7148" column="4" lean-forward="false" selection-start-line="7148" selection-start-column="4" selection-end-line="7148" selection-end-column="4" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/voice.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1201">
<caret line="320" column="33" lean-forward="false" selection-start-line="320" selection-start-column="33" selection-end-line="320" selection-end-column="33" />
<state relative-caret-position="594">
<caret line="51" column="10" lean-forward="false" selection-start-line="51" selection-start-column="10" selection-end-line="51" selection-end-column="10" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/js/codec/Codec.ts">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="198">
<caret line="14" column="35" lean-forward="true" selection-start-line="14" selection-start-column="35" selection-end-line="14" selection-end-column="35" />
<folding />
</state>
</provider>

62
js/codec/Codec.js
View File

@ -5,10 +5,13 @@ class SampleBuffer {
}
}
class Codec {
constructor() {
constructor(codecSampleRate) {
this.on_encoded_data = ($) => { };
this._sampleBuffer = [];
this.sampleRate = 120;
this.samplesPerUnit = 960;
this._codecSampleRate = codecSampleRate;
this._decodeResampler = new Resampler();
this._encodeResampler = new Resampler(codecSampleRate);
}
bufferedSamples(max = 0) {
let value = 0;
@ -17,15 +20,19 @@ class Codec {
console.log(value + " / " + max);
return value;
}
encodeSamples(array) {
console.log("encode");
this._sampleBuffer.push(new SampleBuffer(array));
while (this.bufferedSamples(this.sampleRate) >= this.sampleRate) {
let buffer = new Float32Array(this.sampleRate);
encodeSamples(pcm) {
this._encodeResampler.resample(pcm).then(buffer => this.encodeSamples0(buffer))
.catch(error => console.error("Could not resample PCM data for codec. Error:" + error));
}
encodeSamples0(buffer) {
console.log(buffer);
this._sampleBuffer.push(new SampleBuffer(buffer.getChannelData(0))); //TODO multi channel!
while (this.bufferedSamples(this.samplesPerUnit) >= this.samplesPerUnit) {
let buffer = new Float32Array(this.samplesPerUnit);
let index = 0;
while (index < this.sampleRate) {
while (index < this.samplesPerUnit) {
let buf = this._sampleBuffer[0];
let len = Math.min(buf.buffer.length - buf.index, this.sampleRate - index);
let len = Math.min(buf.buffer.length - buf.index, this.samplesPerUnit - index);
buffer.set(buf.buffer.subarray(buf.index, buf.index + len));
index += len;
buf.index += len;
@ -37,14 +44,17 @@ class Codec {
if (result instanceof Uint8Array)
this.on_encoded_data(result);
else
return result;
console.error("[Codec][" + this.name() + "] Could not encode buffer. Result: " + result);
}
return true;
}
decodeSamples(data) {
return this.decode(data).then(buffer => this._decodeResampler.resample(buffer));
}
}
class OpusCodec extends Codec {
constructor() {
super();
super(48000);
this.channelCount = 1;
}
name() {
@ -56,21 +66,25 @@ class OpusCodec extends Codec {
this.fn_encode = Module.cwrap("codec_opus_encode", "number", ["pointer", "pointer", "number", "number"]);
this.nativeHandle = this.fn_newHandle(1);
}
deinitialise() {
}
deinitialise() { } //TODO
decode(data) {
let maxBytes = 4096;
let buffer = Module._malloc(maxBytes);
let heapBytes = new Uint8Array(Module.HEAPU8.buffer, buffer, maxBytes);
heapBytes.set(data);
let result = this.fn_decode(this.nativeHandle, heapBytes.byteOffset, data.byteLength, maxBytes);
if (result < 0) {
return new Promise((resolve, reject) => {
let maxBytes = 4096;
let buffer = Module._malloc(maxBytes);
let heapBytes = new Uint8Array(Module.HEAPU8.buffer, buffer, maxBytes);
heapBytes.set(data);
let result = this.fn_decode(this.nativeHandle, heapBytes.byteOffset, data.byteLength, maxBytes);
if (result < 0) {
Module._free(buffer);
reject("invalid result on decode (" + result + ")");
return;
}
let buf = Module.HEAPF32.slice(heapBytes.byteOffset / 4, (heapBytes.byteOffset / 4) + (result * this.channelCount));
Module._free(buffer);
return "invalid result on decode (" + result + ")";
}
let buf = Module.HEAPF32.slice(heapBytes.byteOffset / 4, (heapBytes.byteOffset / 4) + (result * this.channelCount));
Module._free(buffer);
return buf;
let audioBuf = AudioController.globalContext.createBuffer(this.channelCount, result, this._codecSampleRate);
audioBuf.copyToChannel(buf, 0);
resolve(audioBuf);
});
}
encode(data) {
let maxBytes = data.byteLength;

2
js/codec/Codec.js.map
View File

File diff suppressed because one or more lines are too long

78
js/codec/Codec.ts
View File

@ -11,20 +11,27 @@ class SampleBuffer {
abstract class Codec {
on_encoded_data: (Uint8Array) => void = ($) => {};
protected _decodeResampler: Resampler;
protected _encodeResampler: Resampler;
protected _codecSampleRate: number;
protected _sampleBuffer: SampleBuffer[] = [];
sampleRate: number = 120;
constructor(){}
samplesPerUnit: number = 960;
protected constructor(codecSampleRate: number){
this._codecSampleRate = codecSampleRate;
this._decodeResampler = new Resampler();
this._encodeResampler = new Resampler(codecSampleRate);
}
abstract name() : string;
abstract initialise();
abstract deinitialise();
abstract decode(data: Uint8Array) : Float32Array | string;
protected abstract decode(data: Uint8Array) : Promise<AudioBuffer>;
protected abstract encode(data: Float32Array) : Uint8Array | string;
protected bufferedSamples(max: number = 0) : number {
let value = 0;
for(let i = 0; i < this._sampleBuffer.length && value < max; i++)
@ -33,16 +40,21 @@ abstract class Codec {
return value;
}
encodeSamples(array: Float32Array) : boolean | string {
console.log("encode");
this._sampleBuffer.push(new SampleBuffer(array));
encodeSamples(pcm: AudioBuffer) {
this._encodeResampler.resample(pcm).then(buffer => this.encodeSamples0(buffer))
.catch(error => console.error("Could not resample PCM data for codec. Error:" + error));
}
while(this.bufferedSamples(this.sampleRate) >= this.sampleRate) {
let buffer = new Float32Array(this.sampleRate);
private encodeSamples0(buffer: AudioBuffer) {
console.log(buffer);
this._sampleBuffer.push(new SampleBuffer(buffer.getChannelData(0))); //TODO multi channel!
while(this.bufferedSamples(this.samplesPerUnit) >= this.samplesPerUnit) {
let buffer = new Float32Array(this.samplesPerUnit);
let index = 0;
while(index < this.sampleRate) {
while(index < this.samplesPerUnit) {
let buf = this._sampleBuffer[0];
let len = Math.min(buf.buffer.length - buf.index, this.sampleRate - index);
let len = Math.min(buf.buffer.length - buf.index, this.samplesPerUnit - index);
buffer.set(buf.buffer.subarray(buf.index, buf.index + len));
index += len;
buf.index += len;
@ -53,10 +65,14 @@ abstract class Codec {
let result = this.encode(buffer);
if(result instanceof Uint8Array) this.on_encoded_data(result);
else return result;
else console.error("[Codec][" + this.name() + "] Could not encode buffer. Result: " + result);
}
return true;
}
decodeSamples(data: Uint8Array) : Promise<AudioBuffer> {
return this.decode(data).then(buffer => this._decodeResampler.resample(buffer));
}
}
class OpusCodec extends Codec {
@ -68,7 +84,7 @@ class OpusCodec extends Codec {
private fn_encode: any;
constructor() {
super();
super(48000);
}
name(): string {
@ -83,23 +99,27 @@ class OpusCodec extends Codec {
this.nativeHandle = this.fn_newHandle(1);
}
deinitialise() {
deinitialise() { } //TODO
}
decode(data: Uint8Array): Float32Array | string {
let maxBytes = 4096;
let buffer = Module._malloc(maxBytes);
let heapBytes = new Uint8Array(Module.HEAPU8.buffer, buffer, maxBytes);
heapBytes.set(data);
let result = this.fn_decode(this.nativeHandle, heapBytes.byteOffset, data.byteLength, maxBytes);
if(result < 0) {
decode(data: Uint8Array): Promise<AudioBuffer> {
return new Promise<AudioBuffer>((resolve, reject) => {
let maxBytes = 4096;
let buffer = Module._malloc(maxBytes);
let heapBytes = new Uint8Array(Module.HEAPU8.buffer, buffer, maxBytes);
heapBytes.set(data);
let result = this.fn_decode(this.nativeHandle, heapBytes.byteOffset, data.byteLength, maxBytes);
if(result < 0) {
Module._free(buffer);
reject("invalid result on decode (" + result + ")");
return;
}
let buf = Module.HEAPF32.slice(heapBytes.byteOffset / 4, (heapBytes.byteOffset / 4) + (result * this.channelCount));
Module._free(buffer);
return "invalid result on decode (" + result + ")";
}
let buf = Module.HEAPF32.slice(heapBytes.byteOffset / 4, (heapBytes.byteOffset / 4) + (result * this.channelCount));
Module._free(buffer);
return buf;
let audioBuf = AudioController.globalContext.createBuffer(this.channelCount, result, this._codecSampleRate);
audioBuf.copyToChannel(buf, 0);
resolve(audioBuf);
});
}
encode(data: Float32Array): Uint8Array | string {

122
js/voice.js
View File

@ -1,17 +1,44 @@
/// <reference path="client.ts" />
class VoiceConnection {
/*
private _voicePacketBuffer: Uint8Array[] = [];
private _voicePacketSender: NodeJS.Timer;
private _triggered = false;
*/
constructor(client) {
this.vpacketId = 0;
this.client = client;
this.voiceRecorder = new VoiceRecorder(this);
this.voiceRecorder.on_data = data => this.sendPCMData(data);
this.voiceRecorder.on_data = data => this.handleVoiceData(data);
this.codec = new OpusCodec();
this.codec.initialise();
this.codec.on_encoded_data = buffer => {
if (this.dataChannel) {
console.log("Send buffer");
this.dataChannel.send(buffer);
this.vpacketId++;
if (this.vpacketId > 65535)
this.vpacketId = 0;
let packet = new Uint8Array(buffer.byteLength + 2 + 3);
packet[0] = this.vpacketId < 6 ? 1 : 0; //Flag header
packet[1] = 0; //Flag fragmented
packet[2] = (this.vpacketId >> 8) & 0xFF; //HIGHT(voiceID)
packet[3] = (this.vpacketId >> 0) & 0xFF; //LOW (voiceID)
packet[4] = 4; //Codec
packet.set(buffer, 5);
//this._voicePacketBuffer.push(packet);
this.dataChannel.send(packet);
}
};
/*
this._voicePacketSender = setInterval(() => {
if(this._voicePacketBuffer.length > 5 || this._triggered) {
let packet = this._voicePacketBuffer.pop_front();
if (packet) {
this.dataChannel.send(packet);
}
this._triggered = this._voicePacketBuffer.length > 0;
}
}, 20);
*/
}
createSession() {
const config = {};
@ -66,26 +93,24 @@ class VoiceConnection {
}
onDataChannelMessage(message) {
let bin = new Uint8Array(message.data);
let clientId = bin[0] << 8 | bin[1];
console.log("Client id " + clientId);
let clientId = bin[2] << 8 | bin[3];
let packetId = bin[0] << 8 | bin[1];
let codec = bin[4];
console.log("Client id " + clientId + " PacketID " + packetId + " Codec: " + codec);
let client = this.client.channelTree.findClient(clientId);
if (!client) {
console.error("Having voice from unknown client? (ClientID: " + clientId + ")");
return;
}
var encodedData = new Uint8Array(message.data, 4);
let result = this.codec.decode(encodedData);
if (result instanceof Float32Array)
client.getAudioController().play(result);
else
console.log("Invalid decode " + result);
var encodedData = new Uint8Array(message.data, 5);
this.codec.decodeSamples(encodedData).then(buffer => client.getAudioController().play(buffer)).catch(error => {
console.error("Could not playback client's (" + clientId + ") audio (" + error + ")");
});
}
sendPCMData(data) {
/*
let result = this.codec.encodeSamples(data);
if(!result) console.error("Could not encode audio: " + result);
*/
this.client.getClient().getAudioController().play(data);
handleVoiceData(data) {
setTimeout(() => {
this.codec.encodeSamples(data);
}, 1);
}
}
class VoiceRecorder {
@ -100,8 +125,9 @@ class VoiceRecorder {
this.processor = this.audioContext.createScriptProcessor(VoiceRecorder.BUFFER_SIZE, VoiceRecorder.CHANNELS, VoiceRecorder.CHANNELS);
const _this = this;
this.processor.addEventListener('audioprocess', ev => {
console.log(ev.inputBuffer);
if (_this.microphoneStream)
this.on_data(ev.inputBuffer.getChannelData(VoiceRecorder.CHANNEL));
this.on_data(ev.inputBuffer);
});
//Not needed but make sure we have data for the preprocessor
this.mute = this.audioContext.createGain();
@ -156,10 +182,15 @@ class VoiceRecorder {
}
VoiceRecorder.CHANNEL = 0;
VoiceRecorder.CHANNELS = 1;
VoiceRecorder.BUFFER_SIZE = 4096;
VoiceRecorder.BUFFER_SIZE = 16384 / 2;
class AudioController {
static get globalContext() {
if (this._globalContext)
return this._globalContext;
this._globalContext = new AudioContext();
return this._globalContext;
}
constructor() {
this.resambler = new Resampler();
this.speakerContext = AudioController.globalContext;
this.nextTime = 0;
this.last = 0;
@ -169,23 +200,9 @@ class AudioController {
this.onSpeaking = function () { };
this.onSilence = function () { };
}
static get globalContext() {
if (this._globalContext)
return this._globalContext;
this._globalContext = new AudioContext();
return this._globalContext;
}
play(pcm) {
//let buffer = this.speakerContext.createBuffer(1, 960, 48000);
//buffer.copyToChannel(pcm, 0);
this.resambler.resample(pcm, (buffer) => this.play0(buffer));
//this.play0(buffer);
}
play0(buffer) {
//960
console.log(buffer);
//let buffer = this.speakerContext.createBuffer(1, 960, 44100);
//buffer.copyToChannel(pcm, 0);
play(buffer) {
if (buffer.sampleRate != this.speakerContext.sampleRate)
console.warn("[AudioController] Source sample rate isnt equal to playback sample rate!");
this.audioCache.push(buffer);
let currentTime = new Date().getTime();
if ((currentTime - this.last) > 50) {
@ -207,8 +224,8 @@ class AudioController {
;
playCache(cache) {
while (cache.length) {
var buffer = cache.shift();
var source = this.speakerContext.createBufferSource();
let buffer = cache.shift();
let source = this.speakerContext.createBufferSource();
source.buffer = buffer;
source.connect(this.speakerContext.destination);
if (this.nextTime == 0) {
@ -234,26 +251,19 @@ class AudioController {
}
}
class Resampler {
constructor() {
constructor(targetSampleRate = 44100) {
this.targetSampleRate = targetSampleRate;
}
resample(pcm, callback) {
/*
let buffer = AudioController.globalContext.createBuffer(1, pcm.length, 48000);
buffer.copyToChannel(pcm, 0);
callback(buffer);
*/
this.context = new OfflineAudioContext(1, 882, 44100);
let buffer = this.context.createBuffer(1, pcm.length, 48000);
buffer.copyToChannel(pcm, 0);
let source = this.context.createBufferSource();
resample(buffer) {
if (buffer.sampleRate == this.targetSampleRate)
return new Promise(resolve => resolve(buffer));
console.log(this.targetSampleRate);
let context = new OfflineAudioContext(1, Math.ceil(buffer.length * this.targetSampleRate / buffer.sampleRate), this.targetSampleRate);
let source = context.createBufferSource();
source.buffer = buffer;
source.connect(this.context.destination);
source.connect(context.destination);
source.start(0);
//console.log(source.buffer.getChannelData(0));
this.context.startRendering().then(e => callback(e)).catch(error => {
console.error("Could not resample audio");
console.error(error);
});
return context.startRendering();
}
}
//# sourceMappingURL=voice.js.map

2
js/voice.js.map
View File

File diff suppressed because one or more lines are too long

116
js/voice.ts
View File

@ -9,19 +9,47 @@ class VoiceConnection {
start: number;
codec: Codec;
vpacketId: number = 0;
/*
private _voicePacketBuffer: Uint8Array[] = [];
private _voicePacketSender: NodeJS.Timer;
private _triggered = false;
*/
constructor(client) {
this.client = client;
this.voiceRecorder = new VoiceRecorder(this);
this.voiceRecorder.on_data = data => this.sendPCMData(data);
this.voiceRecorder.on_data = data => this.handleVoiceData(data);
this.codec = new OpusCodec();
this.codec.initialise();
this.codec.on_encoded_data = buffer => {
if(this.dataChannel) {
console.log("Send buffer");
this.dataChannel.send(buffer);
this.vpacketId++;
if(this.vpacketId > 65535) this.vpacketId = 0;
let packet = new Uint8Array(buffer.byteLength + 2 + 3);
packet[0] = this.vpacketId < 6 ? 1 : 0; //Flag header
packet[1] = 0; //Flag fragmented
packet[2] = (this.vpacketId >> 8) & 0xFF; //HIGHT(voiceID)
packet[3] = (this.vpacketId >> 0) & 0xFF; //LOW (voiceID)
packet[4] = 4; //Codec
packet.set(buffer, 5);
//this._voicePacketBuffer.push(packet);
this.dataChannel.send(packet);
}
};
/*
this._voicePacketSender = setInterval(() => {
if(this._voicePacketBuffer.length > 5 || this._triggered) {
let packet = this._voicePacketBuffer.pop_front();
if (packet) {
this.dataChannel.send(packet);
}
this._triggered = this._voicePacketBuffer.length > 0;
}
}, 20);
*/
}
@ -87,36 +115,35 @@ class VoiceConnection {
onDataChannelMessage(message) {
let bin = new Uint8Array(message.data);
let clientId = bin[0] << 8 | bin[1];
console.log("Client id " + clientId);
let clientId = bin[2] << 8 | bin[3];
let packetId = bin[0] << 8 | bin[1];
let codec = bin[4];
console.log("Client id " + clientId + " PacketID " + packetId + " Codec: " + codec);
let client = this.client.channelTree.findClient(clientId);
if(!client) {
console.error("Having voice from unknown client? (ClientID: " + clientId + ")");
return;
}
var encodedData = new Uint8Array(message.data, 4);
let result = this.codec.decode(encodedData)
if(result instanceof Float32Array)
client.getAudioController().play(result);
else console.log("Invalid decode " + result);
var encodedData = new Uint8Array(message.data, 5);
this.codec.decodeSamples(encodedData).then(buffer => client.getAudioController().play(buffer)).catch(error => {
console.error("Could not playback client's (" + clientId + ") audio (" + error + ")");
});
}
private sendPCMData(data: any) {
/*
let result = this.codec.encodeSamples(data);
if(!result) console.error("Could not encode audio: " + result);
*/
this.client.getClient().getAudioController().play(data);
private handleVoiceData(data: AudioBuffer) {
setTimeout(() => {
this.codec.encodeSamples(data);
}, 1);
}
}
class VoiceRecorder {
private static readonly CHANNEL = 0;
private static readonly CHANNELS = 1;
private static readonly BUFFER_SIZE = 4096;
private static readonly BUFFER_SIZE = 16384 / 2;
handle: VoiceConnection;
on_data: (data: any) => void = (data) => {};
on_data: (data: AudioBuffer) => void = (data) => {};
private _recording: boolean = false;
@ -137,8 +164,9 @@ class VoiceRecorder {
const _this = this;
this.processor.addEventListener('audioprocess', ev => {
console.log(ev.inputBuffer);
if(_this.microphoneStream)
this.on_data(ev.inputBuffer.getChannelData(VoiceRecorder.CHANNEL))
this.on_data(ev.inputBuffer)
});
//Not needed but make sure we have data for the preprocessor
@ -212,7 +240,6 @@ class AudioController {
init: boolean;
stimeout: NodeJS.Timer;
resambler: Resampler = new Resampler();
//Events
onSpeaking: () => void;
onSilence: () => void;
@ -229,18 +256,9 @@ class AudioController {
this.onSilence = function () { }
}
play(pcm: Float32Array) {
//let buffer = this.speakerContext.createBuffer(1, 960, 48000);
//buffer.copyToChannel(pcm, 0);
this.resambler.resample(pcm, (buffer: AudioBuffer) => this.play0(buffer));
//this.play0(buffer);
}
play0(buffer: AudioBuffer) {
//960
console.log(buffer);
//let buffer = this.speakerContext.createBuffer(1, 960, 44100);
//buffer.copyToChannel(pcm, 0);
play(buffer: AudioBuffer) {
if(buffer.sampleRate != this.speakerContext.sampleRate)
console.warn("[AudioController] Source sample rate isnt equal to playback sample rate!");
this.audioCache.push(buffer);
let currentTime = new Date().getTime();
@ -265,8 +283,8 @@ class AudioController {
playCache(cache) {
while (cache.length) {
var buffer = cache.shift();
var source = this.speakerContext.createBufferSource();
let buffer = cache.shift();
let source = this.speakerContext.createBufferSource();
source.buffer = buffer;
source.connect(this.speakerContext.destination);
@ -294,32 +312,24 @@ class AudioController {
}
class Resampler {
context: OfflineAudioContext;
targetSampleRate: number;
constructor(){
constructor(targetSampleRate: number = 44100){
this.targetSampleRate = targetSampleRate;
}
resample(pcm: Float32Array, callback: (AudioBuffer) => void) {
/*
let buffer = AudioController.globalContext.createBuffer(1, pcm.length, 48000);
buffer.copyToChannel(pcm, 0);
callback(buffer);
*/
resample(buffer: AudioBuffer) : Promise<AudioBuffer> {
if(buffer.sampleRate == this.targetSampleRate)
return new Promise<AudioBuffer>(resolve => resolve(buffer));
this.context = new OfflineAudioContext(1, 882, 44100);
let buffer = this.context.createBuffer(1, pcm.length, 48000);
buffer.copyToChannel(pcm, 0);
console.log(this.targetSampleRate);
let context = new OfflineAudioContext(1, Math.ceil(buffer.length * this.targetSampleRate / buffer.sampleRate), this.targetSampleRate);
let source = this.context.createBufferSource();
let source = context.createBufferSource();
source.buffer = buffer;
source.connect(this.context.destination);
source.connect(context.destination);
source.start(0);
//console.log(source.buffer.getChannelData(0));
this.context.startRendering().then(e => callback(e)).catch(error => {
console.error("Could not resample audio");
console.error(error);
});
return context.startRendering();
}
}