TeaSpeak
/
TeaWeb
1
0
Fork 0
Code Issues 1 Pull Requests Activity

Fixed script node decoding

canary
WolverinDEV 2020-06-12 19:33:05 +02:00
parent 99b75f34d6
commit d3df4f0976
11 changed files with 95 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ChangeLog.md
View File

@ -2,6 +2,7 @@
* **12.06.20**
- Added a copy/paste menu for all HTML input elements
- Heavily improved web client audio de/encoding
- Fixed script node voice encoding
* **11.06.20**
- Fixed channel tree deletions

11
shared/js/ConnectionHandler.ts
View File

@ -337,13 +337,12 @@ export class ConnectionHandler {
getClient() : LocalClientEntry { return this._local_client; }
getClientId() { return this._clientId; }
set clientId(id: number) {
this._clientId = id;
this._local_client["_clientId"] = id;
}
initializeLocalClient(clientId: number, acceptedName: string) {
this._clientId = clientId;
this._local_client["_clientId"] = clientId;
get clientId() {
return this._clientId;
this.channelTree.registerClient(this._local_client);
this._local_client.updateVariables( { key: "client_nickname", value: acceptedName });
}
getServerConnection() : AbstractServerConnection { return this.serverConnection; }

6
shared/js/connection/CommandHandler.ts
View File

@ -191,10 +191,8 @@ export class ConnectionCommandHandler extends AbstractCommandHandler {
json = json[0]; //Only one bulk
this.connection_handler.channelTree.registerClient(this.connection_handler.getClient());
this.connection.client.side_bar.channel_conversations().reset();
this.connection.client.clientId = parseInt(json["aclid"]);
this.connection.client.getClient().updateVariables( {key: "client_nickname", value: json["acn"]});
this.connection.client.initializeLocalClient(parseInt(json["aclid"]), json["acn"]);
let updates: {
key: string,
@ -825,7 +823,7 @@ export class ConnectionCommandHandler extends AbstractCommandHandler {
const channel_id = typeof(json["cid"]) !== "undefined" ? parseInt(json["cid"]) : own_channel_id;
const channel = this.connection_handler.channelTree.findChannel(channel_id) || this.connection_handler.getClient().currentChannel();
if(json["invokerid"] == this.connection.client.clientId)
if(json["invokerid"] == this.connection.client.getClientId())
this.connection_handler.sound.play(Sound.MESSAGE_SEND, {default_volume: .5});
else if(channel_id == own_channel_id) {
this.connection_handler.sound.play(Sound.MESSAGE_RECEIVED, {default_volume: .5});

2
shared/js/ui/frames/chat_frame.ts
View File

@ -251,7 +251,7 @@ export class InfoFrame {
client_id: selected_client.clientId()
}, { create: false, attach: false }) : undefined;
const visibility = (selected_client && selected_client.clientId() !== this.handle.handle.clientId) ? "visible" : "hidden";
const visibility = (selected_client && selected_client.clientId() !== this.handle.handle.getClientId()) ? "visible" : "hidden";
if(this._button_conversation.style.visibility !== visibility)
this._button_conversation.style.visibility = visibility;
if(conversation) {

1
web/js/audio/recorder.ts
View File

@ -407,6 +407,7 @@ class JavascriptInput implements AbstractInput {
const callback = this._current_consumer as CallbackInputConsumer;
if(callback.callback_audio)
callback.callback_audio(event.inputBuffer);
if(callback.callback_buffer) {
log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!"));
}

7
web/js/codec/BasicCodec.ts
View File

@ -53,9 +53,10 @@ export abstract class BasicCodec implements Codec {
encodeSamples(cache: CodecClientCache, pcm: AudioBuffer) {
this._encodeResampler.resample(pcm).catch(error => log.error(LogCategory.VOICE, tr("Could not resample PCM data for codec. Error: %o"), error))
.then(buffer => this.encodeSamples0(cache, buffer as any)).catch(error => console.error(tr("Could not encode PCM data for codec. Error: %o"), error))
this._encodeResampler.resample(pcm)
.catch(error => log.error(LogCategory.VOICE, tr("Could not resample PCM data for codec. Error: %o"), error))
.then(buffer => this.encodeSamples0(cache, buffer as any))
.catch(error => console.error(tr("Could not encode PCM data for codec. Error: %o"), error))
}
private encodeSamples0(cache: CodecClientCache, buffer: AudioBuffer) {

1
web/js/codec/Codec.ts
View File

@ -35,7 +35,6 @@ export class BufferChunk {
}
export class CodecClientCache {
_last_access: number;
_chunks: BufferChunk[] = [];
bufferedSamples(max: number = 0) : number {

4
web/js/voice/AudioResampler.ts
View File

@ -2,7 +2,7 @@ import {LogCategory} from "tc-shared/log";
import * as log from "tc-shared/log";
export class AudioResampler {
targetSampleRate: number;
readonly targetSampleRate: number;
private _use_promise: boolean;
constructor(targetSampleRate: number){
@ -15,7 +15,7 @@ export class AudioResampler {
log.warn(LogCategory.AUDIO, tr("Received empty buffer as input! Returning empty output!"));
return Promise.resolve(buffer);
}
//console.log("Encode from %i to %i", buffer.sampleRate, this.targetSampleRate);
if(buffer.sampleRate == this.targetSampleRate)
return Promise.resolve(buffer);

96
web/js/voice/VoiceHandler.ts
View File

@ -137,6 +137,7 @@ export class VoiceConnection extends AbstractVoiceConnection {
private _type: VoiceEncodeType = VoiceEncodeType.NATIVE_ENCODE;
private localAudioStarted = false;
/*
* To ensure we're not sending any audio because the settings activates the input,
* we self mute the audio stream
@ -245,15 +246,15 @@ export class VoiceConnection extends AbstractVoiceConnection {
if(this._audio_source)
await this._audio_source.unmount();
this.handle_local_voice_ended();
this.handleLocalVoiceEnded();
this._audio_source = recorder;
if(recorder) {
recorder.current_handler = this.connection.client;
recorder.callback_unmount = this.on_recorder_yield.bind(this);
recorder.callback_start = this.handle_local_voice_started.bind(this);
recorder.callback_stop = this.handle_local_voice_ended.bind(this);
recorder.callback_start = this.handleLocalVoiceStarted.bind(this);
recorder.callback_stop = this.handleLocalVoiceEnded.bind(this);
recorder.callback_input_change = async (old_input, new_input) => {
if(old_input) {
@ -289,11 +290,16 @@ export class VoiceConnection extends AbstractVoiceConnection {
log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e);
}
} else {
//TODO: Error handling?
await recorder.input.set_consumer({
type: InputConsumerType.CALLBACK,
callback_audio: buffer => this.handle_local_voice(buffer, false)
} as CallbackInputConsumer);
try {
await recorder.input.set_consumer({
type: InputConsumerType.CALLBACK,
callback_audio: buffer => this.handleLocalVoiceBuffer(buffer, false)
} as CallbackInputConsumer);
log.debug(LogCategory.VOICE, tr("Successfully set/updated to the new input for the recorder"));
} catch (e) {
log.warn(LogCategory.VOICE, tr("Failed to set consumer to the new recorder input: %o"), e);
}
}
}
};
@ -333,22 +339,27 @@ export class VoiceConnection extends AbstractVoiceConnection {
const buffer = this.voice_send_queue.pop_front();
if(!buffer)
return;
this.send_voice_packet(buffer.data, buffer.codec);
this.sendVoicePacket(buffer.data, buffer.codec);
}
send_voice_packet(encoded_data: Uint8Array, codec: number) {
private fillVoicePacketHeader(packet: Uint8Array, codec: number) {
packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header
packet[1] = 0; //Flag fragmented
packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID)
packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW (voiceID)
packet[4] = codec; //Codec
}
sendVoicePacket(encoded_data: Uint8Array, codec: number) {
if(this.dataChannel) {
this.voice_packet_id++;
if(this.voice_packet_id > 65535)
this.voice_packet_id = 0;
let packet = new Uint8Array(encoded_data.byteLength + 5);
packet[0] = this.chunkVPacketId++ < 5 ? 1 : 0; //Flag header
packet[1] = 0; //Flag fragmented
packet[2] = (this.voice_packet_id >> 8) & 0xFF; //HIGHT (voiceID)
packet[3] = (this.voice_packet_id >> 0) & 0xFF; //LOW (voiceID)
packet[4] = codec; //Codec
this.fillVoicePacketHeader(packet, codec);
packet.set(encoded_data, 5);
try {
this.dataChannel.send(packet);
} catch (error) {
@ -359,6 +370,20 @@ export class VoiceConnection extends AbstractVoiceConnection {
}
}
sendVoiceStopPacket(codec: number) {
if(!this.dataChannel)
return;
const packet = new Uint8Array(5);
this.fillVoicePacketHeader(packet, codec);
try {
this.dataChannel.send(packet);
} catch (error) {
log.warn(LogCategory.VOICE, tr("Failed to send voice packet. Error: %o"), error);
}
}
private _audio_player_waiting = false;
start_rtc_session() {
if(!aplayer.initialized()) {
@ -390,8 +415,8 @@ export class VoiceConnection extends AbstractVoiceConnection {
const dataChannelConfig = { ordered: false, maxRetransmits: 0 };
this.dataChannel = this.rtcPeerConnection.createDataChannel('main', dataChannelConfig);
this.dataChannel.onmessage = this.on_data_channel_message.bind(this);
this.dataChannel.onopen = this.on_data_channel.bind(this);
this.dataChannel.onmessage = this.onMainDataChannelMessage.bind(this);
this.dataChannel.onopen = this.onMainDataChannelOpen.bind(this);
this.dataChannel.binaryType = "arraybuffer";
let sdpConstraints : RTCOfferOptions = {};
@ -524,15 +549,15 @@ export class VoiceConnection extends AbstractVoiceConnection {
});
}
private on_data_channel(channel) {
private onMainDataChannelOpen(channel) {
log.info(LogCategory.VOICE, tr("Got new data channel! (%s)"), this.dataChannel.readyState);
this.connection.client.update_voice_status();
}
private on_data_channel_message(message: MessageEvent) {
private onMainDataChannelMessage(message: MessageEvent) {
const chandler = this.connection.client;
if(chandler.client_status.output_muted) /* we dont need to do anything with sound playback when we're not listening to it */
if(chandler.isSpeakerMuted() || chandler.isSpeakerDisabled()) /* we dont need to do anything with sound playback when we're not listening to it */
return;
let bin = new Uint8Array(message.data);
@ -571,18 +596,18 @@ export class VoiceConnection extends AbstractVoiceConnection {
}
}
private handle_local_voice(data: AudioBuffer, head: boolean) {
private handleLocalVoiceBuffer(data: AudioBuffer, head: boolean) {
const chandler = this.connection.client;
if(!chandler.connected)
if(!this.localAudioStarted || !chandler.connected)
return false;
if(chandler.client_status.input_muted)
if(chandler.isMicrophoneMuted())
return false;
if(head)
this.chunkVPacketId = 0;
let client = this.find_client(chandler.clientId);
let client = this.find_client(chandler.getClientId());
if(!client) {
log.error(LogCategory.VOICE, tr("Tried to send voice data, but local client hasn't a voice client handle"));
return;
@ -594,25 +619,31 @@ export class VoiceConnection extends AbstractVoiceConnection {
.then(encoder => encoder.encodeSamples(client.get_codec_cache(codec), data));
}
private handle_local_voice_ended() {
private handleLocalVoiceEnded() {
const chandler = this.connection.client;
const ch = chandler.getClient();
if(ch) ch.speaking = false;
if(!chandler.connected)
return false;
if(chandler.client_status.input_muted)
if(chandler.isMicrophoneMuted())
return false;
log.info(LogCategory.VOICE, tr("Local voice ended"));
this.localAudioStarted = false;
if(this.dataChannel && this._encoder_codec >= 0)
this.send_voice_packet(new Uint8Array(0), this._encoder_codec);
if(this._type === VoiceEncodeType.NATIVE_ENCODE) {
setTimeout(() => {
/* first send all data, than send the stop signal */
this.sendVoiceStopPacket(this._encoder_codec);
}, 150);
} else {
this.sendVoiceStopPacket(this._encoder_codec);
}
}
private handle_local_voice_started() {
private handleLocalVoiceStarted() {
const chandler = this.connection.client;
if(chandler.client_status.input_muted) {
/* evail hack due to the settings :D */
if(chandler.isMicrophoneMuted()) {
log.warn(LogCategory.VOICE, tr("Received local voice started event, even thou we're muted! Do not send any voice."));
if(this.local_audio_mute)
this.local_audio_mute.gain.value = 0;
@ -620,6 +651,8 @@ export class VoiceConnection extends AbstractVoiceConnection {
}
if(this.local_audio_mute)
this.local_audio_mute.gain.value = 1;
this.localAudioStarted = true;
log.info(LogCategory.VOICE, tr("Local voice started"));
const ch = chandler.getClient();
@ -683,7 +716,6 @@ export class VoiceConnection extends AbstractVoiceConnection {
}
/* funny fact that typescript dosn't find this */
declare global {
interface RTCPeerConnection {

30
web/js/workers/codec/OpusCodec.ts
View File

@ -121,6 +121,8 @@ const OPUS_ERROR_CODES = [
];
class OpusWorker implements CodecWorker {
private static readonly kProcessBufferSize = 4096 * 2;
private readonly channelCount: number;
private readonly type: OpusType;
private nativeHandle: any;
@ -130,11 +132,8 @@ class OpusWorker implements CodecWorker {
private fn_encode: any;
private fn_reset: any;
private buffer_size = 4096 * 2;
private buffer: any;
private encode_buffer: Float32Array;
private decode_buffer: Uint8Array;
private nativeBufferPtr: number;
private processBuffer: Uint8Array;
constructor(channelCount: number, type: OpusType) {
this.channelCount = channelCount;
@ -153,40 +152,39 @@ class OpusWorker implements CodecWorker {
this.nativeHandle = this.fn_newHandle(this.channelCount, this.type);
this.buffer = Module._malloc(this.buffer_size);
this.encode_buffer = new Float32Array(Module.HEAPF32.buffer, this.buffer, Math.floor(this.buffer_size / 4));
this.decode_buffer = new Uint8Array(Module.HEAPU8.buffer, this.buffer, this.buffer_size);
this.nativeBufferPtr = Module._malloc(OpusWorker.kProcessBufferSize);
this.processBuffer = new Uint8Array(Module.HEAPU8.buffer, this.nativeBufferPtr, OpusWorker.kProcessBufferSize);
return undefined;
}
deinitialise() { } //TODO
decode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string {
if (buffer.byteLength > this.decode_buffer.byteLength)
if (buffer.byteLength > this.processBuffer.byteLength)
return "supplied data exceeds internal buffer";
this.decode_buffer.set(buffer);
this.processBuffer.set(buffer);
let result = this.fn_decode(this.nativeHandle, this.decode_buffer.byteOffset, buffer.byteLength, this.decode_buffer.byteLength);
let result = this.fn_decode(this.nativeHandle, this.processBuffer.byteOffset, buffer.byteLength, this.processBuffer.byteLength);
if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown decode error " + result;
const resultByteLength = result * this.channelCount * 4;
const resultBuffer = responseBuffer(resultByteLength);
resultBuffer.set(this.decode_buffer.subarray(0, resultByteLength), 0);
resultBuffer.set(this.processBuffer.subarray(0, resultByteLength), 0);
return resultByteLength;
}
encode(buffer: Uint8Array, responseBuffer: (length: number) => Uint8Array): number | string {
if (buffer.byteLength > this.decode_buffer.byteLength)
if (buffer.byteLength > this.processBuffer.byteLength)
return "supplied data exceeds internal buffer";
this.encode_buffer.set(buffer);
this.processBuffer.set(buffer);
let result = this.fn_encode(this.nativeHandle, this.encode_buffer.byteOffset, buffer.byteLength, this.encode_buffer.byteLength);
let result = this.fn_encode(this.nativeHandle, this.processBuffer.byteOffset, buffer.byteLength, this.processBuffer.byteLength);
if (result < 0) return OPUS_ERROR_CODES[-result] || "unknown encode error " + result;
const resultBuffer = responseBuffer(result);
resultBuffer.set(Module.HEAP8.subarray(this.encode_buffer.byteOffset, this.encode_buffer.byteOffset + result));
resultBuffer.set(this.processBuffer.subarray(0, result), 0);
return result;
}

2
web/native-codec/src/opus.cpp
View File

@ -87,7 +87,7 @@ int codec_opus_encode(OpusHandle *handle, uint8_t *buffer, size_t byte_length, s
if(handle->channelCount == 2)
sequenced2interleaved_intersecting<2>((float *) buffer, byte_length / (sizeof(float) * 2));
auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / handle->channelCount, buffer, maxLength);
auto result = opus_encode_float(&*handle->encoder, (float *) buffer, byte_length / (handle->channelCount * sizeof(float)), buffer, maxLength);
if (result < 0) return result;
return result;
}