From 6b5adf30c7aa925dbfdeae14abd2b47428461634 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Fri, 8 Jan 2021 21:30:48 +0100 Subject: [PATCH] Fixed some microphone issues --- ChangeLog.md | 6 +- shared/js/ConnectionHandler.ts | 92 +++++++++++++------ shared/js/audio/recorder.ts | 26 ++++-- shared/js/connection/VoiceConnection.ts | 5 +- shared/js/media/Stream.ts | 27 +++--- shared/js/media/Video.ts | 10 +- shared/js/ui/frames/chat.ts | 10 +- shared/js/ui/frames/control-bar/Controller.ts | 8 +- shared/js/ui/modal/settings/Microphone.tsx | 54 ++++++----- shared/js/voice/RecorderBase.ts | 30 +++--- tsconfig.json | 2 +- web/app/audio/Recorder.ts | 84 +++++++++++------ web/app/legacy/audio-lib/index.ts | 2 +- web/app/legacy/voice/VoiceHandler.ts | 8 +- web/app/voice/Connection.ts | 6 +- 15 files changed, 235 insertions(+), 135 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index c23a886f..349bb34c 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,4 +1,8 @@ # Changelog: +* **08.01.21** + - Fixed a bug where the microphone did not started recording after switching the device + - Fixed bug that the web client was only able to use the default microphone + * **07.01.21** - Improved general client ui memory footprint (Don't constantly rendering the channel tree) - Improved channel tree loading performance especially on server join and switch @@ -17,7 +21,7 @@ - Added the option to edit the channel sidebar mode - Remove the phonetic name and the channel title (Both are not used) - Improved property validation - - Adjusting property editibility according to the clients permissions + - Adjusting property edibility according to the clients permissions - Fixed issue [#164](https://github.com/TeaSpeak/TeaWeb/issues/154) ("error: channel description" bug) * **18.12.20** diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts index 2c36b199..dc51b820 100644 --- a/shared/js/ConnectionHandler.ts +++ b/shared/js/ConnectionHandler.ts @@ -10,7 +10,7 @@ import {createErrorModal, createInfoModal, createInputModal, Modal} from "./ui/e import {hashPassword} from "./utils/helpers"; import {HandshakeHandler} from "./connection/HandshakeHandler"; import * as htmltags from "./ui/htmltags"; -import {FilterMode, InputState, MediaStreamRequestResult} from "./voice/RecorderBase"; +import {FilterMode, InputState, InputStartError} from "./voice/RecorderBase"; import {CommandResult} from "./connection/ServerConnectionDeclaration"; import {defaultRecorder, RecorderProfile} from "./voice/RecorderProfile"; import {connection_log, Regex} from "./ui/modal/ModalConnect"; @@ -183,6 +183,7 @@ export class ConnectionHandler { private clientStatusSync = false; private inputHardwareState: InputHardwareState = InputHardwareState.MISSING; + private listenerRecorderInputDeviceChanged: (() => void); log: ServerEventLog; @@ -196,9 +197,21 @@ export class ConnectionHandler { this.serverConnection = getServerConnectionFactory().create(this); this.serverConnection.events.on("notify_connection_state_changed", event => this.on_connection_state_changed(event.oldState, event.newState)); - this.serverConnection.getVoiceConnection().events.on("notify_recorder_changed", () => { + this.serverConnection.getVoiceConnection().events.on("notify_recorder_changed", event => { this.setInputHardwareState(this.getVoiceRecorder() ? InputHardwareState.VALID : InputHardwareState.MISSING); - this.update_voice_status(); + this.updateVoiceStatus(); + + if(this.listenerRecorderInputDeviceChanged) { + this.listenerRecorderInputDeviceChanged(); + this.listenerRecorderInputDeviceChanged = undefined; + } + + if(event.newRecorder) { + this.listenerRecorderInputDeviceChanged = event.newRecorder.input?.events.on("notify_device_changed", () => { + this.setInputHardwareState(InputHardwareState.VALID); + this.updateVoiceStatus(); + }); + } }); this.serverConnection.getVoiceConnection().events.on("notify_connection_status_changed", () => this.update_voice_status()); this.serverConnection.getVoiceConnection().setWhisperSessionInitializer(this.initializeWhisperSession.bind(this)); @@ -265,7 +278,7 @@ export class ConnectionHandler { server_address.port = 9987; } } - log.info(LogCategory.CLIENT, tr("Start connection to %s:%d"), server_address.host, server_address.port); + logInfo(LogCategory.CLIENT, tr("Start connection to %s:%d"), server_address.host, server_address.port); this.log.log("connection.begin", { address: { server_hostname: server_address.host, @@ -385,7 +398,7 @@ export class ConnectionHandler { private handleConnectionStateChanged(event: ConnectionEvents["notify_connection_state_changed"]) { this.connection_state = event.newState; if(event.newState === ConnectionState.CONNECTED) { - log.info(LogCategory.CLIENT, tr("Client connected")); + logInfo(LogCategory.CLIENT, tr("Client connected")); this.log.log("connection.connected", { serverAddress: { server_port: this.channelTree.server.remote_address.port, @@ -674,19 +687,19 @@ export class ConnectionHandler { if(auto_reconnect) { if(!this.serverConnection) { - log.info(LogCategory.NETWORKING, tr("Allowed to auto reconnect but cant reconnect because we dont have any information left...")); + logInfo(LogCategory.NETWORKING, tr("Allowed to auto reconnect but cant reconnect because we dont have any information left...")); return; } this.log.log("reconnect.scheduled", {timeout: 5000}); - log.info(LogCategory.NETWORKING, tr("Allowed to auto reconnect. Reconnecting in 5000ms")); + logInfo(LogCategory.NETWORKING, tr("Allowed to auto reconnect. Reconnecting in 5000ms")); const server_address = this.serverConnection.remote_address(); const profile = this.serverConnection.handshake_handler().profile; this.autoReconnectTimer = setTimeout(() => { this.autoReconnectTimer = undefined; this.log.log("reconnect.execute", {}); - log.info(LogCategory.NETWORKING, tr("Reconnecting...")); + logInfo(LogCategory.NETWORKING, tr("Reconnecting...")); this.startConnection(server_address.host + ":" + server_address.port, profile, false, Object.assign(this.reconnect_properties(profile), {auto_reconnect_attempt: true})); }, 5000); @@ -783,8 +796,10 @@ export class ConnectionHandler { if(Object.keys(localClientUpdates).length > 0) { /* directly update all update locally so we don't send updates twice */ const updates = []; - for(const key of Object.keys(localClientUpdates)) + for(const key of Object.keys(localClientUpdates)) { updates.push({ key: key, value: localClientUpdates[key] ? "1" : "0" }); + } + this.getClient().updateVariables(...updates); this.clientStatusSync = true; @@ -804,7 +819,7 @@ export class ConnectionHandler { if(currentInput) { if(shouldRecord || this.echoTestRunning) { if(this.getInputHardwareState() !== InputHardwareState.START_FAILED) { - this.startVoiceRecorder(Date.now() - this._last_record_error_popup > 10 * 1000).then(() => { + this.startVoiceRecorder(Date.now() - this.lastRecordErrorPopup > 10 * 1000).then(() => { this.events_.fire("notify_state_updated", { state: "microphone" }); }); } @@ -818,7 +833,7 @@ export class ConnectionHandler { } } - private _last_record_error_popup: number = 0; + private lastRecordErrorPopup: number = 0; update_voice_status() { this.updateVoiceStatus(); return; @@ -870,21 +885,23 @@ export class ConnectionHandler { } this.setInputHardwareState(InputHardwareState.VALID); - this.update_voice_status(); + this.updateVoiceStatus(); return { state: "success" }; } catch (error) { this.setInputHardwareState(InputHardwareState.START_FAILED); - this.update_voice_status(); + this.updateVoiceStatus(); let errorMessage; - if(error === MediaStreamRequestResult.ENOTSUPPORTED) { + if(error === InputStartError.ENOTSUPPORTED) { errorMessage = tr("Your browser does not support voice recording"); - } else if(error === MediaStreamRequestResult.EBUSY) { + } else if(error === InputStartError.EBUSY) { errorMessage = tr("The input device is busy"); - } else if(error === MediaStreamRequestResult.EDEVICEUNKNOWN) { + } else if(error === InputStartError.EDEVICEUNKNOWN) { errorMessage = tr("Invalid input device"); - } else if(error === MediaStreamRequestResult.ENOTALLOWED) { + } else if(error === InputStartError.ENOTALLOWED) { errorMessage = tr("No permissions"); + } else if(error === InputStartError.ESYSTEMUNINITIALIZED) { + errorMessage = tr("Window audio not initialized."); } else if(error instanceof Error) { errorMessage = error.message; } else if(typeof error === "string") { @@ -893,9 +910,9 @@ export class ConnectionHandler { errorMessage = tr("lookup the console"); } - log.warn(LogCategory.VOICE, tr("Failed to start microphone input (%s)."), error); + logWarn(LogCategory.VOICE, tr("Failed to start microphone input (%s)."), error); if(notifyError) { - this._last_record_error_popup = Date.now(); + this.lastRecordErrorPopup = Date.now(); createErrorModal(tr("Failed to start recording"), tra("Microphone start failed.\nError: {}", errorMessage)).open(); } return { state: "error", message: errorMessage }; @@ -910,11 +927,11 @@ export class ConnectionHandler { reconnect_properties(profile?: ConnectionProfile) : ConnectParameters { const name = (this.getClient() ? this.getClient().clientNickName() : "") || - (this.serverConnection && this.serverConnection.handshake_handler() ? this.serverConnection.handshake_handler().parameters.nickname : "") || + (this.serverConnection?.handshake_handler() ? this.serverConnection.handshake_handler().parameters.nickname : "") || StaticSettings.instance.static(Settings.KEY_CONNECT_USERNAME, profile ? profile.defaultUsername : undefined) || "Another TeaSpeak user"; const channel = (this.getClient() && this.getClient().currentChannel() ? this.getClient().currentChannel().channelId : 0) || - (this.serverConnection && this.serverConnection.handshake_handler() ? (this.serverConnection.handshake_handler().parameters.channel || {} as any).target : ""); + (this.serverConnection?.handshake_handler() ? (this.serverConnection.handshake_handler().parameters.channel || {} as any).target : ""); const channel_password = (this.getClient() && this.getClient().currentChannel() ? this.getClient().currentChannel().cached_password() : "") || (this.serverConnection && this.serverConnection.handshake_handler() ? (this.serverConnection.handshake_handler().parameters.channel || {} as any).password : ""); return { @@ -926,10 +943,12 @@ export class ConnectionHandler { update_avatar() { spawnAvatarUpload(data => { - if(typeof(data) === "undefined") + if(typeof(data) === "undefined") { return; - if(data === null) { - log.info(LogCategory.CLIENT, tr("Deleting existing avatar")); + } + + if(!data) { + logInfo(LogCategory.CLIENT, tr("Deleting existing avatar")); this.serverConnection.send_command('ftdeletefile', { name: "/avatar_", /* delete own avatar */ path: "", @@ -940,15 +959,19 @@ export class ConnectionHandler { log.error(LogCategory.GENERAL, tr("Failed to reset avatar flag: %o"), error); let message; - if(error instanceof CommandResult) + if(error instanceof CommandResult) { message = formatMessage(tr("Failed to delete avatar.{:br:}Error: {0}"), error.extra_message || error.message); - if(!message) + } + + if(!message) { message = formatMessage(tr("Failed to delete avatar.{:br:}Lookup the console for more details")); + } + createErrorModal(tr("Failed to delete avatar"), message).open(); return; }); } else { - log.info(LogCategory.CLIENT, tr("Uploading new avatar")); + logInfo(LogCategory.CLIENT, tr("Uploading new avatar")); (async () => { const transfer = this.fileManager.initializeFileUpload({ name: "/avatar", @@ -1073,9 +1096,14 @@ export class ConnectionHandler { this.serverFeatures?.destroy(); this.serverFeatures = undefined; - this.settings && this.settings.destroy(); + this.settings?.destroy(); this.settings = undefined; + if(this.listenerRecorderInputDeviceChanged) { + this.listenerRecorderInputDeviceChanged(); + this.listenerRecorderInputDeviceChanged = undefined; + } + if(this.serverConnection) { getServerConnectionFactory().destroy(this.serverConnection); } @@ -1087,7 +1115,10 @@ export class ConnectionHandler { /* state changing methods */ setMicrophoneMuted(muted: boolean, dontPlaySound?: boolean) { - if(this.client_status.input_muted === muted) return; + if(this.client_status.input_muted === muted) { + return; + } + this.client_status.input_muted = muted; if(!dontPlaySound) { this.sound.play(muted ? Sound.MICROPHONE_MUTED : Sound.MICROPHONE_ACTIVATED); @@ -1179,8 +1210,9 @@ export class ConnectionHandler { getInputHardwareState() : InputHardwareState { return this.inputHardwareState; } private setInputHardwareState(state: InputHardwareState) { - if(this.inputHardwareState === state) + if(this.inputHardwareState === state) { return; + } this.inputHardwareState = state; this.events_.fire("notify_state_updated", { state: "microphone" }); diff --git a/shared/js/audio/recorder.ts b/shared/js/audio/recorder.ts index e244646f..9309df10 100644 --- a/shared/js/audio/recorder.ts +++ b/shared/js/audio/recorder.ts @@ -17,9 +17,6 @@ export interface AudioRecorderBacked { } export interface DeviceListEvents { - /* - * Should only trigger if the list really changed. - */ notify_list_updated: { removedDeviceCount: number, addedDeviceCount: number @@ -44,6 +41,7 @@ export interface IDevice { driver: string; name: string; } + export namespace IDevice { export const NoDeviceId = "none"; export const DefaultDeviceId = "default"; @@ -90,8 +88,9 @@ export abstract class AbstractDeviceList implements DeviceList { } protected setState(state: DeviceListState) { - if(this.listState === state) + if(this.listState === state) { return; + } const oldState = this.listState; this.listState = state; @@ -99,8 +98,9 @@ export abstract class AbstractDeviceList implements DeviceList { } protected setPermissionState(state: PermissionState) { - if(this.permissionState === state) + if(this.permissionState === state) { return; + } const oldState = this.permissionState; this.permissionState = state; @@ -108,24 +108,28 @@ export abstract class AbstractDeviceList implements DeviceList { } awaitInitialized(): Promise { - if(this.listState !== "uninitialized") + if(this.listState !== "uninitialized") { return Promise.resolve(); + } return new Promise(resolve => { const callback = (event: DeviceListEvents["notify_state_changed"]) => { - if(event.newState === "uninitialized") + if(event.newState === "uninitialized") { return; + } this.events.off("notify_state_changed", callback); resolve(); }; + this.events.on("notify_state_changed", callback); }); } awaitHealthy(): Promise { - if(this.listState === "healthy") + if(this.listState === "healthy") { return Promise.resolve(); + } return new Promise(resolve => { const callback = (event: DeviceListEvents["notify_state_changed"]) => { @@ -150,15 +154,17 @@ export abstract class AbstractDeviceList implements DeviceList { let recorderBackend: AudioRecorderBacked; export function getRecorderBackend() : AudioRecorderBacked { - if(typeof recorderBackend === "undefined") + if(typeof recorderBackend === "undefined") { throw tr("the recorder backend hasn't been set yet"); + } return recorderBackend; } export function setRecorderBackend(instance: AudioRecorderBacked) { - if(typeof recorderBackend !== "undefined") + if(typeof recorderBackend !== "undefined") { throw tr("a recorder backend has already been initialized"); + } recorderBackend = instance; } diff --git a/shared/js/connection/VoiceConnection.ts b/shared/js/connection/VoiceConnection.ts index 56cb627a..98fd6664 100644 --- a/shared/js/connection/VoiceConnection.ts +++ b/shared/js/connection/VoiceConnection.ts @@ -22,7 +22,10 @@ export interface VoiceConnectionEvents { newStatus: VoiceConnectionStatus }, - "notify_recorder_changed": {}, + "notify_recorder_changed": { + oldRecorder: RecorderProfile | undefined, + newRecorder: RecorderProfile | undefined + }, "notify_whisper_created": { session: WhisperSession diff --git a/shared/js/media/Stream.ts b/shared/js/media/Stream.ts index 6019a4fb..a42515bf 100644 --- a/shared/js/media/Stream.ts +++ b/shared/js/media/Stream.ts @@ -1,7 +1,7 @@ -import {MediaStreamRequestResult} from "tc-shared/voice/RecorderBase"; +import {InputStartError} from "tc-shared/voice/RecorderBase"; import * as log from "tc-shared/log"; -import {LogCategory, logWarn} from "tc-shared/log"; -import { tr } from "tc-shared/i18n/localize"; +import {LogCategory, logInfo, logWarn} from "tc-shared/log"; +import {tr} from "tc-shared/i18n/localize"; export type MediaStreamType = "audio" | "video"; @@ -19,22 +19,20 @@ export interface MediaStreamEvents { export const mediaStreamEvents = new Registry(); */ -export async function requestMediaStreamWithConstraints(constraints: MediaTrackConstraints, type: MediaStreamType) : Promise { +export async function requestMediaStreamWithConstraints(constraints: MediaTrackConstraints, type: MediaStreamType) : Promise { const beginTimestamp = Date.now(); try { - log.info(LogCategory.AUDIO, tr("Requesting a %s stream for device %s in group %s"), type, constraints.deviceId, constraints.groupId); - const stream = await navigator.mediaDevices.getUserMedia(type === "audio" ? { audio: constraints } : { video: constraints }); - - return stream; + logInfo(LogCategory.AUDIO, tr("Requesting a %s stream for device %s in group %s"), type, constraints.deviceId, constraints.groupId); + return await navigator.mediaDevices.getUserMedia(type === "audio" ? {audio: constraints} : {video: constraints}); } catch(error) { if('name' in error) { if(error.name === "NotAllowedError") { if(Date.now() - beginTimestamp < 250) { log.warn(LogCategory.AUDIO, tr("Media stream request failed (System denied). Browser message: %o"), error.message); - return MediaStreamRequestResult.ESYSTEMDENIED; + return InputStartError.ESYSTEMDENIED; } else { log.warn(LogCategory.AUDIO, tr("Media stream request failed (No permissions). Browser message: %o"), error.message); - return MediaStreamRequestResult.ENOTALLOWED; + return InputStartError.ENOTALLOWED; } } else { log.warn(LogCategory.AUDIO, tr("Media stream request failed. Request resulted in error: %o: %o"), error.name, error); @@ -43,13 +41,13 @@ export async function requestMediaStreamWithConstraints(constraints: MediaTrackC log.warn(LogCategory.AUDIO, tr("Failed to initialize media stream (%o)"), error); } - return MediaStreamRequestResult.EUNKNOWN; + return InputStartError.EUNKNOWN; } } /* request permission for devices only one per time! */ -let currentMediaStreamRequest: Promise; -export async function requestMediaStream(deviceId: string | undefined, groupId: string | undefined, type: MediaStreamType) : Promise { +let currentMediaStreamRequest: Promise; +export async function requestMediaStream(deviceId: string | undefined, groupId: string | undefined, type: MediaStreamType) : Promise { /* wait for the current media stream requests to finish */ while(currentMediaStreamRequest) { try { @@ -78,8 +76,9 @@ export async function requestMediaStream(deviceId: string | undefined, groupId: try { return await currentMediaStreamRequest; } finally { - if(currentMediaStreamRequest === promise) + if(currentMediaStreamRequest === promise) { currentMediaStreamRequest = undefined; + } } } diff --git a/shared/js/media/Video.ts b/shared/js/media/Video.ts index 68a1d98a..670a4cf0 100644 --- a/shared/js/media/Video.ts +++ b/shared/js/media/Video.ts @@ -7,7 +7,7 @@ import { VideoSource, VideoSourceCapabilities, VideoSourceInitialSettings } from "tc-shared/video/VideoSource"; import {Registry} from "tc-shared/events"; -import {MediaStreamRequestResult} from "tc-shared/voice/RecorderBase"; +import {InputStartError} from "tc-shared/voice/RecorderBase"; import {LogCategory, logDebug, logError, logWarn} from "tc-shared/log"; import {queryMediaPermissions, requestMediaStream, stopMediaStream} from "tc-shared/media/Stream"; import {tr} from "tc-shared/i18n/localize"; @@ -126,10 +126,10 @@ export class WebVideoDriver implements VideoDriver { async requestPermissions(): Promise { const result = await requestMediaStream("default", undefined, "video"); - if(result === MediaStreamRequestResult.ENOTALLOWED) { + if(result === InputStartError.ENOTALLOWED) { this.setPermissionStatus(VideoPermissionStatus.UserDenied); return false; - } else if(result === MediaStreamRequestResult.ESYSTEMDENIED) { + } else if(result === InputStartError.ESYSTEMDENIED) { this.setPermissionStatus(VideoPermissionStatus.SystemDenied); return false; } @@ -177,10 +177,10 @@ export class WebVideoDriver implements VideoDriver { * This also applies to Firefox since the user has to manually update the flag after that. * Only the initial state for Firefox is and should be "Granted". */ - if(result === MediaStreamRequestResult.ENOTALLOWED) { + if(result === InputStartError.ENOTALLOWED) { this.setPermissionStatus(VideoPermissionStatus.UserDenied); throw tr("Device access has been denied"); - } else if(result === MediaStreamRequestResult.ESYSTEMDENIED) { + } else if(result === InputStartError.ESYSTEMDENIED) { this.setPermissionStatus(VideoPermissionStatus.SystemDenied); throw tr("Device access has been denied"); } diff --git a/shared/js/ui/frames/chat.ts b/shared/js/ui/frames/chat.ts index 019e1f36..58b752fd 100644 --- a/shared/js/ui/frames/chat.ts +++ b/shared/js/ui/frames/chat.ts @@ -306,14 +306,14 @@ export function format_time(time: number, default_value: string) { return result.length > 0 ? result.substring(1) : default_value; } -let _icon_size_style: HTMLStyleElement; +let iconStyleSize: HTMLStyleElement; export function set_icon_size(size: string) { - if(!_icon_size_style) { - _icon_size_style = document.createElement("style"); - document.head.append(_icon_size_style); + if(!iconStyleSize) { + iconStyleSize = document.createElement("style"); + document.head.append(iconStyleSize); } - _icon_size_style.innerText = ("\n" + + iconStyleSize.innerText = ("\n" + ".chat-emoji {\n" + " height: " + size + "!important;\n" + " width: " + size + "!important;\n" + diff --git a/shared/js/ui/frames/control-bar/Controller.ts b/shared/js/ui/frames/control-bar/Controller.ts index b9b0a20c..0fc15285 100644 --- a/shared/js/ui/frames/control-bar/Controller.ts +++ b/shared/js/ui/frames/control-bar/Controller.ts @@ -379,7 +379,13 @@ export function initializeControlBarController(events: Registry {}); + if(current_connection_handler.getVoiceRecorder()) { + if(event.enabled) { + current_connection_handler.startVoiceRecorder(true).then(undefined); + } + } else { + current_connection_handler.acquireInputHardware().then(() => {}); + } } }); diff --git a/shared/js/ui/modal/settings/Microphone.tsx b/shared/js/ui/modal/settings/Microphone.tsx index 65d79360..50dce282 100644 --- a/shared/js/ui/modal/settings/Microphone.tsx +++ b/shared/js/ui/modal/settings/Microphone.tsx @@ -87,31 +87,34 @@ export function initialize_audio_microphone_controller(events: Registry } = {}; - const level_info: { [key: string]: any } = {}; - let level_update_task; + const levelMeterInitializePromises: { [key: string]: Promise } = {}; + const deviceLevelInfo: { [key: string]: any } = {}; + let deviceLevelUpdateTask; - const destroy_meters = () => { - Object.keys(level_meters).forEach(e => { - const meter = level_meters[e]; - delete level_meters[e]; + const destroyLevelIndicators = () => { + Object.keys(levelMeterInitializePromises).forEach(e => { + const meter = levelMeterInitializePromises[e]; + delete levelMeterInitializePromises[e]; meter.then(e => e.destroy()); }); - Object.keys(level_info).forEach(e => delete level_info[e]); + Object.keys(deviceLevelInfo).forEach(e => delete deviceLevelInfo[e]); }; - const update_level_meter = () => { - destroy_meters(); + const updateLevelMeter = () => { + destroyLevelIndicators(); - level_info["none"] = {deviceId: "none", status: "success", level: 0}; + deviceLevelInfo["none"] = {deviceId: "none", status: "success", level: 0}; for (const device of recorderBackend.getDeviceList().getDevices()) { let promise = recorderBackend.createLevelMeter(device).then(meter => { meter.setObserver(level => { - if (level_meters[device.deviceId] !== promise) return; /* old level meter */ + if (levelMeterInitializePromises[device.deviceId] !== promise) { + /* old level meter */ + return; + } - level_info[device.deviceId] = { + deviceLevelInfo[device.deviceId] = { deviceId: device.deviceId, status: "success", level: level @@ -119,8 +122,11 @@ export function initialize_audio_microphone_controller(events: Registry { - if (level_meters[device.deviceId] !== promise) return; /* old level meter */ - level_info[device.deviceId] = { + if (levelMeterInitializePromises[device.deviceId] !== promise) { + /* old level meter */ + return; + } + deviceLevelInfo[device.deviceId] = { deviceId: device.deviceId, status: "error", @@ -130,28 +136,30 @@ export function initialize_audio_microphone_controller(events: Registry { + deviceLevelUpdateTask = setInterval(() => { const deviceListStatus = recorderBackend.getDeviceList().getStatus(); events.fire("notify_device_level", { - level: level_info, + level: deviceLevelInfo, status: deviceListStatus === "error" ? "uninitialized" : deviceListStatus }); }, 50); events.on("notify_devices", event => { - if (event.status !== "success") return; + if (event.status !== "success") { + return; + } - update_level_meter(); + updateLevelMeter(); }); - events.on("notify_destroy", event => { - destroy_meters(); - clearInterval(level_update_task); + events.on("notify_destroy", () => { + destroyLevelIndicators(); + clearInterval(deviceLevelUpdateTask); }); } diff --git a/shared/js/voice/RecorderBase.ts b/shared/js/voice/RecorderBase.ts index 912df2e9..09c53cda 100644 --- a/shared/js/voice/RecorderBase.ts +++ b/shared/js/voice/RecorderBase.ts @@ -25,33 +25,40 @@ export interface NativeInputConsumer { export type InputConsumer = CallbackInputConsumer | NodeInputConsumer | NativeInputConsumer; - export enum InputState { /* Input recording has been paused */ PAUSED, /* * Recording has been requested, and is currently initializing. - * This state may persist, when the audio context hasn't been initialized yet */ INITIALIZING, - /* we're currently recording the input */ + /* we're currently recording the input. */ RECORDING } -export enum MediaStreamRequestResult { +export enum InputStartError { EUNKNOWN = "eunknown", EDEVICEUNKNOWN = "edeviceunknown", EBUSY = "ebusy", ENOTALLOWED = "enotallowed", ESYSTEMDENIED = "esystemdenied", - ENOTSUPPORTED = "enotsupported" + ENOTSUPPORTED = "enotsupported", + ESYSTEMUNINITIALIZED = "esystemuninitialized" } export interface InputEvents { + notify_state_changed: { + oldState: InputState, + newState: InputState + }, + notify_voice_start: {}, - notify_voice_end: {} + notify_voice_end: {}, + + notify_filter_mode_changed: { oldMode: FilterMode, newMode: FilterMode }, + notify_device_changed: { oldDeviceId: string, newDeviceId: string }, } export enum FilterMode { @@ -77,7 +84,7 @@ export interface AbstractInput { currentState() : InputState; destroy(); - start() : Promise; + start() : Promise; stop() : Promise; /* @@ -91,10 +98,12 @@ export interface AbstractInput { currentDeviceId() : string | undefined; - /* + /** * This method should not throw! - * If the target device is unknown than it should return EDEVICEUNKNOWN on start. - * After changing the device, the input state falls to InputState.PAUSED. + * If the target device is unknown, it should return `InputStartError.EDEVICEUNKNOWN` on start. + * If the device is different than the current device the recorder stops. + * + * When the device has been changed the event `notify_device_changed` will be fired. */ setDeviceId(device: string) : Promise; @@ -104,7 +113,6 @@ export interface AbstractInput { supportsFilter(type: FilterType) : boolean; createFilter(type: T, priority: number) : FilterTypeClass; removeFilter(filter: Filter); - /* resetFilter(); */ getVolume() : number; setVolume(volume: number); diff --git a/tsconfig.json b/tsconfig.json index 8a64507a..1d263934 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,7 @@ "target": "es6", "module": "commonjs", "sourceMap": true, - "lib": ["es6", "dom", "dom.iterable"], + "lib": ["ES7", "dom", "dom.iterable"], "removeComments": true, /* we dont really need them within the target files */ "jsx": "react", "baseUrl": ".", diff --git a/web/app/audio/Recorder.ts b/web/app/audio/Recorder.ts index 74ac8ef1..4d400f38 100644 --- a/web/app/audio/Recorder.ts +++ b/web/app/audio/Recorder.ts @@ -6,19 +6,19 @@ import { InputConsumer, InputConsumerType, InputEvents, - MediaStreamRequestResult, + InputStartError, InputState, LevelMeter, NodeInputConsumer } from "tc-shared/voice/RecorderBase"; import * as log from "tc-shared/log"; -import {LogCategory, logDebug, logWarn} from "tc-shared/log"; +import {LogCategory, logDebug} from "tc-shared/log"; import * as aplayer from "./player"; import {JAbstractFilter, JStateFilter, JThresholdFilter} from "./RecorderFilter"; import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; import {inputDeviceList} from "tc-backend/web/audio/RecorderDeviceList"; -import {requestMediaStream} from "tc-shared/media/Stream"; -import { tr } from "tc-shared/i18n/localize"; +import {requestMediaStream, stopMediaStream} from "tc-shared/media/Stream"; +import {tr} from "tc-shared/i18n/localize"; declare global { interface MediaStream { @@ -75,7 +75,7 @@ class JavascriptInput implements AbstractInput { private inputFiltered: boolean = false; private filterMode: FilterMode = FilterMode.Block; - private startPromise: Promise; + private startPromise: Promise; private volumeModifier: number = 1; @@ -101,11 +101,19 @@ class JavascriptInput implements AbstractInput { this.audioNodeVolume.gain.value = this.volumeModifier; this.initializeFilters(); - if(this.state === InputState.INITIALIZING) { - this.start().catch(error => { - logWarn(LogCategory.AUDIO, tr("Failed to automatically start audio recording: %s"), error); - }); + } + + private setState(state: InputState) { + if(this.state === state) { + return; } + + const oldState = this.state; + this.state = state; + this.events.fire("notify_state_changed", { + oldState, + newState: state + }); } private initializeFilters() { @@ -153,48 +161,58 @@ class JavascriptInput implements AbstractInput { } } - async start() : Promise { + async start() : Promise { while(this.startPromise) { try { await this.startPromise; } catch {} } - if(this.state != InputState.PAUSED) + if(this.state != InputState.PAUSED) { return; + } /* do it async since if the doStart fails on the first iteration, we're setting the start promise, after it's getting cleared */ return await (this.startPromise = Promise.resolve().then(() => this.doStart())); } - private async doStart() : Promise { + private async doStart() : Promise { try { + if(!aplayer.initialized() || !this.audioContext) { + return InputStartError.ESYSTEMUNINITIALIZED; + } + if(this.state != InputState.PAUSED) { throw tr("recorder already started"); } + this.setState(InputState.INITIALIZING); - this.state = InputState.INITIALIZING; let deviceId; if(this.deviceId === IDevice.NoDeviceId) { throw tr("no device selected"); } else if(this.deviceId === IDevice.DefaultDeviceId) { deviceId = undefined; + } else { + deviceId = this.deviceId; } - if(!this.audioContext) { - /* Awaiting the audio context to be initialized */ - return; - } - + console.error("Starting input recorder on %o - %o", this.deviceId, deviceId); const requestResult = await requestMediaStream(deviceId, undefined, "audio"); if(!(requestResult instanceof MediaStream)) { - this.state = InputState.PAUSED; + this.setState(InputState.PAUSED); return requestResult; } /* added the then body to avoid a inspection warning... */ inputDeviceList.refresh().then(() => {}); + if(this.currentStream) { + stopMediaStream(this.currentStream); + this.currentStream = undefined; + } + this.currentAudioStream?.disconnect(); + this.currentAudioStream = undefined; + this.currentStream = requestResult; for(const filter of this.registeredFilters) { @@ -202,19 +220,20 @@ class JavascriptInput implements AbstractInput { filter.setPaused(false); } } + /* TODO: Only add if we're really having a callback consumer */ this.audioNodeCallbackConsumer.addEventListener('audioprocess', this.audioScriptProcessorCallback); this.currentAudioStream = this.audioContext.createMediaStreamSource(this.currentStream); this.currentAudioStream.connect(this.audioNodeVolume); - this.state = InputState.RECORDING; + this.setState(InputState.RECORDING); this.updateFilterStatus(true); return true; } catch(error) { if(this.state == InputState.INITIALIZING) { - this.state = InputState.PAUSED; + this.setState(InputState.PAUSED); } throw error; @@ -231,7 +250,7 @@ class JavascriptInput implements AbstractInput { } catch {} } - this.state = InputState.PAUSED; + this.setState(InputState.PAUSED); if(this.currentAudioStream) { this.currentAudioStream.disconnect(); } @@ -248,9 +267,9 @@ class JavascriptInput implements AbstractInput { this.currentStream = undefined; this.currentAudioStream = undefined; - for(const f of this.registeredFilters) { - if(f.isEnabled()) { - f.setPaused(true); + for(const filter of this.registeredFilters) { + if(filter.isEnabled()) { + filter.setPaused(true); } } @@ -271,7 +290,9 @@ class JavascriptInput implements AbstractInput { log.warn(LogCategory.AUDIO, tr("Failed to stop previous record session (%o)"), error); } + const oldDeviceId = deviceId; this.deviceId = deviceId; + this.events.fire("notify_device_changed", { newDeviceId: deviceId, oldDeviceId }) } @@ -452,9 +473,14 @@ class JavascriptInput implements AbstractInput { return; } + const oldMode = this.filterMode; this.filterMode = mode; this.updateFilterStatus(false); this.initializeFilters(); + this.events.fire("notify_filter_mode_changed", { + oldMode, + newMode: mode + }); } } @@ -511,13 +537,13 @@ class JavascriptLevelMeter implements LevelMeter { /* starting stream */ const _result = await requestMediaStream(this._device.deviceId, this._device.groupId, "audio"); if(!(_result instanceof MediaStream)){ - if(_result === MediaStreamRequestResult.ENOTALLOWED) + if(_result === InputStartError.ENOTALLOWED) throw tr("No permissions"); - if(_result === MediaStreamRequestResult.ENOTSUPPORTED) + if(_result === InputStartError.ENOTSUPPORTED) throw tr("Not supported"); - if(_result === MediaStreamRequestResult.EBUSY) + if(_result === InputStartError.EBUSY) throw tr("Device busy"); - if(_result === MediaStreamRequestResult.EUNKNOWN) + if(_result === InputStartError.EUNKNOWN) throw tr("an error occurred"); throw _result; } diff --git a/web/app/legacy/audio-lib/index.ts b/web/app/legacy/audio-lib/index.ts index c68d01a4..f72c3298 100644 --- a/web/app/legacy/audio-lib/index.ts +++ b/web/app/legacy/audio-lib/index.ts @@ -22,7 +22,7 @@ export class AudioLibrary { } private static spawnNewWorker() : Worker { - /* + /* * Attention don't use () => new Worker(...). * This confuses the worker plugin and will not emit any modules */ diff --git a/web/app/legacy/voice/VoiceHandler.ts b/web/app/legacy/voice/VoiceHandler.ts index ad4279fa..556e84df 100644 --- a/web/app/legacy/voice/VoiceHandler.ts +++ b/web/app/legacy/voice/VoiceHandler.ts @@ -92,7 +92,7 @@ export class VoiceConnection extends AbstractVoiceConnection { this.acquireVoiceRecorder(undefined, true).catch(error => { log.warn(LogCategory.VOICE, tr("Failed to release voice recorder: %o"), error); }).then(() => { - for(const client of Object.values(this.voiceClients)) { + for(const client of Object.keys(this.voiceClients).map(clientId => this.voiceClients[clientId])) { client.abortReplay(); } this.voiceClients = undefined; @@ -127,6 +127,7 @@ export class VoiceConnection extends AbstractVoiceConnection { await recorder?.unmount(); this.handleRecorderStop(); + const oldRecorder = recorder; this.currentAudioSource = recorder; if(recorder) { @@ -156,7 +157,10 @@ export class VoiceConnection extends AbstractVoiceConnection { await this.voiceBridge?.setInput(undefined); } - this.events.fire("notify_recorder_changed"); + this.events.fire("notify_recorder_changed", { + oldRecorder: oldRecorder, + newRecorder: recorder + }); } public startVoiceBridge() { diff --git a/web/app/voice/Connection.ts b/web/app/voice/Connection.ts index c09a7887..289b10da 100644 --- a/web/app/voice/Connection.ts +++ b/web/app/voice/Connection.ts @@ -160,6 +160,7 @@ export class RtpVoiceConnection extends AbstractVoiceConnection { await recorder?.unmount(); this.handleRecorderStop(); + const oldRecorder = recorder; this.currentAudioSource = recorder; if(recorder) { @@ -195,7 +196,10 @@ export class RtpVoiceConnection extends AbstractVoiceConnection { } } - this.events.fire("notify_recorder_changed"); + this.events.fire("notify_recorder_changed", { + oldRecorder, + newRecorder: recorder + }); } private handleRecorderStop() {