From 0d7a34c31acb2f734a85bd9728f75a97cb044cc7 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Thu, 13 Aug 2020 13:05:37 +0200 Subject: [PATCH 1/9] Commiting changes before going on vacation (Still contains errors) --- ChangeLog.md | 2 +- loader/app/targets/shared.ts | 16 +- shared/backend.d/audio/recorder.d.ts | 9 - shared/img/client-icons/microphone_broken.svg | 25 + shared/js/ConnectionHandler.ts | 10 +- shared/js/audio/recorder.ts | 157 ++++ shared/js/main.tsx | 5 +- shared/js/ui/modal/settings/Microphone.scss | 18 +- shared/js/ui/modal/settings/Microphone.tsx | 137 ++- .../ui/modal/settings/MicrophoneRenderer.tsx | 142 ++- shared/js/voice/Filter.ts | 53 ++ shared/js/voice/RecorderBase.ts | 92 +- shared/js/voice/RecorderProfile.ts | 164 ++-- shared/svg-sprites/client-icons.d.ts | 5 +- web/app/audio/Recorder.ts | 766 +++++++++++++++ web/app/audio/RecorderFilter.ts | 242 +++++ web/app/audio/player.ts | 30 +- web/app/audio/recorder.ts | 877 ------------------ web/app/connection/ServerConnection.ts | 1 - web/app/hooks/AudioRecorder.ts | 4 + web/app/{factories => hooks}/ExternalModal.ts | 0 .../{factories => hooks}/ServerConnection.ts | 0 web/app/index.ts | 5 +- 23 files changed, 1624 insertions(+), 1136 deletions(-) delete mode 100644 shared/backend.d/audio/recorder.d.ts create mode 100644 shared/img/client-icons/microphone_broken.svg create mode 100644 shared/js/audio/recorder.ts create mode 100644 shared/js/voice/Filter.ts create mode 100644 web/app/audio/Recorder.ts create mode 100644 web/app/audio/RecorderFilter.ts delete mode 100644 web/app/audio/recorder.ts create mode 100644 web/app/hooks/AudioRecorder.ts rename web/app/{factories => hooks}/ExternalModal.ts (100%) rename web/app/{factories => hooks}/ServerConnection.ts (100%) diff --git a/ChangeLog.md b/ChangeLog.md index f56b5f25..5159bc25 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,7 +1,7 @@ # Changelog: * **11.08.20** - Fixed the voice push to talk delay - + /* FIXME: Newcomer modal with the microphone */ * **09.08.20** - Added a "watch to gather" context menu entry for clients - Disassembled the current client icon sprite into his icons diff --git a/loader/app/targets/shared.ts b/loader/app/targets/shared.ts index 834a934a..3ca6cf57 100644 --- a/loader/app/targets/shared.ts +++ b/loader/app/targets/shared.ts @@ -1,6 +1,15 @@ import * as loader from "../loader/loader"; import {Stage} from "../loader/loader"; -import {detect as detectBrowser} from "detect-browser"; +import { + BrowserInfo, + detect as detectBrowser, +} from "detect-browser"; + +declare global { + interface Window { + detectedBrowser: BrowserInfo + } +} if(__build.target === "web") { loader.register_task(Stage.SETUP, { @@ -13,14 +22,15 @@ if(__build.target === "web") { return; console.log("Resolved browser manufacturer to \"%s\" version \"%s\" on %s", browser.name, browser.version, browser.os); - if(browser.type && browser.type !== "browser") { + if(browser.type !== "browser") { loader.critical_error("Your device isn't supported.", "User agent type " + browser.type + " isn't supported."); throw "unsupported user type"; } + window.detectedBrowser = browser; + switch (browser?.name) { case "aol": - case "bot": case "crios": case "ie": loader.critical_error("Browser not supported", "We're sorry, but your browser isn't supported."); diff --git a/shared/backend.d/audio/recorder.d.ts b/shared/backend.d/audio/recorder.d.ts deleted file mode 100644 index a5fdd45d..00000000 --- a/shared/backend.d/audio/recorder.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {AbstractInput, InputDevice, LevelMeter} from "tc-shared/voice/RecorderBase"; - -export function devices() : InputDevice[]; - -export function device_refresh_available() : boolean; -export function refresh_devices() : Promise; - -export function create_input() : AbstractInput; -export function create_levelmeter(device: InputDevice) : Promise; \ No newline at end of file diff --git a/shared/img/client-icons/microphone_broken.svg b/shared/img/client-icons/microphone_broken.svg new file mode 100644 index 00000000..c632b55d --- /dev/null +++ b/shared/img/client-icons/microphone_broken.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts index 2c8f2a5a..a307cf25 100644 --- a/shared/js/ConnectionHandler.ts +++ b/shared/js/ConnectionHandler.ts @@ -38,6 +38,7 @@ import {PluginCmdRegistry} from "tc-shared/connection/PluginCmdHandler"; import {W2GPluginCmdHandler} from "tc-shared/video-viewer/W2GPlugin"; import {VoiceConnectionStatus} from "tc-shared/connection/VoiceConnection"; import {getServerConnectionFactory} from "tc-shared/connection/ConnectionFactory"; +import {getRecorderBackend} from "tc-shared/audio/recorder"; export enum DisconnectReason { HANDLER_DESTROYED, @@ -738,6 +739,8 @@ export class ConnectionHandler { const support_record = basic_voice_support && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); const support_playback = basic_voice_support && (!targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec)); + const hasInputDevice = getRecorderBackend().getDeviceList().getPermissionState() === "granted" && !!vconnection.voice_recorder(); + const property_update = { client_input_muted: this.client_status.input_muted, client_output_muted: this.client_status.output_muted @@ -749,12 +752,11 @@ export class ConnectionHandler { if(!this.serverConnection.connected() || vconnection.getConnectionState() !== VoiceConnectionStatus.Connected) { property_update["client_input_hardware"] = false; property_update["client_output_hardware"] = false; - this.client_status.input_hardware = true; /* IDK if we have input hardware or not, but it dosn't matter at all so */ + this.client_status.input_hardware = hasInputDevice; /* no icons are shown so no update at all */ } else { - const audio_source = vconnection.voice_recorder(); - const recording_supported = typeof(audio_source) !== "undefined" && audio_source.record_supported && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); + const recording_supported = hasInputDevice && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); const playback_supported = !targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec); property_update["client_input_hardware"] = recording_supported; @@ -807,7 +809,7 @@ export class ConnectionHandler { this.client_status.sound_record_supported = support_record; this.client_status.sound_playback_supported = support_playback; - if(vconnection && vconnection.voice_recorder() && vconnection.voice_recorder().record_supported) { + if(vconnection && vconnection.voice_recorder()) { const active = !this.client_status.input_muted && !this.client_status.output_muted; /* No need to start the microphone when we're not even connected */ diff --git a/shared/js/audio/recorder.ts b/shared/js/audio/recorder.ts new file mode 100644 index 00000000..c70f8085 --- /dev/null +++ b/shared/js/audio/recorder.ts @@ -0,0 +1,157 @@ +import {AbstractInput, LevelMeter} from "tc-shared/voice/RecorderBase"; +import {Registry} from "tc-shared/events"; + +export type DeviceQueryResult = {} + +export interface AudioRecorderBacked { + createInput() : AbstractInput; + createLevelMeter(device: IDevice) : Promise; + + getDeviceList() : DeviceList; +} + +export interface DeviceListEvents { + /* + * Should only trigger if the list really changed. + */ + notify_list_updated: { + removedDeviceCount: number, + addedDeviceCount: number + }, + + notify_state_changed: { + oldState: DeviceListState; + newState: DeviceListState; + }, + + notify_permissions_changed: { + oldState: PermissionState, + newState: PermissionState + } +} + +export type DeviceListState = "healthy" | "uninitialized" | "no-permissions" | "error"; + +export interface IDevice { + deviceId: string; + + driver: string; + name: string; +} +export namespace IDevice { + export const NoDeviceId = "none"; +} + +export type PermissionState = "granted" | "denied" | "unknown"; + +export interface DeviceList { + getEvents() : Registry; + + isRefreshAvailable() : boolean; + refresh() : Promise; + + /* implicitly update our own permission state */ + requestPermissions() : Promise; + getPermissionState() : PermissionState; + + getStatus() : DeviceListState; + getDevices() : IDevice[]; + + getDefaultDeviceId() : string; + + awaitHealthy(): Promise; + awaitInitialized() : Promise; +} + +export abstract class AbstractDeviceList implements DeviceList { + protected readonly events: Registry; + protected listState: DeviceListState; + protected permissionState: PermissionState; + + protected constructor() { + this.events = new Registry(); + this.permissionState = "unknown"; + this.listState = "uninitialized"; + } + + getStatus(): DeviceListState { + return this.listState; + } + + getPermissionState(): PermissionState { + return this.permissionState; + } + + protected setState(state: DeviceListState) { + if(this.listState === state) + return; + + const oldState = this.listState; + this.listState = state; + this.events.fire("notify_state_changed", { oldState: oldState, newState: state }); + } + + protected setPermissionState(state: PermissionState) { + if(this.permissionState === state) + return; + + const oldState = this.permissionState; + this.permissionState = state; + this.events.fire("notify_permissions_changed", { oldState: oldState, newState: state }); + } + + awaitInitialized(): Promise { + if(this.listState !== "uninitialized") + return Promise.resolve(); + + return new Promise(resolve => { + const callback = (event: DeviceListEvents["notify_state_changed"]) => { + if(event.newState !== "uninitialized") + return; + + this.events.off("notify_state_changed", callback); + resolve(); + }; + this.events.on("notify_state_changed", callback); + }); + } + + awaitHealthy(): Promise { + if(this.listState === "healthy") + return Promise.resolve(); + + return new Promise(resolve => { + const callback = (event: DeviceListEvents["notify_state_changed"]) => { + if(event.newState !== "healthy") + return; + + this.events.off("notify_state_changed", callback); + resolve(); + }; + this.events.on("notify_state_changed", callback); + }); + } + + abstract getDefaultDeviceId(): string; + abstract getDevices(): IDevice[]; + abstract getEvents(): Registry; + abstract isRefreshAvailable(): boolean; + abstract refresh(): Promise; + abstract requestPermissions(): Promise; +} + +let recorderBackend: AudioRecorderBacked; + +export function getRecorderBackend() : AudioRecorderBacked { + if(typeof recorderBackend === "undefined") + throw tr("the recorder backend hasn't been set yet"); + + return recorderBackend; +} + +export function setRecorderBackend(instance: AudioRecorderBacked) { + if(typeof recorderBackend !== "undefined") + throw tr("a recorder backend has already been initialized"); + + recorderBackend = instance; +} diff --git a/shared/js/main.tsx b/shared/js/main.tsx index f36c7c2d..86502736 100644 --- a/shared/js/main.tsx +++ b/shared/js/main.tsx @@ -21,7 +21,6 @@ import {spawnYesNo} from "tc-shared/ui/modal/ModalYesNo"; import {formatMessage} from "tc-shared/ui/frames/chat"; import {openModalNewcomer} from "tc-shared/ui/modal/ModalNewcomer"; import * as aplayer from "tc-backend/audio/player"; -import * as arecorder from "tc-backend/audio/recorder"; import * as ppt from "tc-backend/ppt"; import * as keycontrol from "./KeyControl"; import * as React from "react"; @@ -182,8 +181,6 @@ async function initialize_app() { aplayer.on_ready(() => aplayer.set_master_volume(settings.global(Settings.KEY_SOUND_MASTER) / 100)); else log.warn(LogCategory.GENERAL, tr("Client does not support aplayer.set_master_volume()... May client is too old?")); - if(arecorder.device_refresh_available()) - arecorder.refresh_devices(); }); set_default_recorder(new RecorderProfile("default")); @@ -512,7 +509,7 @@ const task_teaweb_starter: loader.Task = { if(!aplayer.initializeFromGesture) { console.error(tr("Missing aplayer.initializeFromGesture")); } else - $(document).one('click', event => aplayer.initializeFromGesture()); + $(document).one('click', () => aplayer.initializeFromGesture()); } } catch (ex) { console.error(ex.stack); diff --git a/shared/js/ui/modal/settings/Microphone.scss b/shared/js/ui/modal/settings/Microphone.scss index 99de55c1..d1b4446e 100644 --- a/shared/js/ui/modal/settings/Microphone.scss +++ b/shared/js/ui/modal/settings/Microphone.scss @@ -241,7 +241,7 @@ } .header { - height: 2.6em; + height: 3em; flex-grow: 0; flex-shrink: 0; @@ -266,7 +266,7 @@ white-space: nowrap; } - .btn { + button { flex-shrink: 0; flex-grow: 0; @@ -452,10 +452,24 @@ text-align: center; } + button { + width: 10em; + align-self: center; + margin-top: 2em; + } + &.hidden { pointer-events: none; opacity: 0; } + + :global(.icon_em) { + align-self: center; + font-size: 10em; + + margin-bottom: .25em; + margin-top: -.25em; + } } } diff --git a/shared/js/ui/modal/settings/Microphone.tsx b/shared/js/ui/modal/settings/Microphone.tsx index 3d8449a6..77bdb194 100644 --- a/shared/js/ui/modal/settings/Microphone.tsx +++ b/shared/js/ui/modal/settings/Microphone.tsx @@ -2,7 +2,6 @@ import * as aplayer from "tc-backend/audio/player"; import * as React from "react"; import {Registry} from "tc-shared/events"; import {LevelMeter} from "tc-shared/voice/RecorderBase"; -import * as arecorder from "tc-backend/audio/recorder"; import * as log from "tc-shared/log"; import {LogCategory, logWarn} from "tc-shared/log"; import {default_recorder} from "tc-shared/voice/RecorderProfile"; @@ -12,6 +11,7 @@ import {spawnReactModal} from "tc-shared/ui/react-elements/Modal"; import {InternalModal} from "tc-shared/ui/react-elements/internal-modal/Controller"; import {Translatable} from "tc-shared/ui/react-elements/i18n"; import {MicrophoneSettings} from "tc-shared/ui/modal/settings/MicrophoneRenderer"; +import {DeviceListState, getRecorderBackend, IDevice} from "tc-shared/audio/recorder"; export type MicrophoneSetting = "volume" | "vad-type" | "ppt-key" | "ppt-release-delay" | "ppt-release-delay-active" | "threshold-threshold"; @@ -29,6 +29,7 @@ export interface MicrophoneSettingsEvents { setting: MicrophoneSetting }, + "action_request_permissions": {}, "action_set_selected_device": { deviceId: string }, "action_set_selected_device_result": { deviceId: string, /* on error it will contain the current selected device */ @@ -48,9 +49,11 @@ export interface MicrophoneSettingsEvents { } "notify_devices": { - status: "success" | "error" | "audio-not-initialized", + status: "success" | "error" | "audio-not-initialized" | "no-permissions", error?: string, + shouldAsk?: boolean, + devices?: MicrophoneDevice[] selectedDevice?: string; }, @@ -62,13 +65,17 @@ export interface MicrophoneSettingsEvents { level?: number, error?: string - }} + }}, + + status: Exclude }, notify_destroy: {} } export function initialize_audio_microphone_controller(events: Registry) { + const recorderBackend = getRecorderBackend(); + /* level meters */ { const level_meters: {[key: string]:Promise} = {}; @@ -80,7 +87,7 @@ export function initialize_audio_microphone_controller(events: Registry e.destory()); + meter.then(e => e.destroy()); }); Object.keys(level_info).forEach(e => delete level_info[e]); }; @@ -88,37 +95,42 @@ export function initialize_audio_microphone_controller(events: Registry { destroy_meters(); - for(const device of arecorder.devices()) { - let promise = arecorder.create_levelmeter(device).then(meter => { - meter.set_observer(level => { - if(level_meters[device.unique_id] !== promise) return; /* old level meter */ + level_info["none"] = { deviceId: "none", status: "success", level: 0 }; - level_info[device.unique_id] = { - deviceId: device.unique_id, + for(const device of recorderBackend.getDeviceList().getDevices()) { + let promise = recorderBackend.createLevelMeter(device).then(meter => { + meter.set_observer(level => { + if(level_meters[device.deviceId] !== promise) return; /* old level meter */ + + level_info[device.deviceId] = { + deviceId: device.deviceId, status: "success", level: level }; }); return Promise.resolve(meter); }).catch(error => { - if(level_meters[device.unique_id] !== promise) return; /* old level meter */ - level_info[device.unique_id] = { - deviceId: device.unique_id, + if(level_meters[device.deviceId] !== promise) return; /* old level meter */ + level_info[device.deviceId] = { + deviceId: device.deviceId, status: "error", error: error }; - log.warn(LogCategory.AUDIO, tr("Failed to initialize a level meter for device %s (%s): %o"), device.unique_id, device.driver + ":" + device.name, error); + log.warn(LogCategory.AUDIO, tr("Failed to initialize a level meter for device %s (%s): %o"), device.deviceId, device.driver + ":" + device.name, error); return Promise.reject(error); }); - level_meters[device.unique_id] = promise; + level_meters[device.deviceId] = promise; } }; level_update_task = setInterval(() => { + const deviceListStatus = recorderBackend.getDeviceList().getStatus(); + events.fire("notify_device_level", { - level: level_info + level: level_info, + status: deviceListStatus === "error" ? "uninitialized" : deviceListStatus }); }, 50); @@ -142,34 +154,43 @@ export function initialize_audio_microphone_controller(events: Registry { - return arecorder.device_refresh_available() && event.refresh_list ? arecorder.refresh_devices() : Promise.resolve(); - }).catch(error => { - log.warn(LogCategory.AUDIO, tr("Failed to refresh device list: %o"), error); - return Promise.resolve(); - }).then(() => { - const devices = arecorder.devices(); + const deviceList = recorderBackend.getDeviceList(); + switch (deviceList.getStatus()) { + case "no-permissions": + events.fire_async("notify_devices", { status: "no-permissions", shouldAsk: deviceList.getPermissionState() === "denied" }); + return; + + case "uninitialized": + events.fire_async("notify_devices", { status: "audio-not-initialized" }); + return; + } + + if(event.refresh_list && deviceList.isRefreshAvailable()) { + /* will automatically trigger a device list changed event if something has changed */ + deviceList.refresh().then(() => {}); + } else { + const devices = deviceList.getDevices(); events.fire_async("notify_devices", { status: "success", - selectedDevice: default_recorder.current_device() ? default_recorder.current_device().unique_id : "none", - devices: devices.map(e => { return { id: e.unique_id, name: e.name, driver: e.driver }}) + selectedDevice: default_recorder.getDeviceId(), + devices: devices.map(e => { return { id: e.deviceId, name: e.name, driver: e.driver }}) }); - }); + } }); events.on("action_set_selected_device", event => { - const device = arecorder.devices().find(e => e.unique_id === event.deviceId); - if(!device && event.deviceId !== "none") { - events.fire_async("action_set_selected_device_result", { status: "error", error: tr("Invalid device id"), deviceId: default_recorder.current_device().unique_id }); + const device = recorderBackend.getDeviceList().getDevices().find(e => e.deviceId === event.deviceId); + if(!device && event.deviceId !== IDevice.NoDeviceId) { + events.fire_async("action_set_selected_device_result", { status: "error", error: tr("Invalid device id"), deviceId: default_recorder.getDeviceId() }); return; } default_recorder.set_device(device).then(() => { - console.debug(tr("Changed default microphone device")); + console.debug(tr("Changed default microphone device to %s"), event.deviceId); events.fire_async("action_set_selected_device_result", { status: "success", deviceId: event.deviceId }); }).catch((error) => { - log.warn(LogCategory.AUDIO, tr("Failed to change microphone to device %s: %o"), device ? device.unique_id : "none", error); + log.warn(LogCategory.AUDIO, tr("Failed to change microphone to device %s: %o"), device ? device.deviceId : IDevice.NoDeviceId, error); events.fire_async("action_set_selected_device_result", { status: "success", deviceId: event.deviceId }); }); }); @@ -265,7 +286,59 @@ export function initialize_audio_microphone_controller(events: Registry recorderBackend.getDeviceList().requestPermissions().then(result => { + console.error("Permission request result: %o", result); + + if(result === "granted") { + /* we've nothing to do, the device change event will already update out list */ + } else { + events.fire_async("notify_devices", { status: "no-permissions", shouldAsk: result === "denied" }); + return; + } + })); + + events.on("notify_destroy", recorderBackend.getDeviceList().getEvents().on("notify_list_updated", () => { + events.fire("query_devices"); + })); + + events.on("notify_destroy", recorderBackend.getDeviceList().getEvents().on("notify_state_changed", () => { + events.fire("query_devices"); + })); + if(!aplayer.initialized()) { aplayer.on_ready(() => { events.fire_async("query_devices"); }); } -} \ No newline at end of file +} + + +loader.register_task(Stage.LOADED, { + name: "test", + function: async () => { + aplayer.on_ready(() => { + const modal = spawnReactModal(class extends InternalModal { + settings = new Registry(); + constructor() { + super(); + + initialize_audio_microphone_controller(this.settings); + } + + renderBody(): React.ReactElement { + return
+ +
; + } + + title(): string | React.ReactElement { + return "test"; + } + }); + + modal.show(); + }); + }, + priority: -2 +}) diff --git a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx index 89369a89..2ad2fb7c 100644 --- a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx +++ b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx @@ -9,13 +9,13 @@ import {ClientIcon} from "svg-sprites/client-icons"; import {LoadingDots} from "tc-shared/ui/react-elements/LoadingDots"; import {createErrorModal} from "tc-shared/ui/elements/Modal"; import {Slider} from "tc-shared/ui/react-elements/Slider"; -import MicrophoneSettings = modal.settings.MicrophoneSettings; import {RadioButton} from "tc-shared/ui/react-elements/RadioButton"; import {VadType} from "tc-shared/voice/RecorderProfile"; import {key_description, KeyDescriptor} from "tc-shared/PPTListener"; import {spawnKeySelect} from "tc-shared/ui/modal/ModalKeySelect"; import {Checkbox} from "tc-shared/ui/react-elements/Checkbox"; import {BoxedInputField} from "tc-shared/ui/react-elements/InputField"; +import {IDevice} from "tc-shared/audio/recorder"; const cssStyle = require("./Microphone.scss"); @@ -37,28 +37,41 @@ const MicrophoneStatus = (props: { state: MicrophoneSelectedState }) => { } } -type ActivityBarStatus = { mode: "success" } | { mode: "error", message: string } | { mode: "loading" }; +type ActivityBarStatus = { mode: "success" } | { mode: "error", message: string } | { mode: "loading" } | { mode: "uninitialized" }; const ActivityBar = (props: { events: Registry, deviceId: string, disabled?: boolean }) => { const refHider = useRef(); const [ status, setStatus ] = useState({ mode: "loading" }); props.events.reactUse("notify_device_level", event => { - const device = event.level[props.deviceId]; - if(!device) { - if(status.mode === "loading") + if(event.status === "uninitialized") { + if(status.mode === "uninitialized") return; - setStatus({ mode: "loading" }); - } else if(device.status === "success") { - if(status.mode !== "success") { - setStatus({ mode: "success" }); - } - refHider.current.style.width = (100 - device.level) + "%"; + setStatus({ mode: "uninitialized" }); + } else if(event.status === "no-permissions") { + const noPermissionsMessage = tr("no permissions"); + if(status.mode === "error" && status.message === noPermissionsMessage) + return; + + setStatus({ mode: "error", message: noPermissionsMessage }); } else { - if(status.mode === "error" && status.message === device.error) - return; + const device = event.level[props.deviceId]; + if(!device) { + if(status.mode === "loading") + return; - setStatus({ mode: "error", message: device.error }); + setStatus({ mode: "loading" }); + } else if(device.status === "success") { + if(status.mode !== "success") { + setStatus({ mode: "success" }); + } + refHider.current.style.width = (100 - device.level) + "%"; + } else { + if(status.mode === "error" && status.message === device.error) + return; + + setStatus({ mode: "error", message: device.error + "" }); + } } }); @@ -96,16 +109,51 @@ const Microphone = (props: { events: Registry, device:
{props.device.name}
- + {props.device.id === IDevice.NoDeviceId ? undefined : + + }
); }; +type MicrophoneListState = { + type: "normal" | "loading" | "audio-not-initialized" +} | { + type: "error", + message: string +} | { + type: "no-permissions", + bySystem: boolean +} + +const PermissionDeniedOverlay = (props: { bySystem: boolean, shown: boolean, onRequestPermission: () => void }) => { + if(props.bySystem) { + return ( + + ); + } else { + return ( + + ); + } +} + const MicrophoneList = (props: { events: Registry }) => { - const [ state, setState ] = useState<"normal" | "loading" | "error" | "audio-not-initialized">(() => { + const [ state, setState ] = useState(() => { props.events.fire("query_devices"); - return "loading"; + return { type: "loading" }; }); const [ selectedDevice, setSelectedDevice ] = useState<{ deviceId: string, mode: "selected" | "selecting" }>(); const [ deviceList, setDeviceList ] = useState([]); @@ -116,17 +164,20 @@ const MicrophoneList = (props: { events: Registry }) = switch (event.status) { case "success": setDeviceList(event.devices.slice(0)); - setState("normal"); + setState({ type: "normal" }); setSelectedDevice({ mode: "selected", deviceId: event.selectedDevice }); break; case "error": - setError(event.error || tr("Unknown error")); - setState("error"); + setState({ type: "error", message: event.error || tr("Unknown error") }); break; case "audio-not-initialized": - setState("audio-not-initialized"); + setState({ type: "audio-not-initialized" }); + break; + + case "no-permissions": + setState({ type: "no-permissions", bySystem: event.shouldAsk }); break; } }); @@ -144,25 +195,50 @@ const MicrophoneList = (props: { events: Registry }) = return (
-
+ -
+ -
+
+ Please grant access to your microphone. + +
+ props.events.fire("action_request_permissions")} + /> + + { + if(state.type !== "normal" || selectedDevice?.mode === "selecting") + return; + + props.events.fire("action_set_selected_device", { deviceId: IDevice.NoDeviceId }); + }} + /> + {deviceList.map(e => { - if(state !== "normal" || selectedDevice?.mode === "selecting") + if(state.type !== "normal" || selectedDevice?.mode === "selecting") return; props.events.fire("action_set_selected_device", { deviceId: e.id }); @@ -187,7 +263,7 @@ const ListRefreshButton = (props: { events: Registry } props.events.reactUse("query_devices", () => setUpdateTimeout(Date.now() + 2000)); - return ; } @@ -203,7 +279,6 @@ const VolumeSettings = (props: { events: Registry }) = if(event.setting !== "volume") return; - console.error("Set value: %o", event.value); refSlider.current?.setState({ value: event.value }); setValue(event.value); }); @@ -386,6 +461,7 @@ const ThresholdSelector = (props: { events: Registry } return "loading"; }); + const [ currentDevice, setCurrentDevice ] = useState(undefined); const [ isActive, setActive ] = useState(false); props.events.reactUse("notify_setting", event => { @@ -397,10 +473,18 @@ const ThresholdSelector = (props: { events: Registry } } }); + props.events.reactUse("notify_devices", event => { + setCurrentDevice(event.selectedDevice); + }); + + props.events.reactUse("action_set_selected_device_result", event => { + setCurrentDevice(event.deviceId); + }); + return (
- +
} disabled={value === "loading" || !isActive} - onChange={value => {}} + onChange={value => { props.events.fire("action_set_setting", { setting: "threshold-threshold", value: value })}} />
) diff --git a/shared/js/voice/Filter.ts b/shared/js/voice/Filter.ts new file mode 100644 index 00000000..58ff5345 --- /dev/null +++ b/shared/js/voice/Filter.ts @@ -0,0 +1,53 @@ +export enum FilterType { + THRESHOLD, + VOICE_LEVEL, + STATE +} + +export interface FilterBase { + readonly priority: number; + + set_enabled(flag: boolean) : void; + is_enabled() : boolean; +} + +export interface MarginedFilter { + get_margin_frames() : number; + set_margin_frames(value: number); +} + +export interface ThresholdFilter extends FilterBase, MarginedFilter { + readonly type: FilterType.THRESHOLD; + + get_threshold() : number; + set_threshold(value: number) : Promise; + + get_attack_smooth() : number; + get_release_smooth() : number; + + set_attack_smooth(value: number); + set_release_smooth(value: number); + + callback_level?: (value: number) => any; +} + +export interface VoiceLevelFilter extends FilterBase, MarginedFilter { + type: FilterType.VOICE_LEVEL; + + get_level() : number; +} + +export interface StateFilter extends FilterBase { + type: FilterType.STATE; + + set_state(state: boolean) : Promise; + is_active() : boolean; /* if true the the filter allows data to pass */ +} + +export type FilterTypeClass = + T extends FilterType.STATE ? StateFilter : + T extends FilterType.VOICE_LEVEL ? VoiceLevelFilter : + T extends FilterType.THRESHOLD ? ThresholdFilter : + never; + +export type Filter = ThresholdFilter | VoiceLevelFilter | StateFilter; \ No newline at end of file diff --git a/shared/js/voice/RecorderBase.ts b/shared/js/voice/RecorderBase.ts index deab0e8f..e6d36b29 100644 --- a/shared/js/voice/RecorderBase.ts +++ b/shared/js/voice/RecorderBase.ts @@ -1,14 +1,6 @@ -export interface InputDevice { - unique_id: string; - driver: string; - name: string; - default_input: boolean; - - supported: boolean; - - sample_rate: number; - channels: number; -} +import {IDevice} from "tc-shared/audio/recorder"; +import {Registry} from "tc-shared/events"; +import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; export enum InputConsumerType { CALLBACK, @@ -31,47 +23,6 @@ export interface NodeInputConsumer extends InputConsumer { } -export namespace filter { - export enum Type { - THRESHOLD, - VOICE_LEVEL, - STATE - } - - export interface Filter { - type: Type; - - is_enabled() : boolean; - } - - export interface MarginedFilter { - get_margin_frames() : number; - set_margin_frames(value: number); - } - - export interface ThresholdFilter extends Filter, MarginedFilter { - get_threshold() : number; - set_threshold(value: number) : Promise; - - get_attack_smooth() : number; - get_release_smooth() : number; - - set_attack_smooth(value: number); - set_release_smooth(value: number); - - callback_level?: (value: number) => any; - } - - export interface VoiceLevelFilter extends Filter, MarginedFilter { - get_level() : number; - } - - export interface StateFilter extends Filter { - set_state(state: boolean) : Promise; - is_active() : boolean; /* if true the the filter allows data to pass */ - } -} - export enum InputState { PAUSED, INITIALIZING, @@ -87,36 +38,39 @@ export enum InputStartResult { ENOTSUPPORTED = "enotsupported" } -export interface AbstractInput { - callback_begin: () => any; - callback_end: () => any; +export interface InputEvents { + notify_voice_start: {}, + notify_voice_end: {} +} - current_state() : InputState; +export interface AbstractInput { + readonly events: Registry; + + currentState() : InputState; start() : Promise; stop() : Promise; - current_device() : InputDevice | undefined; - set_device(device: InputDevice | undefined) : Promise; + currentDevice() : IDevice | undefined; + setDevice(device: IDevice | undefined) : Promise; - current_consumer() : InputConsumer | undefined; - set_consumer(consumer: InputConsumer) : Promise; + currentConsumer() : InputConsumer | undefined; + setConsumer(consumer: InputConsumer) : Promise; - get_filter(type: filter.Type) : filter.Filter | undefined; - supports_filter(type: filter.Type) : boolean; + supportsFilter(type: FilterType) : boolean; + createFilter(type: T, priority: number) : FilterTypeClass; - clear_filter(); - disable_filter(type: filter.Type); - enable_filter(type: filter.Type); + removeFilter(filter: Filter); + resetFilter(); - get_volume() : number; - set_volume(volume: number); + getVolume() : number; + setVolume(volume: number); } export interface LevelMeter { - device() : InputDevice; + device() : IDevice; set_observer(callback: (value: number) => any); - destory(); + destroy(); } \ No newline at end of file diff --git a/shared/js/voice/RecorderProfile.ts b/shared/js/voice/RecorderProfile.ts index c8d97bc9..b8984369 100644 --- a/shared/js/voice/RecorderProfile.ts +++ b/shared/js/voice/RecorderProfile.ts @@ -1,12 +1,13 @@ import * as log from "tc-shared/log"; -import {AbstractInput, filter, InputDevice} from "tc-shared/voice/RecorderBase"; +import {LogCategory, logWarn} from "tc-shared/log"; +import {AbstractInput} from "tc-shared/voice/RecorderBase"; import {KeyDescriptor, KeyHook} from "tc-shared/PPTListener"; -import {LogCategory} from "tc-shared/log"; import {Settings, settings} from "tc-shared/settings"; import {ConnectionHandler} from "tc-shared/ConnectionHandler"; import * as aplayer from "tc-backend/audio/player"; -import * as arecorder from "tc-backend/audio/recorder"; import * as ppt from "tc-backend/ppt"; +import {getRecorderBackend, IDevice} from "tc-shared/audio/recorder"; +import {FilterType, StateFilter} from "tc-shared/voice/Filter"; export type VadType = "threshold" | "push_to_talk" | "active"; export interface RecorderProfileConfig { @@ -46,18 +47,20 @@ export class RecorderProfile { current_handler: ConnectionHandler; - callback_input_change: (old_input: AbstractInput, new_input: AbstractInput) => Promise; + callback_input_change: (oldInput: AbstractInput | undefined, newInput: AbstractInput | undefined) => Promise; callback_start: () => any; callback_stop: () => any; callback_unmount: () => any; /* called if somebody else takes the ownership */ - record_supported: boolean; - - private pptHook: KeyHook; + private readonly pptHook: KeyHook; private pptTimeout: number; private pptHookRegistered: boolean; + private registeredFilter = { + "ppt-gate": undefined as StateFilter + } + constructor(name: string, volatile?: boolean) { this.name = name; this.volatile = typeof(volatile) === "boolean" ? volatile : false; @@ -68,84 +71,96 @@ export class RecorderProfile { clearTimeout(this.pptTimeout); this.pptTimeout = setTimeout(() => { - const f = this.input.get_filter(filter.Type.STATE) as filter.StateFilter; - if(f) f.set_state(true); + this.registeredFilter["ppt-gate"]?.set_state(true); }, Math.max(this.config.vad_push_to_talk.delay, 0)); }, + callback_press: () => { if(this.pptTimeout) clearTimeout(this.pptTimeout); - const f = this.input.get_filter(filter.Type.STATE) as filter.StateFilter; - if(f) f.set_state(false); + this.registeredFilter["ppt-gate"]?.set_state(false); }, cancel: false } as KeyHook; this.pptHookRegistered = false; - this.record_supported = true; } async initialize() : Promise { + { + let config = {}; + try { + config = settings.static_global(Settings.FN_PROFILE_RECORD(this.name), {}) as RecorderProfileConfig; + } catch (error) { + logWarn(LogCategory.AUDIO, tr("Failed to load old recorder profile config for %s"), this.name); + } + + /* default values */ + this.config = { + version: 1, + device_id: undefined, + volume: 100, + + vad_threshold: { + threshold: 25 + }, + vad_type: "threshold", + vad_push_to_talk: { + delay: 300, + key_alt: false, + key_ctrl: false, + key_shift: false, + key_windows: false, + key_code: 't' + } + }; + + Object.assign(this.config, config || {}); + } + aplayer.on_ready(async () => { + await getRecorderBackend().getDeviceList().awaitHealthy(); + this.initialize_input(); await this.load(); - await this.reinitialize_filter(); + await this.reinitializeFilter(); }); } private initialize_input() { - this.input = arecorder.create_input(); - this.input.callback_begin = () => { + this.input = getRecorderBackend().createInput(); + + this.input.events.on("notify_voice_start", () => { log.debug(LogCategory.VOICE, "Voice start"); if(this.callback_start) this.callback_start(); - }; + }); - this.input.callback_end = () => { + this.input.events.on("notify_voice_end", () => { log.debug(LogCategory.VOICE, "Voice end"); if(this.callback_stop) this.callback_stop(); - }; + }); //TODO: Await etc? this.callback_input_change && this.callback_input_change(undefined, this.input); } private async load() { - const config = settings.static_global(Settings.FN_PROFILE_RECORD(this.name), {}) as RecorderProfileConfig; - - /* default values */ - this.config = { - version: 1, - device_id: undefined, - volume: 100, - - vad_threshold: { - threshold: 25 - }, - vad_type: "threshold", - vad_push_to_talk: { - delay: 300, - key_alt: false, - key_ctrl: false, - key_shift: false, - key_windows: false, - key_code: 't' - } - }; - - Object.assign(this.config, config || {}); - this.input.set_volume(this.config.volume / 100); + this.input.setVolume(this.config.volume / 100); { - const all_devices = arecorder.devices(); - const devices = all_devices.filter(e => e.default_input || e.unique_id === this.config.device_id); - const device = devices.find(e => e.unique_id === this.config.device_id) || devices[0]; + const allDevices = getRecorderBackend().getDeviceList().getDevices(); + const defaultDeviceId = getRecorderBackend().getDeviceList().getDefaultDeviceId(); + console.error("Devices: %o | Searching: %s", allDevices, this.config.device_id); - log.info(LogCategory.VOICE, tr("Loaded record profile device %s | %o (%o)"), this.config.device_id, device, all_devices); + const devices = allDevices.filter(e => e.deviceId === defaultDeviceId || e.deviceId === this.config.device_id); + const device = devices.find(e => e.deviceId === this.config.device_id) || devices[0]; + + log.info(LogCategory.VOICE, tr("Loaded record profile device %s | %o (%o)"), this.config.device_id, device, allDevices); try { - await this.input.set_device(device); + await this.input.setDevice(device); } catch(error) { log.error(LogCategory.VOICE, tr("Failed to set input device (%o)"), error); } @@ -157,38 +172,36 @@ export class RecorderProfile { settings.changeGlobal(Settings.FN_PROFILE_RECORD(this.name), this.config); } - private async reinitialize_filter() { + private async reinitializeFilter() { if(!this.input) return; - this.input.clear_filter(); + /* TODO: Really required? If still same input we can just use the registered filters */ + + this.input.resetFilter(); + delete this.registeredFilter["ppt-gate"]; + if(this.pptHookRegistered) { ppt.unregister_key_hook(this.pptHook); this.pptHookRegistered = false; } if(this.config.vad_type === "threshold") { - const filter_ = this.input.get_filter(filter.Type.THRESHOLD) as filter.ThresholdFilter; - await filter_.set_threshold(this.config.vad_threshold.threshold); - await filter_.set_margin_frames(10); /* 500ms */ + const filter = this.input.createFilter(FilterType.THRESHOLD, 100); + await filter.set_threshold(this.config.vad_threshold.threshold); - /* legacy client support */ - if('set_attack_smooth' in filter_) - filter_.set_attack_smooth(.25); - - if('set_release_smooth' in filter_) - filter_.set_release_smooth(.9); - - this.input.enable_filter(filter.Type.THRESHOLD); + filter.set_margin_frames(10); /* 500ms */ + filter.set_attack_smooth(.25); + filter.set_release_smooth(.9); } else if(this.config.vad_type === "push_to_talk") { - const filter_ = this.input.get_filter(filter.Type.STATE) as filter.StateFilter; - await filter_.set_state(true); + const filter = this.input.createFilter(FilterType.STATE, 100); + await filter.set_state(true); + this.registeredFilter["ppt-gate"] = filter; for(const key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) this.pptHook[key] = this.config.vad_push_to_talk[key]; + ppt.register_key_hook(this.pptHook); this.pptHookRegistered = true; - - this.input.enable_filter(filter.Type.STATE); } else if(this.config.vad_type === "active") {} } @@ -199,7 +212,7 @@ export class RecorderProfile { if(this.input) { try { - await this.input.set_consumer(undefined); + await this.input.setConsumer(undefined); } catch(error) { log.warn(LogCategory.VOICE, tr("Failed to unmount input consumer for profile (%o)"), error); } @@ -220,7 +233,7 @@ export class RecorderProfile { return false; this.config.vad_type = type; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); return true; } @@ -231,7 +244,7 @@ export class RecorderProfile { return; this.config.vad_threshold.threshold = value; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); } @@ -240,7 +253,7 @@ export class RecorderProfile { for(const _key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) this.config.vad_push_to_talk[_key] = key[_key]; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); } @@ -250,25 +263,24 @@ export class RecorderProfile { return; this.config.vad_push_to_talk.delay = value; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); } - - current_device() : InputDevice | undefined { return this.input?.current_device(); } - set_device(device: InputDevice | undefined) : Promise { - this.config.device_id = device ? device.unique_id : undefined; + getDeviceId() : string { return this.config.device_id; } + set_device(device: IDevice | undefined) : Promise { + this.config.device_id = device ? device.deviceId : IDevice.NoDeviceId; this.save(); - return this.input.set_device(device); + return this.input?.setDevice(device) || Promise.resolve(); } - get_volume() : number { return this.input ? (this.input.get_volume() * 100) : this.config.volume; } + get_volume() : number { return this.input ? (this.input.getVolume() * 100) : this.config.volume; } set_volume(volume: number) { if(this.config.volume === volume) return; this.config.volume = volume; - this.input && this.input.set_volume(volume / 100); + this.input && this.input.setVolume(volume / 100); this.save(); } } \ No newline at end of file diff --git a/shared/svg-sprites/client-icons.d.ts b/shared/svg-sprites/client-icons.d.ts index 2d574769..d6d53744 100644 --- a/shared/svg-sprites/client-icons.d.ts +++ b/shared/svg-sprites/client-icons.d.ts @@ -3,9 +3,9 @@ * * This file has been auto generated by the svg-sprite generator. * Sprite source directory: D:\TeaSpeak\web\shared\img\client-icons - * Sprite count: 201 + * Sprite count: 202 */ -export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; +export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-microphone_broken" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; export enum ClientIcon { About = "client-about", @@ -114,6 +114,7 @@ export enum ClientIcon { MessageInfo = "client-message_info", MessageOutgoing = "client-message_outgoing", Messages = "client-messages", + MicrophoneBroken = "client-microphone_broken", MinimizeButton = "client-minimize_button", Moderated = "client-moderated", MoveClientToOwnChannel = "client-move_client_to_own_channel", diff --git a/web/app/audio/Recorder.ts b/web/app/audio/Recorder.ts new file mode 100644 index 00000000..a63de9b1 --- /dev/null +++ b/web/app/audio/Recorder.ts @@ -0,0 +1,766 @@ +import { + AbstractDeviceList, + AudioRecorderBacked, + DeviceList, + DeviceListEvents, + DeviceListState, + IDevice, + PermissionState +} from "tc-shared/audio/recorder"; +import {Registry} from "tc-shared/events"; +import * as rbase from "tc-shared/voice/RecorderBase"; +import { + AbstractInput, + CallbackInputConsumer, + InputConsumer, + InputConsumerType, InputEvents, + InputStartResult, + InputState, + LevelMeter, + NodeInputConsumer +} from "tc-shared/voice/RecorderBase"; +import * as log from "tc-shared/log"; +import {LogCategory, logWarn} from "tc-shared/log"; +import * as aplayer from "./player"; +import {JAbstractFilter, JStateFilter, JThresholdFilter} from "./RecorderFilter"; +import * as loader from "tc-loader"; +import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; + +declare global { + interface MediaStream { + stop(); + } +} + +export interface WebIDevice extends IDevice { + groupId: string; +} + +function getUserMediaFunctionPromise() : (constraints: MediaStreamConstraints) => Promise { + if('mediaDevices' in navigator && 'getUserMedia' in navigator.mediaDevices) + return constraints => navigator.mediaDevices.getUserMedia(constraints); + + const _callbacked_function = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; + if(!_callbacked_function) + return undefined; + + return constraints => new Promise((resolve, reject) => _callbacked_function(constraints, resolve, reject)); +} + +async function requestMicrophoneMediaStream(constraints: MediaTrackConstraints, updateDeviceList: boolean) : Promise { + const mediaFunction = getUserMediaFunctionPromise(); + if(!mediaFunction) return InputStartResult.ENOTSUPPORTED; + + try { + log.info(LogCategory.AUDIO, tr("Requesting a microphone stream for device %s in group %s"), constraints.deviceId, constraints.groupId); + const stream = mediaFunction({ audio: constraints }); + + if(updateDeviceList && inputDeviceList.getStatus() === "no-permissions") { + inputDeviceList.refresh().then(() => {}); /* added the then body to avoid a inspection warning... */ + } + + return stream; + } catch(error) { + if('name' in error) { + if(error.name === "NotAllowedError") { + log.warn(LogCategory.AUDIO, tr("Microphone request failed (No permissions). Browser message: %o"), error.message); + return InputStartResult.ENOTALLOWED; + } else { + log.warn(LogCategory.AUDIO, tr("Microphone request failed. Request resulted in error: %o: %o"), error.name, error); + } + } else { + log.warn(LogCategory.AUDIO, tr("Failed to initialize recording stream (%o)"), error); + } + + return InputStartResult.EUNKNOWN; + } +} + +async function requestMicrophonePermissions() : Promise { + const begin = Date.now(); + try { + await getUserMediaFunctionPromise()({ audio: { deviceId: "default" }, video: false }); + return "granted"; + } catch (error) { + const end = Date.now(); + const isSystem = (end - begin) < 250; + log.debug(LogCategory.AUDIO, tr("Microphone device request took %d milliseconds. System answered: %s"), end - begin, isSystem); + return "denied"; + } +} + +let inputDeviceList: WebInputDeviceList; +class WebInputDeviceList extends AbstractDeviceList { + private devices: WebIDevice[]; + + private deviceListQueryPromise: Promise; + + constructor() { + super(); + + this.devices = []; + } + + getDefaultDeviceId(): string { + return "default"; + } + + getDevices(): IDevice[] { + return this.devices; + } + + getEvents(): Registry { + return this.events; + } + + getStatus(): DeviceListState { + return this.listState; + } + + isRefreshAvailable(): boolean { + return true; + } + + refresh(askPermissions?: boolean): Promise { + return this.queryDevices(askPermissions === true); + } + + async requestPermissions(): Promise { + if(this.permissionState !== "unknown") + return this.permissionState; + + let result = await requestMicrophonePermissions(); + if(result === "granted" && this.listState === "no-permissions") { + /* if called within doQueryDevices, queryDevices will just return the promise */ + this.queryDevices(false).then(() => {}); + } + this.setPermissionState(result); + return result; + } + + private queryDevices(askPermissions: boolean) : Promise { + if(this.deviceListQueryPromise) + return this.deviceListQueryPromise; + + this.deviceListQueryPromise = this.doQueryDevices(askPermissions).catch(error => { + log.error(LogCategory.AUDIO, tr("Failed to query microphone devices (%o)"), error); + + if(this.listState !== "healthy") + this.listState = "error"; + }).then(() => { + this.deviceListQueryPromise = undefined; + }); + + return this.deviceListQueryPromise || Promise.resolve(); + } + + private async doQueryDevices(askPermissions: boolean) { + let devices = await navigator.mediaDevices.enumerateDevices(); + let hasPermissions = devices.findIndex(e => e.label !== "") !== -1; + + if(!hasPermissions && askPermissions) { + this.setState("no-permissions"); + + let skipPermissionAsk = false; + if('permissions' in navigator && 'query' in navigator.permissions) { + try { + const result = await navigator.permissions.query({ name: "microphone" }); + if(result.state === "denied") { + this.setPermissionState("denied"); + skipPermissionAsk = true; + } + } catch (error) { + logWarn(LogCategory.GENERAL, tr("Failed to query for microphone permissions: %s"), error); + } + } + + if(skipPermissionAsk) { + /* request permissions */ + hasPermissions = await this.requestPermissions() === "granted"; + if(hasPermissions) { + devices = await navigator.mediaDevices.enumerateDevices(); + } + } + } + if(hasPermissions) { + this.setPermissionState("granted"); + } + + if(window.detectedBrowser?.name === "firefox") { + devices = [{ + label: tr("Default Firefox device"), + groupId: "default", + deviceId: "default", + kind: "audioinput", + + toJSON: undefined + }]; + } + + const inputDevices = devices.filter(e => e.kind === "audioinput"); + + const oldDeviceList = this.devices; + this.devices = []; + + let devicesAdded = 0; + for(const device of inputDevices) { + const oldIndex = oldDeviceList.findIndex(e => e.deviceId === device.deviceId); + if(oldIndex === -1) { + devicesAdded++; + } else { + oldDeviceList.splice(oldIndex, 1); + } + + this.devices.push({ + deviceId: device.deviceId, + driver: "WebAudio", + groupId: device.groupId, + name: device.label + }); + } + + this.events.fire("notify_list_updated", { addedDeviceCount: devicesAdded, removedDeviceCount: oldDeviceList.length }); + if(hasPermissions) { + this.setState("healthy"); + } else { + this.setState("no-permissions"); + } + } +} + +export class WebAudioRecorder implements AudioRecorderBacked { + createInput(): AbstractInput { + return new JavascriptInput(); + } + + async createLevelMeter(device: IDevice): Promise { + const meter = new JavascriptLevelmeter(device as any); + await meter.initialize(); + return meter; + } + + getDeviceList(): DeviceList { + return inputDeviceList; + } +} + +class JavascriptInput implements AbstractInput { + public readonly events: Registry; + + private _state: InputState = InputState.PAUSED; + private _current_device: WebIDevice | undefined; + private _current_consumer: InputConsumer; + + private _current_stream: MediaStream; + private _current_audio_stream: MediaStreamAudioSourceNode; + + private _audio_context: AudioContext; + private _source_node: AudioNode; /* last node which could be connected to the target; target might be the _consumer_node */ + private _consumer_callback_node: ScriptProcessorNode; + private readonly _consumer_audio_callback; + private _volume_node: GainNode; + private _mute_node: GainNode; + + private registeredFilters: (Filter & JAbstractFilter)[] = []; + private _filter_active: boolean = false; + + private _volume: number = 1; + + callback_begin: () => any = undefined; + callback_end: () => any = undefined; + + constructor() { + this.events = new Registry(); + + aplayer.on_ready(() => this._audio_initialized()); + this._consumer_audio_callback = this._audio_callback.bind(this); + } + + private _audio_initialized() { + this._audio_context = aplayer.context(); + if(!this._audio_context) + return; + + this._mute_node = this._audio_context.createGain(); + this._mute_node.gain.value = 0; + this._mute_node.connect(this._audio_context.destination); + + this._consumer_callback_node = this._audio_context.createScriptProcessor(1024 * 4); + this._consumer_callback_node.connect(this._mute_node); + + this._volume_node = this._audio_context.createGain(); + this._volume_node.gain.value = this._volume; + + this.initializeFilters(); + if(this._state === InputState.INITIALIZING) + this.start(); + } + + private initializeFilters() { + for(const filter of this.registeredFilters) { + if(filter.is_enabled()) + filter.finalize(); + } + + this.registeredFilters.sort((a, b) => a.priority - b.priority); + if(this._audio_context && this._volume_node) { + const active_filter = this.registeredFilters.filter(e => e.is_enabled()); + let stream: AudioNode = this._volume_node; + for(const f of active_filter) { + f.initialize(this._audio_context, stream); + stream = f.audio_node; + } + this._switch_source_node(stream); + } + } + + private _audio_callback(event: AudioProcessingEvent) { + if(!this._current_consumer || this._current_consumer.type !== InputConsumerType.CALLBACK) + return; + + const callback = this._current_consumer as CallbackInputConsumer; + if(callback.callback_audio) + callback.callback_audio(event.inputBuffer); + + if(callback.callback_buffer) { + log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!")); + } + } + + current_state() : InputState { return this._state; }; + + private _start_promise: Promise; + async start() : Promise { + if(this._start_promise) { + try { + await this._start_promise; + if(this._state != InputState.PAUSED) + return; + } catch(error) { + log.debug(LogCategory.AUDIO, tr("JavascriptInput:start() Start promise await resulted in an error: %o"), error); + } + } + + return await (this._start_promise = this._start()); + } + + /* request permission for devices only one per time! */ + private static _running_request: Promise; + static async request_media_stream(device_id: string, group_id: string) : Promise { + while(this._running_request) { + try { + await this._running_request; + } catch(error) { } + } + + const audio_constrains: MediaTrackConstraints = {}; + if(window.detectedBrowser?.name === "firefox") { + /* + * Firefox only allows to open one mic as well deciding whats the input device it. + * It does not respect the deviceId nor the groupId + */ + } else { + audio_constrains.deviceId = device_id; + audio_constrains.groupId = group_id; + } + + audio_constrains.echoCancellation = true; + audio_constrains.autoGainControl = true; + audio_constrains.noiseSuppression = true; + + const promise = (this._running_request = requestMicrophoneMediaStream(audio_constrains, true)); + try { + return await this._running_request; + } finally { + if(this._running_request === promise) + this._running_request = undefined; + } + } + + private async _start() : Promise { + try { + if(this._state != InputState.PAUSED) + throw tr("recorder already started"); + + this._state = InputState.INITIALIZING; + if(!this._current_device) + throw tr("invalid device"); + + if(!this._audio_context) { + debugger; + throw tr("missing audio context"); + } + + const _result = await JavascriptInput.request_media_stream(this._current_device.deviceId, this._current_device.groupId); + if(!(_result instanceof MediaStream)) { + this._state = InputState.PAUSED; + return _result; + } + this._current_stream = _result; + + for(const f of this.registeredFilters) { + if(f.is_enabled()) { + f.set_pause(false); + } + } + this._consumer_callback_node.addEventListener('audioprocess', this._consumer_audio_callback); + + this._current_audio_stream = this._audio_context.createMediaStreamSource(this._current_stream); + this._current_audio_stream.connect(this._volume_node); + this._state = InputState.RECORDING; + return InputStartResult.EOK; + } catch(error) { + if(this._state == InputState.INITIALIZING) { + this._state = InputState.PAUSED; + } + throw error; + } finally { + this._start_promise = undefined; + } + } + + async stop() { + /* await all starts */ + try { + if(this._start_promise) + await this._start_promise; + } catch(error) {} + + this._state = InputState.PAUSED; + if(this._current_audio_stream) { + this._current_audio_stream.disconnect(); + } + + if(this._current_stream) { + if(this._current_stream.stop) { + this._current_stream.stop(); + } else { + this._current_stream.getTracks().forEach(value => { + value.stop(); + }); + } + } + + this._current_stream = undefined; + this._current_audio_stream = undefined; + for(const f of this.registeredFilters) { + if(f.is_enabled()) { + f.set_pause(true); + } + } + + if(this._consumer_callback_node) { + this._consumer_callback_node.removeEventListener('audioprocess', this._consumer_audio_callback); + } + return undefined; + } + + + current_device(): IDevice | undefined { + return this._current_device; + } + + async set_device(device: IDevice | undefined) { + if(this._current_device === device) + return; + + const savedState = this._state; + try { + await this.stop(); + } catch(error) { + log.warn(LogCategory.AUDIO, tr("Failed to stop previous record session (%o)"), error); + } + + this._current_device = device as any; + if(!device) { + this._state = savedState === InputState.PAUSED ? InputState.PAUSED : InputState.DRY; + return; + } + + if(savedState !== InputState.PAUSED) { + try { + await this.start() + } catch(error) { + log.warn(LogCategory.AUDIO, tr("Failed to start new recording stream (%o)"), error); + throw "failed to start record"; + } + } + return; + } + + + createFilter(type: T, priority: number): FilterTypeClass { + let filter: JAbstractFilter & Filter; + switch (type) { + case FilterType.STATE: + filter = new JStateFilter(priority); + break; + + case FilterType.THRESHOLD: + filter = new JThresholdFilter(priority); + break; + + case FilterType.VOICE_LEVEL: + throw tr("voice filter isn't supported!"); + + default: + throw tr("unknown filter type"); + } + + filter.callback_active_change = () => this._recalculate_filter_status(); + this.registeredFilters.push(filter); + this.initializeFilters(); + this._recalculate_filter_status(); + return filter as any; + } + + supportsFilter(type: FilterType): boolean { + switch (type) { + case FilterType.THRESHOLD: + case FilterType.STATE: + return true; + default: + return false; + } + } + + resetFilter() { + for(const filter of this.registeredFilters) { + filter.finalize(); + filter.enabled = false; + } + + this.registeredFilters = []; + this.initializeFilters(); + this._recalculate_filter_status(); + } + + removeFilter(filterInstance: Filter) { + const index = this.registeredFilters.indexOf(filterInstance as any); + if(index === -1) return; + + const [ filter ] = this.registeredFilters.splice(index, 1); + filter.finalize(); + filter.enabled = false; + + this.initializeFilters(); + this._recalculate_filter_status(); + } + + private _recalculate_filter_status() { + let filtered = this.registeredFilters.filter(e => e.is_enabled()).filter(e => (e as JAbstractFilter).active).length > 0; + if(filtered === this._filter_active) + return; + + this._filter_active = filtered; + if(filtered) { + if(this.callback_end) + this.callback_end(); + } else { + if(this.callback_begin) + this.callback_begin(); + } + } + + current_consumer(): InputConsumer | undefined { + return this._current_consumer; + } + + async set_consumer(consumer: InputConsumer) { + if(this._current_consumer) { + if(this._current_consumer.type == InputConsumerType.NODE) { + if(this._source_node) + (this._current_consumer as NodeInputConsumer).callback_disconnect(this._source_node) + } else if(this._current_consumer.type === InputConsumerType.CALLBACK) { + if(this._source_node) + this._source_node.disconnect(this._consumer_callback_node); + } + } + + if(consumer) { + if(consumer.type == InputConsumerType.CALLBACK) { + if(this._source_node) + this._source_node.connect(this._consumer_callback_node); + } else if(consumer.type == InputConsumerType.NODE) { + if(this._source_node) + (consumer as NodeInputConsumer).callback_node(this._source_node); + } else { + throw "native callback consumers are not supported!"; + } + } + this._current_consumer = consumer; + } + + private _switch_source_node(new_node: AudioNode) { + if(this._current_consumer) { + if(this._current_consumer.type == InputConsumerType.NODE) { + const node_consumer = this._current_consumer as NodeInputConsumer; + if(this._source_node) + node_consumer.callback_disconnect(this._source_node); + if(new_node) + node_consumer.callback_node(new_node); + } else if(this._current_consumer.type == InputConsumerType.CALLBACK) { + this._source_node.disconnect(this._consumer_callback_node); + if(new_node) + new_node.connect(this._consumer_callback_node); + } + } + this._source_node = new_node; + } + + get_volume(): number { + return this._volume; + } + + set_volume(volume: number) { + if(volume === this._volume) + return; + this._volume = volume; + this._volume_node.gain.value = volume; + } +} + +class JavascriptLevelmeter implements LevelMeter { + private static _instances: JavascriptLevelmeter[] = []; + private static _update_task: number; + + readonly _device: WebIDevice; + + private _callback: (num: number) => any; + + private _context: AudioContext; + private _gain_node: GainNode; + private _source_node: MediaStreamAudioSourceNode; + private _analyser_node: AnalyserNode; + + private _media_stream: MediaStream; + + private _analyse_buffer: Uint8Array; + + private _current_level = 0; + + constructor(device: WebIDevice) { + this._device = device; + } + + async initialize() { + try { + await new Promise((resolve, reject) => { + const timeout = setTimeout(reject, 5000); + aplayer.on_ready(() => { + clearTimeout(timeout); + resolve(); + }); + }); + } catch(error) { + throw tr("audio context timeout"); + } + this._context = aplayer.context(); + if(!this._context) throw tr("invalid context"); + + this._gain_node = this._context.createGain(); + this._gain_node.gain.setValueAtTime(0, 0); + + /* analyser node */ + this._analyser_node = this._context.createAnalyser(); + + const optimal_ftt_size = Math.ceil(this._context.sampleRate * (JThresholdFilter.update_task_interval / 1000)); + this._analyser_node.fftSize = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); + + if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser_node.fftSize) + this._analyse_buffer = new Uint8Array(this._analyser_node.fftSize); + + /* starting stream */ + const _result = await JavascriptInput.request_media_stream(this._device.deviceId, this._device.groupId); + if(!(_result instanceof MediaStream)){ + if(_result === InputStartResult.ENOTALLOWED) + throw tr("No permissions"); + if(_result === InputStartResult.ENOTSUPPORTED) + throw tr("Not supported"); + if(_result === InputStartResult.EBUSY) + throw tr("Device busy"); + if(_result === InputStartResult.EUNKNOWN) + throw tr("an error occurred"); + throw _result; + } + this._media_stream = _result; + + this._source_node = this._context.createMediaStreamSource(this._media_stream); + this._source_node.connect(this._analyser_node); + this._analyser_node.connect(this._gain_node); + this._gain_node.connect(this._context.destination); + + JavascriptLevelmeter._instances.push(this); + if(JavascriptLevelmeter._instances.length == 1) { + clearInterval(JavascriptLevelmeter._update_task); + JavascriptLevelmeter._update_task = setInterval(() => JavascriptLevelmeter._analyse_all(), JThresholdFilter.update_task_interval) as any; + } + } + + destroy() { + JavascriptLevelmeter._instances.remove(this); + if(JavascriptLevelmeter._instances.length == 0) { + clearInterval(JavascriptLevelmeter._update_task); + JavascriptLevelmeter._update_task = 0; + } + + if(this._source_node) { + this._source_node.disconnect(); + this._source_node = undefined; + } + if(this._media_stream) { + if(this._media_stream.stop) + this._media_stream.stop(); + else + this._media_stream.getTracks().forEach(value => { + value.stop(); + }); + this._media_stream = undefined; + } + if(this._gain_node) { + this._gain_node.disconnect(); + this._gain_node = undefined; + } + if(this._analyser_node) { + this._analyser_node.disconnect(); + this._analyser_node = undefined; + } + } + + device(): IDevice { + return this._device; + } + + set_observer(callback: (value: number) => any) { + this._callback = callback; + } + + private static _analyse_all() { + for(const instance of [...this._instances]) + instance._analyse(); + } + + private _analyse() { + this._analyser_node.getByteTimeDomainData(this._analyse_buffer); + + this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser_node.fftSize, this._current_level, .75); + if(this._callback) + this._callback(this._current_level); + } +} + +loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { + function: async () => { + inputDeviceList = new WebInputDeviceList(); + }, + priority: 80, + name: "initialize media devices" +}); + +loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { + function: async () => { + inputDeviceList.refresh().then(() => {}); + }, + priority: 10, + name: "query media devices" +}); \ No newline at end of file diff --git a/web/app/audio/RecorderFilter.ts b/web/app/audio/RecorderFilter.ts new file mode 100644 index 00000000..cde5ffbc --- /dev/null +++ b/web/app/audio/RecorderFilter.ts @@ -0,0 +1,242 @@ +import {FilterType, StateFilter, ThresholdFilter} from "tc-shared/voice/Filter"; + +export abstract class JAbstractFilter { + readonly priority: number; + + source_node: AudioNode; + audio_node: NodeType; + + context: AudioContext; + enabled: boolean = false; + + active: boolean = false; /* if true the filter filters! */ + callback_active_change: (new_state: boolean) => any; + + paused: boolean = true; + + constructor(priority: number) { + this.priority = priority; + } + + abstract initialize(context: AudioContext, source_node: AudioNode); + abstract finalize(); + + /* whatever the input has been paused and we don't expect any input */ + abstract set_pause(flag: boolean); + + is_enabled(): boolean { + return this.enabled; + } + + set_enabled(flag: boolean) { + this.enabled = flag; + } +} + +export class JThresholdFilter extends JAbstractFilter implements ThresholdFilter { + public static update_task_interval = 20; /* 20ms */ + + readonly type = FilterType.THRESHOLD; + callback_level?: (value: number) => any; + + private _threshold = 50; + + private _update_task: any; + private _analyser: AnalyserNode; + private _analyse_buffer: Uint8Array; + + private _silence_count = 0; + private _margin_frames = 5; + + private _current_level = 0; + private _smooth_release = 0; + private _smooth_attack = 0; + + finalize() { + this.set_pause(true); + + if(this.source_node) { + try { this.source_node.disconnect(this._analyser) } catch (error) {} + try { this.source_node.disconnect(this.audio_node) } catch (error) {} + } + + this._analyser = undefined; + this.source_node = undefined; + this.audio_node = undefined; + this.context = undefined; + } + + initialize(context: AudioContext, source_node: AudioNode) { + this.context = context; + this.source_node = source_node; + + this.audio_node = context.createGain(); + this._analyser = context.createAnalyser(); + + const optimal_ftt_size = Math.ceil((source_node.context || context).sampleRate * (JThresholdFilter.update_task_interval / 1000)); + const base2_ftt = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); + this._analyser.fftSize = base2_ftt; + + if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser.fftSize) + this._analyse_buffer = new Uint8Array(this._analyser.fftSize); + + this.active = false; + this.audio_node.gain.value = 1; + + this.source_node.connect(this.audio_node); + this.source_node.connect(this._analyser); + + /* force update paused state */ + this.set_pause(!(this.paused = !this.paused)); + } + + get_margin_frames(): number { return this._margin_frames; } + set_margin_frames(value: number) { + this._margin_frames = value; + } + + get_attack_smooth(): number { + return this._smooth_attack; + } + + get_release_smooth(): number { + return this._smooth_release; + } + + set_attack_smooth(value: number) { + this._smooth_attack = value; + } + + set_release_smooth(value: number) { + this._smooth_release = value; + } + + get_threshold(): number { + return this._threshold; + } + + set_threshold(value: number): Promise { + this._threshold = value; + return Promise.resolve(); + } + + public static process(buffer: Uint8Array, ftt_size: number, previous: number, smooth: number) { + let level; + { + let total = 0, float, rms; + + for(let index = 0; index < ftt_size; index++) { + float = ( buffer[index++] / 0x7f ) - 1; + total += (float * float); + } + rms = Math.sqrt(total / ftt_size); + let db = 20 * ( Math.log(rms) / Math.log(10) ); + // sanity check + + db = Math.max(-192, Math.min(db, 0)); + level = 100 + ( db * 1.92 ); + } + + return previous * smooth + level * (1 - smooth); + } + + private _analyse() { + this._analyser.getByteTimeDomainData(this._analyse_buffer); + + let smooth; + if(this._silence_count == 0) + smooth = this._smooth_release; + else + smooth = this._smooth_attack; + + this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser.fftSize, this._current_level, smooth); + + this._update_gain_node(); + if(this.callback_level) + this.callback_level(this._current_level); + } + + private _update_gain_node() { + let state; + if(this._current_level > this._threshold) { + this._silence_count = 0; + state = true; + } else { + state = this._silence_count++ < this._margin_frames; + } + if(state) { + this.audio_node.gain.value = 1; + if(this.active) { + this.active = false; + this.callback_active_change(false); + } + } else { + this.audio_node.gain.value = 0; + if(!this.active) { + this.active = true; + this.callback_active_change(true); + } + } + } + + set_pause(flag: boolean) { + if(flag === this.paused) return; + this.paused = flag; + + if(this.paused) { + clearInterval(this._update_task); + this._update_task = undefined; + + if(this.active) { + this.active = false; + this.callback_active_change(false); + } + } else { + if(!this._update_task && this._analyser) + this._update_task = setInterval(() => this._analyse(), JThresholdFilter.update_task_interval); + } + } +} + +export class JStateFilter extends JAbstractFilter implements StateFilter { + public readonly type = FilterType.STATE; + + finalize() { + if(this.source_node) { + try { this.source_node.disconnect(this.audio_node) } catch (error) {} + } + + this.source_node = undefined; + this.audio_node = undefined; + this.context = undefined; + } + + initialize(context: AudioContext, source_node: AudioNode) { + this.context = context; + this.source_node = source_node; + + this.audio_node = context.createGain(); + this.audio_node.gain.value = this.active ? 0 : 1; + + this.source_node.connect(this.audio_node); + } + + is_active(): boolean { + return this.active; + } + + set_state(state: boolean): Promise { + if(this.active === state) + return Promise.resolve(); + + this.active = state; + if(this.audio_node) + this.audio_node.gain.value = state ? 0 : 1; + this.callback_active_change(state); + return Promise.resolve(); + } + + set_pause(flag: boolean) { + this.paused = flag; + } +} \ No newline at end of file diff --git a/web/app/audio/player.ts b/web/app/audio/player.ts index 684524b2..e8cca985 100644 --- a/web/app/audio/player.ts +++ b/web/app/audio/player.ts @@ -1,25 +1,3 @@ -/* -import {Device} from "tc-shared/audio/player"; - -export function initialize() : boolean; -export function initialized() : boolean; - -export function context() : AudioContext; -export function get_master_volume() : number; -export function set_master_volume(volume: number); - -export function destination() : AudioNode; - -export function on_ready(cb: () => any); - -export function available_devices() : Promise; -export function set_device(device_id: string) : Promise; - -export function current_device() : Device; - -export function initializeFromGesture(); -*/ - import {Device} from "tc-shared/audio/player"; import * as log from "tc-shared/log"; import {LogCategory} from "tc-shared/log"; @@ -52,6 +30,10 @@ function fire_initialized() { function createNewContext() { audioContextInstance = new (window.webkitAudioContext || window.AudioContext)(); + audioContextInstance.onstatechange = () => { + if(audioContextInstance.state === "running") + fire_initialized(); + }; audioContextInitializeCallbacks.unshift(() => { globalAudioGainInstance = audioContextInstance.createGain(); @@ -128,9 +110,7 @@ export function current_device() : Device { export function initializeFromGesture() { if(audioContextInstance) { if(audioContextInstance.state !== "running") { - audioContextInstance.resume().then(() => { - fire_initialized(); - }).catch(error => { + audioContextInstance.resume().catch(error => { log.error(LogCategory.AUDIO, tr("Failed to initialize audio context instance from gesture: %o"), error); }); } diff --git a/web/app/audio/recorder.ts b/web/app/audio/recorder.ts deleted file mode 100644 index 66375a9a..00000000 --- a/web/app/audio/recorder.ts +++ /dev/null @@ -1,877 +0,0 @@ -import { - AbstractInput, CallbackInputConsumer, - InputConsumer, - InputConsumerType, - InputDevice, InputStartResult, - InputState, - LevelMeter, NodeInputConsumer -} from "tc-shared/voice/RecorderBase"; -import * as log from "tc-shared/log"; -import * as loader from "tc-loader"; -import {LogCategory} from "tc-shared/log"; -import * as aplayer from "./player"; -import * as rbase from "tc-shared/voice/RecorderBase"; - -declare global { - interface MediaStream { - stop(); - } -} - -let _queried_devices: JavascriptInputDevice[]; -let _queried_permissioned: boolean = false; - -export interface JavascriptInputDevice extends InputDevice { - device_id: string; - group_id: string; -} - -function getUserMediaFunctionPromise() : (constraints: MediaStreamConstraints) => Promise { - if('mediaDevices' in navigator && 'getUserMedia' in navigator.mediaDevices) - return constraints => navigator.mediaDevices.getUserMedia(constraints); - - const _callbacked_function = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; - if(!_callbacked_function) - return undefined; - - return constraints => new Promise((resolve, reject) => _callbacked_function(constraints, resolve, reject)); -} - -async function query_devices() { - const general_supported = !!getUserMediaFunctionPromise(); - - try { - const context = aplayer.context(); - const devices = await navigator.mediaDevices.enumerateDevices(); - - _queried_permissioned = false; - if(devices.filter(e => !!e.label).length > 0) - _queried_permissioned = true; - - _queried_devices = devices.filter(e => e.kind === "audioinput").map((e: MediaDeviceInfo): JavascriptInputDevice => { - return { - channels: context ? context.destination.channelCount : 2, - sample_rate: context ? context.sampleRate : 44100, - - default_input: e.deviceId == "default", - - driver: "WebAudio", - name: e.label || "device-id{" + e.deviceId+ "}", - - supported: general_supported, - - device_id: e.deviceId, - group_id: e.groupId, - - unique_id: e.deviceId - } - }); - if(_queried_devices.length > 0 && _queried_devices.filter(e => e.default_input).length == 0) - _queried_devices[0].default_input = true; - } catch(error) { - log.error(LogCategory.AUDIO, tr("Failed to query microphone devices (%o)"), error); - _queried_devices = []; - } -} - -export function devices() : InputDevice[] { - if(typeof(_queried_devices) === "undefined") - query_devices(); - - return _queried_devices || []; -} - - -export function device_refresh_available() : boolean { return true; } -export function refresh_devices() : Promise { return query_devices(); } - -export function create_input() : AbstractInput { return new JavascriptInput(); } - -export async function create_levelmeter(device: InputDevice) : Promise { - const meter = new JavascriptLevelmeter(device as any); - await meter.initialize(); - return meter; -} - -loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { - function: async () => { query_devices(); }, /* May wait for it? */ - priority: 10, - name: "query media devices" -}); - -export namespace filter { - export abstract class JAbstractFilter implements rbase.filter.Filter { - type; - - source_node: AudioNode; - audio_node: NodeType; - - context: AudioContext; - enabled: boolean = false; - - active: boolean = false; /* if true the filter filters! */ - callback_active_change: (new_state: boolean) => any; - - paused: boolean = true; - - abstract initialize(context: AudioContext, source_node: AudioNode); - abstract finalize(); - - /* whatever the input has been paused and we don't expect any input */ - abstract set_pause(flag: boolean); - - is_enabled(): boolean { - return this.enabled; - } - } - - export class JThresholdFilter extends JAbstractFilter implements rbase.filter.ThresholdFilter { - public static update_task_interval = 20; /* 20ms */ - - type = rbase.filter.Type.THRESHOLD; - callback_level?: (value: number) => any; - - private _threshold = 50; - - private _update_task: any; - private _analyser: AnalyserNode; - private _analyse_buffer: Uint8Array; - - private _silence_count = 0; - private _margin_frames = 5; - - private _current_level = 0; - private _smooth_release = 0; - private _smooth_attack = 0; - - finalize() { - this.set_pause(true); - - if(this.source_node) { - try { this.source_node.disconnect(this._analyser) } catch (error) {} - try { this.source_node.disconnect(this.audio_node) } catch (error) {} - } - - this._analyser = undefined; - this.source_node = undefined; - this.audio_node = undefined; - this.context = undefined; - } - - initialize(context: AudioContext, source_node: AudioNode) { - this.context = context; - this.source_node = source_node; - - this.audio_node = context.createGain(); - this._analyser = context.createAnalyser(); - - const optimal_ftt_size = Math.ceil((source_node.context || context).sampleRate * (JThresholdFilter.update_task_interval / 1000)); - const base2_ftt = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); - this._analyser.fftSize = base2_ftt; - - if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser.fftSize) - this._analyse_buffer = new Uint8Array(this._analyser.fftSize); - - this.active = false; - this.audio_node.gain.value = 1; - - this.source_node.connect(this.audio_node); - this.source_node.connect(this._analyser); - - /* force update paused state */ - this.set_pause(!(this.paused = !this.paused)); - } - - get_margin_frames(): number { return this._margin_frames; } - set_margin_frames(value: number) { - this._margin_frames = value; - } - - get_attack_smooth(): number { - return this._smooth_attack; - } - - get_release_smooth(): number { - return this._smooth_release; - } - - set_attack_smooth(value: number) { - this._smooth_attack = value; - } - - set_release_smooth(value: number) { - this._smooth_release = value; - } - - get_threshold(): number { - return this._threshold; - } - - set_threshold(value: number): Promise { - this._threshold = value; - return Promise.resolve(); - } - - public static process(buffer: Uint8Array, ftt_size: number, previous: number, smooth: number) { - let level; - { - let total = 0, float, rms; - - for(let index = 0; index < ftt_size; index++) { - float = ( buffer[index++] / 0x7f ) - 1; - total += (float * float); - } - rms = Math.sqrt(total / ftt_size); - let db = 20 * ( Math.log(rms) / Math.log(10) ); - // sanity check - - db = Math.max(-192, Math.min(db, 0)); - level = 100 + ( db * 1.92 ); - } - - return previous * smooth + level * (1 - smooth); - } - - private _analyse() { - this._analyser.getByteTimeDomainData(this._analyse_buffer); - - let smooth; - if(this._silence_count == 0) - smooth = this._smooth_release; - else - smooth = this._smooth_attack; - - this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser.fftSize, this._current_level, smooth); - - this._update_gain_node(); - if(this.callback_level) - this.callback_level(this._current_level); - } - - private _update_gain_node() { - let state; - if(this._current_level > this._threshold) { - this._silence_count = 0; - state = true; - } else { - state = this._silence_count++ < this._margin_frames; - } - if(state) { - this.audio_node.gain.value = 1; - if(this.active) { - this.active = false; - this.callback_active_change(false); - } - } else { - this.audio_node.gain.value = 0; - if(!this.active) { - this.active = true; - this.callback_active_change(true); - } - } - } - - set_pause(flag: boolean) { - if(flag === this.paused) return; - this.paused = flag; - - if(this.paused) { - clearInterval(this._update_task); - this._update_task = undefined; - - if(this.active) { - this.active = false; - this.callback_active_change(false); - } - } else { - if(!this._update_task && this._analyser) - this._update_task = setInterval(() => this._analyse(), JThresholdFilter.update_task_interval); - } - } - } - - export class JStateFilter extends JAbstractFilter implements rbase.filter.StateFilter { - type = rbase.filter.Type.STATE; - - finalize() { - if(this.source_node) { - try { this.source_node.disconnect(this.audio_node) } catch (error) {} - } - - this.source_node = undefined; - this.audio_node = undefined; - this.context = undefined; - } - - initialize(context: AudioContext, source_node: AudioNode) { - this.context = context; - this.source_node = source_node; - - this.audio_node = context.createGain(); - this.audio_node.gain.value = this.active ? 0 : 1; - - this.source_node.connect(this.audio_node); - } - - is_active(): boolean { - return this.active; - } - - set_state(state: boolean): Promise { - if(this.active === state) - return Promise.resolve(); - - this.active = state; - if(this.audio_node) - this.audio_node.gain.value = state ? 0 : 1; - this.callback_active_change(state); - return Promise.resolve(); - } - - set_pause(flag: boolean) { - this.paused = flag; - } - } -} - -class JavascriptInput implements AbstractInput { - private _state: InputState = InputState.PAUSED; - private _current_device: JavascriptInputDevice | undefined; - private _current_consumer: InputConsumer; - - private _current_stream: MediaStream; - private _current_audio_stream: MediaStreamAudioSourceNode; - - private _audio_context: AudioContext; - private _source_node: AudioNode; /* last node which could be connected to the target; target might be the _consumer_node */ - private _consumer_callback_node: ScriptProcessorNode; - private readonly _consumer_audio_callback; - private _volume_node: GainNode; - private _mute_node: GainNode; - - private _filters: rbase.filter.Filter[] = []; - private _filter_active: boolean = false; - - private _volume: number = 1; - - callback_begin: () => any = undefined; - callback_end: () => any = undefined; - - constructor() { - aplayer.on_ready(() => this._audio_initialized()); - this._consumer_audio_callback = this._audio_callback.bind(this); - } - - private _audio_initialized() { - this._audio_context = aplayer.context(); - if(!this._audio_context) - return; - - this._mute_node = this._audio_context.createGain(); - this._mute_node.gain.value = 0; - this._mute_node.connect(this._audio_context.destination); - - this._consumer_callback_node = this._audio_context.createScriptProcessor(1024 * 4); - this._consumer_callback_node.connect(this._mute_node); - - this._volume_node = this._audio_context.createGain(); - this._volume_node.gain.value = this._volume; - - this._initialize_filters(); - if(this._state === InputState.INITIALIZING) - this.start(); - } - - private _initialize_filters() { - const filters = this._filters as any as filter.JAbstractFilter[]; - for(const filter of filters) { - if(filter.is_enabled()) - filter.finalize(); - } - - if(this._audio_context && this._volume_node) { - const active_filter = filters.filter(e => e.is_enabled()); - let stream: AudioNode = this._volume_node; - for(const f of active_filter) { - f.initialize(this._audio_context, stream); - stream = f.audio_node; - } - this._switch_source_node(stream); - } - } - - private _audio_callback(event: AudioProcessingEvent) { - if(!this._current_consumer || this._current_consumer.type !== InputConsumerType.CALLBACK) - return; - - const callback = this._current_consumer as CallbackInputConsumer; - if(callback.callback_audio) - callback.callback_audio(event.inputBuffer); - - if(callback.callback_buffer) { - log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!")); - } - } - - current_state() : InputState { return this._state; }; - - private _start_promise: Promise; - async start() : Promise { - if(this._start_promise) { - try { - await this._start_promise; - if(this._state != InputState.PAUSED) - return; - } catch(error) { - log.debug(LogCategory.AUDIO, tr("JavascriptInput:start() Start promise await resulted in an error: %o"), error); - } - } - - return await (this._start_promise = this._start()); - } - - /* request permission for devices only one per time! */ - private static _running_request: Promise; - static async request_media_stream(device_id: string, group_id: string) : Promise { - while(this._running_request) { - try { - await this._running_request; - } catch(error) { } - } - const promise = (this._running_request = this.request_media_stream0(device_id, group_id)); - try { - return await this._running_request; - } finally { - if(this._running_request === promise) - this._running_request = undefined; - } - } - - static async request_media_stream0(device_id: string, group_id: string) : Promise { - const media_function = getUserMediaFunctionPromise(); - if(!media_function) return InputStartResult.ENOTSUPPORTED; - - try { - log.info(LogCategory.AUDIO, tr("Requesting a microphone stream for device %s in group %s"), device_id, group_id); - - const audio_constrains: MediaTrackConstraints = {}; - audio_constrains.deviceId = device_id; - audio_constrains.groupId = group_id; - - audio_constrains.echoCancellation = true; - audio_constrains.autoGainControl = true; - audio_constrains.noiseSuppression = true; - /* disabled because most the time we get a OverconstrainedError */ //audio_constrains.sampleSize = {min: 420, max: 960 * 10, ideal: 960}; - - const stream = await media_function({ - audio: audio_constrains, - video: undefined - }); - if(!_queried_permissioned) query_devices(); /* we now got permissions, requery devices */ - return stream; - } catch(error) { - if('name' in error) { - if(error.name === "NotAllowedError") { - //createErrorModal(tr("Failed to create microphone"), tr("Microphone recording failed. Please allow TeaWeb access to your microphone")).open(); - //FIXME: Move this to somewhere else! - - log.warn(LogCategory.AUDIO, tr("Microphone request failed (No permissions). Browser message: %o"), error.message); - return InputStartResult.ENOTALLOWED; - } else { - log.warn(LogCategory.AUDIO, tr("Microphone request failed. Request resulted in error: %o: %o"), error.name, error); - } - } else { - log.warn(LogCategory.AUDIO, tr("Failed to initialize recording stream (%o)"), error); - } - return InputStartResult.EUNKNOWN; - } - } - - private async _start() : Promise { - try { - if(this._state != InputState.PAUSED) - throw tr("recorder already started"); - - this._state = InputState.INITIALIZING; - if(!this._current_device) - throw tr("invalid device"); - - if(!this._audio_context) { - debugger; - throw tr("missing audio context"); - } - - const _result = await JavascriptInput.request_media_stream(this._current_device.device_id, this._current_device.group_id); - if(!(_result instanceof MediaStream)) { - this._state = InputState.PAUSED; - return _result; - } - this._current_stream = _result; - - for(const f of this._filters) - if(f.is_enabled() && f instanceof filter.JAbstractFilter) - f.set_pause(false); - this._consumer_callback_node.addEventListener('audioprocess', this._consumer_audio_callback); - - this._current_audio_stream = this._audio_context.createMediaStreamSource(this._current_stream); - this._current_audio_stream.connect(this._volume_node); - this._state = InputState.RECORDING; - return InputStartResult.EOK; - } catch(error) { - if(this._state == InputState.INITIALIZING) { - this._state = InputState.PAUSED; - } - throw error; - } finally { - this._start_promise = undefined; - } - } - - async stop() { - /* await all starts */ - try { - if(this._start_promise) - await this._start_promise; - } catch(error) {} - - this._state = InputState.PAUSED; - if(this._current_audio_stream) - this._current_audio_stream.disconnect(); - - if(this._current_stream) { - if(this._current_stream.stop) - this._current_stream.stop(); - else - this._current_stream.getTracks().forEach(value => { - value.stop(); - }); - } - - this._current_stream = undefined; - this._current_audio_stream = undefined; - for(const f of this._filters) - if(f.is_enabled() && f instanceof filter.JAbstractFilter) - f.set_pause(true); - if(this._consumer_callback_node) - this._consumer_callback_node.removeEventListener('audioprocess', this._consumer_audio_callback); - return undefined; - } - - - current_device(): InputDevice | undefined { - return this._current_device; - } - - async set_device(device: InputDevice | undefined) { - if(this._current_device === device) - return; - - - const saved_state = this._state; - try { - await this.stop(); - } catch(error) { - log.warn(LogCategory.AUDIO, tr("Failed to stop previous record session (%o)"), error); - } - - this._current_device = device as any; /* TODO: Test for device_id and device_group */ - if(!device) { - this._state = saved_state === InputState.PAUSED ? InputState.PAUSED : InputState.DRY; - return; - } - - if(saved_state !== InputState.PAUSED) { - try { - await this.start() - } catch(error) { - log.warn(LogCategory.AUDIO, tr("Failed to start new recording stream (%o)"), error); - throw "failed to start record"; - } - } - return; - } - - - get_filter(type: rbase.filter.Type): rbase.filter.Filter | undefined { - for(const filter of this._filters) - if(filter.type == type) - return filter; - - let new_filter: filter.JAbstractFilter; - switch (type) { - case rbase.filter.Type.STATE: - new_filter = new filter.JStateFilter(); - break; - case rbase.filter.Type.VOICE_LEVEL: - throw "voice filter isn't supported!"; - case rbase.filter.Type.THRESHOLD: - new_filter = new filter.JThresholdFilter(); - break; - default: - throw "invalid filter type, or type isn't implemented! (" + type + ")"; - } - - new_filter.callback_active_change = () => this._recalculate_filter_status(); - this._filters.push(new_filter as any); - this.enable_filter(type); - return new_filter as any; - } - - supports_filter(type: rbase.filter.Type) : boolean { - switch (type) { - case rbase.filter.Type.THRESHOLD: - case rbase.filter.Type.STATE: - return true; - default: - return false; - } - } - - private find_filter(type: rbase.filter.Type) : filter.JAbstractFilter | undefined { - for(const filter of this._filters) - if(filter.type == type) - return filter as any; - return undefined; - } - - clear_filter() { - for(const _filter of this._filters) { - if(!_filter.is_enabled()) - continue; - - const c_filter = _filter as any as filter.JAbstractFilter; - c_filter.finalize(); - c_filter.enabled = false; - } - - this._initialize_filters(); - this._recalculate_filter_status(); - } - - disable_filter(type: rbase.filter.Type) { - const filter = this.find_filter(type); - if(!filter) return; - - /* test if the filter is active */ - if(!filter.is_enabled()) - return; - - filter.enabled = false; - filter.set_pause(true); - filter.finalize(); - this._initialize_filters(); - this._recalculate_filter_status(); - } - - enable_filter(type: rbase.filter.Type) { - const filter = this.get_filter(type) as any as filter.JAbstractFilter; - if(filter.is_enabled()) - return; - - filter.enabled = true; - filter.set_pause(typeof this._current_audio_stream !== "object"); - this._initialize_filters(); - this._recalculate_filter_status(); - } - - private _recalculate_filter_status() { - let filtered = this._filters.filter(e => e.is_enabled()).filter(e => (e as any as filter.JAbstractFilter).active).length > 0; - if(filtered === this._filter_active) - return; - - this._filter_active = filtered; - if(filtered) { - if(this.callback_end) - this.callback_end(); - } else { - if(this.callback_begin) - this.callback_begin(); - } - } - - current_consumer(): InputConsumer | undefined { - return this._current_consumer; - } - - async set_consumer(consumer: InputConsumer) { - if(this._current_consumer) { - if(this._current_consumer.type == InputConsumerType.NODE) { - if(this._source_node) - (this._current_consumer as NodeInputConsumer).callback_disconnect(this._source_node) - } else if(this._current_consumer.type === InputConsumerType.CALLBACK) { - if(this._source_node) - this._source_node.disconnect(this._consumer_callback_node); - } - } - - if(consumer) { - if(consumer.type == InputConsumerType.CALLBACK) { - if(this._source_node) - this._source_node.connect(this._consumer_callback_node); - } else if(consumer.type == InputConsumerType.NODE) { - if(this._source_node) - (consumer as NodeInputConsumer).callback_node(this._source_node); - } else { - throw "native callback consumers are not supported!"; - } - } - this._current_consumer = consumer; - } - - private _switch_source_node(new_node: AudioNode) { - if(this._current_consumer) { - if(this._current_consumer.type == InputConsumerType.NODE) { - const node_consumer = this._current_consumer as NodeInputConsumer; - if(this._source_node) - node_consumer.callback_disconnect(this._source_node); - if(new_node) - node_consumer.callback_node(new_node); - } else if(this._current_consumer.type == InputConsumerType.CALLBACK) { - this._source_node.disconnect(this._consumer_callback_node); - if(new_node) - new_node.connect(this._consumer_callback_node); - } - } - this._source_node = new_node; - } - - get_volume(): number { - return this._volume; - } - - set_volume(volume: number) { - if(volume === this._volume) - return; - this._volume = volume; - this._volume_node.gain.value = volume; - } -} - -class JavascriptLevelmeter implements LevelMeter { - private static _instances: JavascriptLevelmeter[] = []; - private static _update_task: number; - - readonly _device: JavascriptInputDevice; - - private _callback: (num: number) => any; - - private _context: AudioContext; - private _gain_node: GainNode; - private _source_node: MediaStreamAudioSourceNode; - private _analyser_node: AnalyserNode; - - private _media_stream: MediaStream; - - private _analyse_buffer: Uint8Array; - - private _current_level = 0; - - constructor(device: JavascriptInputDevice) { - this._device = device; - } - - async initialize() { - try { - await new Promise((resolve, reject) => { - const timeout = setTimeout(reject, 5000); - aplayer.on_ready(() => { - clearTimeout(timeout); - resolve(); - }); - }); - } catch(error) { - throw tr("audio context timeout"); - } - this._context = aplayer.context(); - if(!this._context) throw tr("invalid context"); - - this._gain_node = this._context.createGain(); - this._gain_node.gain.setValueAtTime(0, 0); - - /* analyser node */ - this._analyser_node = this._context.createAnalyser(); - - const optimal_ftt_size = Math.ceil(this._context.sampleRate * (filter.JThresholdFilter.update_task_interval / 1000)); - this._analyser_node.fftSize = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); - - if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser_node.fftSize) - this._analyse_buffer = new Uint8Array(this._analyser_node.fftSize); - - /* starting stream */ - const _result = await JavascriptInput.request_media_stream(this._device.device_id, this._device.group_id); - if(!(_result instanceof MediaStream)){ - if(_result === InputStartResult.ENOTALLOWED) - throw tr("No permissions"); - if(_result === InputStartResult.ENOTSUPPORTED) - throw tr("Not supported"); - if(_result === InputStartResult.EBUSY) - throw tr("Device busy"); - if(_result === InputStartResult.EUNKNOWN) - throw tr("an error occurred"); - throw _result; - } - this._media_stream = _result; - - this._source_node = this._context.createMediaStreamSource(this._media_stream); - this._source_node.connect(this._analyser_node); - this._analyser_node.connect(this._gain_node); - this._gain_node.connect(this._context.destination); - - JavascriptLevelmeter._instances.push(this); - if(JavascriptLevelmeter._instances.length == 1) { - clearInterval(JavascriptLevelmeter._update_task); - JavascriptLevelmeter._update_task = setInterval(() => JavascriptLevelmeter._analyse_all(), filter.JThresholdFilter.update_task_interval) as any; - } - } - - destory() { - JavascriptLevelmeter._instances.remove(this); - if(JavascriptLevelmeter._instances.length == 0) { - clearInterval(JavascriptLevelmeter._update_task); - JavascriptLevelmeter._update_task = 0; - } - - if(this._source_node) { - this._source_node.disconnect(); - this._source_node = undefined; - } - if(this._media_stream) { - if(this._media_stream.stop) - this._media_stream.stop(); - else - this._media_stream.getTracks().forEach(value => { - value.stop(); - }); - this._media_stream = undefined; - } - if(this._gain_node) { - this._gain_node.disconnect(); - this._gain_node = undefined; - } - if(this._analyser_node) { - this._analyser_node.disconnect(); - this._analyser_node = undefined; - } - } - - device(): InputDevice { - return this._device; - } - - set_observer(callback: (value: number) => any) { - this._callback = callback; - } - - private static _analyse_all() { - for(const instance of [...this._instances]) - instance._analyse(); - } - - private _analyse() { - this._analyser_node.getByteTimeDomainData(this._analyse_buffer); - - this._current_level = filter.JThresholdFilter.process(this._analyse_buffer, this._analyser_node.fftSize, this._current_level, .75); - if(this._callback) - this._callback(this._current_level); - } -} \ No newline at end of file diff --git a/web/app/connection/ServerConnection.ts b/web/app/connection/ServerConnection.ts index 8cc4912c..1993b83c 100644 --- a/web/app/connection/ServerConnection.ts +++ b/web/app/connection/ServerConnection.ts @@ -19,7 +19,6 @@ import {EventType} from "tc-shared/ui/frames/log/Definitions"; import {WrappedWebSocket} from "tc-backend/web/connection/WrappedWebSocket"; import {AbstractVoiceConnection} from "tc-shared/connection/VoiceConnection"; import {DummyVoiceConnection} from "tc-shared/connection/DummyVoiceConnection"; -import {ServerConnectionFactory, setServerConnectionFactory} from "tc-shared/connection/ConnectionFactory"; class ReturnListener { resolve: (value?: T | PromiseLike) => void; diff --git a/web/app/hooks/AudioRecorder.ts b/web/app/hooks/AudioRecorder.ts new file mode 100644 index 00000000..592d56c1 --- /dev/null +++ b/web/app/hooks/AudioRecorder.ts @@ -0,0 +1,4 @@ +import {setRecorderBackend} from "tc-shared/audio/recorder"; +import {WebAudioRecorder} from "../audio/Recorder"; + +setRecorderBackend(new WebAudioRecorder()); \ No newline at end of file diff --git a/web/app/factories/ExternalModal.ts b/web/app/hooks/ExternalModal.ts similarity index 100% rename from web/app/factories/ExternalModal.ts rename to web/app/hooks/ExternalModal.ts diff --git a/web/app/factories/ServerConnection.ts b/web/app/hooks/ServerConnection.ts similarity index 100% rename from web/app/factories/ServerConnection.ts rename to web/app/hooks/ServerConnection.ts diff --git a/web/app/index.ts b/web/app/index.ts index 6d5bf472..4fbad1bb 100644 --- a/web/app/index.ts +++ b/web/app/index.ts @@ -2,7 +2,8 @@ import "webrtc-adapter"; import "./index.scss"; import "./FileTransfer"; -import "./factories/ServerConnection"; -import "./factories/ExternalModal"; +import "./hooks/ServerConnection"; +import "./hooks/ExternalModal"; +import "./hooks/AudioRecorder"; export = require("tc-shared/main"); \ No newline at end of file From 400b4f42933b89483dae6eca7a1796806ccb00b7 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 19:33:57 +0200 Subject: [PATCH 2/9] Improved the audio API especially for the web client --- shared/js/ConnectionHandler.ts | 326 ++++----- shared/js/audio/recorder.ts | 2 +- shared/js/connection/CommandHandler.ts | 32 +- shared/js/connection/HandshakeHandler.ts | 4 +- shared/js/main.tsx | 19 +- shared/js/ui/elements/Modal.ts | 2 +- shared/js/ui/frames/connection_handlers.ts | 3 - shared/js/ui/frames/control-bar/index.tsx | 11 +- shared/js/ui/modal/ModalSettings.tsx | 5 - shared/js/ui/view.tsx | 12 +- shared/js/voice/Filter.ts | 29 +- shared/js/voice/RecorderBase.ts | 48 +- shared/js/voice/RecorderProfile.ts | 117 ++-- shared/svg-sprites/client-icons.d.ts | 5 +- web/app/audio/Recorder.ts | 644 +++++++----------- web/app/audio/RecorderDeviceList.ts | 190 ++++++ web/app/audio/RecorderFilter.ts | 220 +++--- web/app/connection/WrappedWebSocket.ts | 2 +- web/app/voice/VoiceHandler.ts | 29 +- .../voice/bridge/NativeWebRTCVoiceBridge.ts | 6 +- 20 files changed, 906 insertions(+), 800 deletions(-) create mode 100644 web/app/audio/RecorderDeviceList.ts diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts index a307cf25..dfc97e10 100644 --- a/shared/js/ConnectionHandler.ts +++ b/shared/js/ConnectionHandler.ts @@ -5,10 +5,10 @@ import {GroupManager} from "tc-shared/permission/GroupManager"; import {ServerSettings, Settings, settings, StaticSettings} from "tc-shared/settings"; import {Sound, SoundManager} from "tc-shared/sound/Sounds"; import {LocalClientEntry} from "tc-shared/ui/client"; -import {ConnectionProfile, default_profile, find_profile} from "tc-shared/profiles/ConnectionProfile"; +import {ConnectionProfile} from "tc-shared/profiles/ConnectionProfile"; import {ServerAddress} from "tc-shared/ui/server"; import * as log from "tc-shared/log"; -import {LogCategory} from "tc-shared/log"; +import {LogCategory, logError} from "tc-shared/log"; import {createErrorModal, createInfoModal, createInputModal, Modal} from "tc-shared/ui/elements/Modal"; import {hashPassword} from "tc-shared/utils/helpers"; import {HandshakeHandler} from "tc-shared/connection/HandshakeHandler"; @@ -16,8 +16,7 @@ import * as htmltags from "./ui/htmltags"; import {ChannelEntry} from "tc-shared/ui/channel"; import {InputStartResult, InputState} from "tc-shared/voice/RecorderBase"; import {CommandResult} from "tc-shared/connection/ServerConnectionDeclaration"; -import * as bipc from "./ipc/BrowserIPC"; -import {RecorderProfile} from "tc-shared/voice/RecorderProfile"; +import {default_recorder, RecorderProfile} from "tc-shared/voice/RecorderProfile"; import {Frame} from "tc-shared/ui/frames/chat_frame"; import {Hostbanner} from "tc-shared/ui/frames/hostbanner"; import {server_connections} from "tc-shared/ui/frames/connection_handlers"; @@ -38,7 +37,12 @@ import {PluginCmdRegistry} from "tc-shared/connection/PluginCmdHandler"; import {W2GPluginCmdHandler} from "tc-shared/video-viewer/W2GPlugin"; import {VoiceConnectionStatus} from "tc-shared/connection/VoiceConnection"; import {getServerConnectionFactory} from "tc-shared/connection/ConnectionFactory"; -import {getRecorderBackend} from "tc-shared/audio/recorder"; + +export enum InputHardwareState { + MISSING, + START_FAILED, + VALID +} export enum DisconnectReason { HANDLER_DESTROYED, @@ -102,7 +106,6 @@ export enum ViewReasonId { } export interface LocalClientStatus { - input_hardware: boolean; input_muted: boolean; output_muted: boolean; @@ -129,7 +132,6 @@ export interface ConnectParameters { auto_reconnect_attempt?: boolean; } -declare const native_client; export class ConnectionHandler { readonly handlerId: string; @@ -164,8 +166,8 @@ export class ConnectionHandler { private pluginCmdRegistry: PluginCmdRegistry; private client_status: LocalClientStatus = { - input_hardware: false, input_muted: false, + output_muted: false, away: false, channel_subscribe_all: true, @@ -177,7 +179,8 @@ export class ConnectionHandler { channel_codec_decoding_supported: undefined }; - invoke_resized_on_activate: boolean = false; + private inputHardwareState: InputHardwareState = InputHardwareState.MISSING; + log: ServerEventLog; constructor() { @@ -190,7 +193,10 @@ export class ConnectionHandler { this.serverConnection = getServerConnectionFactory().create(this); this.serverConnection.events.on("notify_connection_state_changed", event => this.on_connection_state_changed(event.oldState, event.newState)); - this.serverConnection.getVoiceConnection().events.on("notify_recorder_changed", () => this.update_voice_status()); + this.serverConnection.getVoiceConnection().events.on("notify_recorder_changed", () => { + this.setInputHardwareState(this.getVoiceRecorder() ? InputHardwareState.VALID : InputHardwareState.MISSING); + this.update_voice_status(); + }); this.serverConnection.getVoiceConnection().events.on("notify_connection_status_changed", () => this.update_voice_status()); this.channelTree = new ChannelTree(this); @@ -252,11 +258,10 @@ export class ConnectionHandler { } async startConnection(addr: string, profile: ConnectionProfile, user_action: boolean, parameters: ConnectParameters) { - this.tab_set_name(tr("Connecting")); this.cancel_reconnect(false); this._reconnect_attempt = parameters.auto_reconnect_attempt || false; - if(this.serverConnection) - this.handleDisconnect(DisconnectReason.REQUESTED); + this.handleDisconnect(DisconnectReason.REQUESTED); + this.tab_set_name(tr("Connecting")); let server_address: ServerAddress = { host: "", @@ -345,7 +350,7 @@ export class ConnectionHandler { this.cancel_reconnect(true); if(!this.connected) return; - this.handleDisconnect(DisconnectReason.REQUESTED); //TODO message? + this.handleDisconnect(DisconnectReason.REQUESTED); try { await this.serverConnection.disconnect(); } catch (error) { @@ -370,42 +375,44 @@ export class ConnectionHandler { @EventHandler("notify_connection_state_changed") - private handleConnectionConnected(event: ConnectionEvents["notify_connection_state_changed"]) { - if(event.new_state !== ConnectionState.CONNECTED) return; + private handleConnectionStateChanged(event: ConnectionEvents["notify_connection_state_changed"]) { this.connection_state = event.new_state; + if(event.new_state === ConnectionState.CONNECTED) { + log.info(LogCategory.CLIENT, tr("Client connected")); + this.log.log(EventType.CONNECTION_CONNECTED, { + serverAddress: { + server_port: this.channelTree.server.remote_address.port, + server_hostname: this.channelTree.server.remote_address.host + }, + serverName: this.channelTree.server.properties.virtualserver_name, + own_client: this.getClient().log_data() + }); + this.sound.play(Sound.CONNECTION_CONNECTED); - log.info(LogCategory.CLIENT, tr("Client connected")); - this.log.log(EventType.CONNECTION_CONNECTED, { - serverAddress: { - server_port: this.channelTree.server.remote_address.port, - server_hostname: this.channelTree.server.remote_address.host - }, - serverName: this.channelTree.server.properties.virtualserver_name, - own_client: this.getClient().log_data() - }); - this.sound.play(Sound.CONNECTION_CONNECTED); + this.permissions.requestPermissionList(); + if(this.groups.serverGroups.length == 0) + this.groups.requestGroups(); - this.permissions.requestPermissionList(); - if(this.groups.serverGroups.length == 0) - this.groups.requestGroups(); + this.settings.setServer(this.channelTree.server.properties.virtualserver_unique_identifier); - this.settings.setServer(this.channelTree.server.properties.virtualserver_unique_identifier); + /* apply the server settings */ + if(this.client_status.channel_subscribe_all) + this.channelTree.subscribe_all_channels(); + else + this.channelTree.unsubscribe_all_channels(); + this.channelTree.toggle_server_queries(this.client_status.queries_visible); - /* apply the server settings */ - if(this.client_status.channel_subscribe_all) - this.channelTree.subscribe_all_channels(); - else - this.channelTree.unsubscribe_all_channels(); - this.channelTree.toggle_server_queries(this.client_status.queries_visible); - - this.sync_status_with_server(); - this.channelTree.server.updateProperties(); - /* - No need to update the voice stuff because as soon we see ourself we're doing it - this.update_voice_status(); - if(control_bar.current_connection_handler() === this) - control_bar.apply_server_voice_state(); - */ + this.sync_status_with_server(); + this.channelTree.server.updateProperties(); + /* + No need to update the voice stuff because as soon we see ourself we're doing it + this.update_voice_status(); + if(control_bar.current_connection_handler() === this) + control_bar.apply_server_voice_state(); + */ + } else { + this.setInputHardwareState(this.getVoiceRecorder() ? InputHardwareState.VALID : InputHardwareState.MISSING); + } } get connected() : boolean { @@ -440,52 +447,7 @@ export class ConnectionHandler { if(pathname.endsWith(".php")) pathname = pathname.substring(0, pathname.lastIndexOf("/")); - /* certaccept is currently not working! */ - if(bipc.supported() && false) { - tag.attr('href', "#"); - let popup: Window; - tag.on('click', event => { - const features = { - status: "no", - location: "no", - toolbar: "no", - menubar: "no", - width: 600, - height: 400 - }; - - if(popup) - popup.close(); - - properties["certificate_callback"] = bipc.getInstance().register_certificate_accept_callback(() => { - log.info(LogCategory.GENERAL, tr("Received notification that the certificate has been accepted! Attempting reconnect!")); - if(this._certificate_modal) - this._certificate_modal.close(); - - popup.close(); /* no need, but nicer */ - - const profile = find_profile(properties.connect_profile) || default_profile(); - const cprops = this.reconnect_properties(profile); - this.startConnection(properties.connect_address, profile, true, cprops); - }); - - const url = build_url(document.location.origin + pathname + "/popup/certaccept/", "", properties); - const features_string = Object.keys(features).map(e => e + "=" + features[e]).join(","); - popup = window.open(url, "TeaWeb certificate accept", features_string); - try { - popup.focus(); - } catch(e) { - log.warn(LogCategory.GENERAL, tr("Certificate accept popup has been blocked. Trying a blank page and replacing href")); - - window.open(url, "TeaWeb certificate accept"); /* trying without features */ - tag.attr("target", "_blank"); - tag.attr("href", url); - tag.unbind('click'); - } - }); - } else { - tag.attr('href', build_url(document.location.origin + pathname, document.location.search, properties)); - } + tag.attr('href', build_url(document.location.origin + pathname, document.location.search, properties)); return tag; } @@ -527,7 +489,7 @@ export class ConnectionHandler { else log.error(LogCategory.CLIENT, tr("Could not connect to remote host!"), data); - if(native_client || !dns.resolve_address_ipv4) { + if(__build.target === "client" || !dns.resolve_address_ipv4) { createErrorModal( tr("Could not connect"), tr("Could not connect to remote host (Connection refused)") @@ -727,43 +689,47 @@ export class ConnectionHandler { }); } - private _last_record_error_popup: number; + private _last_record_error_popup: number = 0; update_voice_status(targetChannel?: ChannelEntry) { - //TODO: Simplify this - if(!this._local_client) return; /* we've been destroyed */ + if(!this._local_client) { + /* we've been destroyed */ + return; + } - targetChannel = targetChannel || this.getClient().currentChannel(); + if(typeof targetChannel === "undefined") + targetChannel = this.getClient().currentChannel(); const vconnection = this.serverConnection.getVoiceConnection(); - const basic_voice_support = vconnection.getConnectionState() === VoiceConnectionStatus.Connected && targetChannel; - const support_record = basic_voice_support && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); - const support_playback = basic_voice_support && (!targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec)); - const hasInputDevice = getRecorderBackend().getDeviceList().getPermissionState() === "granted" && !!vconnection.voice_recorder(); + const codecEncodeSupported = !targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec); + const codecDecodeSupported = !targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec); const property_update = { client_input_muted: this.client_status.input_muted, client_output_muted: this.client_status.output_muted }; - if(support_record && basic_voice_support) + /* update the encoding codec */ + if(codecEncodeSupported && targetChannel) { vconnection.set_encoder_codec(targetChannel.properties.channel_codec); + } if(!this.serverConnection.connected() || vconnection.getConnectionState() !== VoiceConnectionStatus.Connected) { property_update["client_input_hardware"] = false; property_update["client_output_hardware"] = false; - this.client_status.input_hardware = hasInputDevice; - - /* no icons are shown so no update at all */ } else { - const recording_supported = hasInputDevice && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); - const playback_supported = !targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec); + const recording_supported = + this.getInputHardwareState() === InputHardwareState.VALID && + (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)) && + vconnection.getConnectionState() === VoiceConnectionStatus.Connected; + + const playback_supported = this.hasOutputHardware() && (!targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec)); property_update["client_input_hardware"] = recording_supported; property_update["client_output_hardware"] = playback_supported; - this.client_status.input_hardware = recording_supported; + } - /* update icons */ + { const client_properties = this.getClient().properties; for(const key of Object.keys(property_update)) { if(client_properties[key] === property_update[key]) @@ -773,7 +739,7 @@ export class ConnectionHandler { if(Object.keys(property_update).length > 0) { this.serverConnection.send_command("clientupdate", property_update).catch(error => { log.warn(LogCategory.GENERAL, tr("Failed to update client audio hardware properties. Error: %o"), error); - this.log.log(EventType.ERROR_CUSTOM, {message: tr("Failed to update audio hardware properties.")}); + this.log.log(EventType.ERROR_CUSTOM, { message: tr("Failed to update audio hardware properties.") }); /* Update these properties anyways (for case the server fails to handle the command) */ const updates = []; @@ -784,50 +750,39 @@ export class ConnectionHandler { } } - - if(targetChannel && basic_voice_support) { - const encoding_supported = vconnection && vconnection.encoding_supported(targetChannel.properties.channel_codec); - const decoding_supported = vconnection && vconnection.decoding_supported(targetChannel.properties.channel_codec); - - if(this.client_status.channel_codec_decoding_supported !== decoding_supported || this.client_status.channel_codec_encoding_supported !== encoding_supported) { - this.client_status.channel_codec_decoding_supported = decoding_supported; - this.client_status.channel_codec_encoding_supported = encoding_supported; + if(targetChannel) { + if(this.client_status.channel_codec_decoding_supported !== codecDecodeSupported || this.client_status.channel_codec_encoding_supported !== codecEncodeSupported) { + this.client_status.channel_codec_decoding_supported = codecDecodeSupported; + this.client_status.channel_codec_encoding_supported = codecEncodeSupported; let message; - if(!encoding_supported && !decoding_supported) + if(!codecEncodeSupported && !codecDecodeSupported) { message = tr("This channel has an unsupported codec.
You cant speak or listen to anybody within this channel!"); - else if(!encoding_supported) + } else if(!codecEncodeSupported) { message = tr("This channel has an unsupported codec.
You cant speak within this channel!"); - else if(!decoding_supported) - message = tr("This channel has an unsupported codec.
You listen to anybody within this channel!"); /* implies speaking does not work as well */ - if(message) + } else if(!codecDecodeSupported) { + message = tr("This channel has an unsupported codec.
You cant listen to anybody within this channel!"); + } + + if(message) { createErrorModal(tr("Channel codec unsupported"), message).open(); + } } } this.client_status = this.client_status || {} as any; - this.client_status.sound_record_supported = support_record; - this.client_status.sound_playback_supported = support_playback; + this.client_status.sound_record_supported = codecEncodeSupported; + this.client_status.sound_playback_supported = codecDecodeSupported; - if(vconnection && vconnection.voice_recorder()) { - const active = !this.client_status.input_muted && !this.client_status.output_muted; + { + const enableRecording = !this.client_status.input_muted && !this.client_status.output_muted; /* No need to start the microphone when we're not even connected */ - const input = vconnection.voice_recorder().input; + const input = vconnection.voice_recorder()?.input; if(input) { - if(active && this.serverConnection.connected()) { - if(input.current_state() === InputState.PAUSED) { - input.start().then(result => { - if(result != InputStartResult.EOK) - throw result; - }).catch(error => { - log.warn(LogCategory.VOICE, tr("Failed to start microphone input (%s)."), error); - if(Date.now() - (this._last_record_error_popup || 0) > 10 * 1000) { - this._last_record_error_popup = Date.now(); - createErrorModal(tr("Failed to start recording"), formatMessage(tr("Microphone start failed.{:br:}Error: {}"), error)).open(); - } - }); - } + if(enableRecording && this.serverConnection.connected()) { + if(this.getInputHardwareState() !== InputHardwareState.START_FAILED) + this.startVoiceRecorder(Date.now() - this._last_record_error_popup > 10 * 1000); } else { input.stop(); } @@ -838,6 +793,7 @@ export class ConnectionHandler { this.event_registry.fire("notify_state_updated", { state: "microphone" }); + this.event_registry.fire("notify_state_updated", { state: "speaker" }); @@ -851,7 +807,7 @@ export class ConnectionHandler { client_output_muted: this.client_status.output_muted, client_away: typeof(this.client_status.away) === "string" || this.client_status.away, client_away_message: typeof(this.client_status.away) === "string" ? this.client_status.away : "", - client_input_hardware: this.client_status.sound_record_supported && this.client_status.input_hardware, + client_input_hardware: this.client_status.sound_record_supported && this.getInputHardwareState() === InputHardwareState.VALID, client_output_hardware: this.client_status.sound_playback_supported }).catch(error => { log.warn(LogCategory.GENERAL, tr("Failed to sync handler state with server. Error: %o"), error); @@ -859,15 +815,67 @@ export class ConnectionHandler { }); } - resize_elements() { - this.invoke_resized_on_activate = false; + /* can be called as much as you want, does nothing if nothing changed */ + async acquireInputHardware() { + /* if we're having multiple recorders, try to get the right one */ + let recorder: RecorderProfile = default_recorder; + + try { + await this.serverConnection.getVoiceConnection().acquire_voice_recorder(recorder); + } catch (error) { + logError(LogCategory.AUDIO, tr("Failed to acquire recorder: %o"), error); + createErrorModal(tr("Failed to acquire recorder"), tr("Failed to acquire recorder.\nLookup the console for more details.")).open(); + return; + } + + if(this.connection_state === ConnectionState.CONNECTED) { + await this.startVoiceRecorder(true); + } else { + this.setInputHardwareState(InputHardwareState.VALID); + } } - acquire_recorder(voice_recoder: RecorderProfile, update_control_bar: boolean) { - const vconnection = this.serverConnection.getVoiceConnection(); - vconnection.acquire_voice_recorder(voice_recoder).catch(error => { - log.warn(LogCategory.VOICE, tr("Failed to acquire recorder (%o)"), error); - }); + async startVoiceRecorder(notifyError: boolean) { + const input = this.getVoiceRecorder()?.input; + if(!input) return; + + if(input.currentState() === InputState.PAUSED && this.connection_state === ConnectionState.CONNECTED) { + try { + const result = await input.start(); + if(result !== InputStartResult.EOK) { + throw result; + } + + this.setInputHardwareState(InputHardwareState.VALID); + this.update_voice_status(); + } catch (error) { + this.setInputHardwareState(InputHardwareState.START_FAILED); + + let errorMessage; + if(error === InputStartResult.ENOTSUPPORTED) { + errorMessage = tr("Your browser does not support voice recording"); + } else if(error === InputStartResult.EBUSY) { + errorMessage = tr("The input device is busy"); + } else if(error === InputStartResult.EDEVICEUNKNOWN) { + errorMessage = tr("Invalid input device"); + } else if(error === InputStartResult.ENOTALLOWED) { + errorMessage = tr("No permissions"); + } else if(error instanceof Error) { + errorMessage = error.message; + } else if(typeof error === "string") { + errorMessage = error; + } else { + errorMessage = tr("lookup the console"); + } + log.warn(LogCategory.VOICE, tr("Failed to start microphone input (%s)."), error); + if(notifyError) { + this._last_record_error_popup = Date.now(); + createErrorModal(tr("Failed to start recording"), tra("Microphone start failed.\nError: {}", errorMessage)).open(); + } + } + } else { + this.setInputHardwareState(InputHardwareState.VALID); + } } getVoiceRecorder() : RecorderProfile | undefined { return this.serverConnection.getVoiceConnection().voice_recorder(); } @@ -1017,15 +1025,9 @@ export class ConnectionHandler { this.update_voice_status(); } toggleMicrophone() { this.setMicrophoneMuted(!this.isMicrophoneMuted()); } - isMicrophoneMuted() { return this.client_status.input_muted; } - /* - * Returns whatever the client is able to talk or not. Reasons for returning true could be: - * - Channel codec isn't supported - * - No recorder has been acquired - * - Voice bridge hasn't been set upped yet - */ - isMicrophoneDisabled() { return !this.client_status.input_hardware; } + isMicrophoneMuted() { return this.client_status.input_muted; } + isMicrophoneDisabled() { return this.inputHardwareState !== InputHardwareState.VALID; } setSpeakerMuted(muted: boolean) { if(this.client_status.output_muted === muted) return; @@ -1101,8 +1103,16 @@ export class ConnectionHandler { return this.client_status.queries_visible; } - hasInputHardware() : boolean { return this.client_status.input_hardware; } - hasOutputHardware() : boolean { return this.client_status.output_muted; } + getInputHardwareState() : InputHardwareState { return this.inputHardwareState; } + private setInputHardwareState(state: InputHardwareState) { + if(this.inputHardwareState === state) + return; + + this.inputHardwareState = state; + this.event_registry.fire("notify_state_updated", { state: "microphone" }); + } + + hasOutputHardware() : boolean { return true; } getPluginCmdRegistry() : PluginCmdRegistry { return this.pluginCmdRegistry; } } diff --git a/shared/js/audio/recorder.ts b/shared/js/audio/recorder.ts index c70f8085..cca8c7de 100644 --- a/shared/js/audio/recorder.ts +++ b/shared/js/audio/recorder.ts @@ -106,7 +106,7 @@ export abstract class AbstractDeviceList implements DeviceList { return new Promise(resolve => { const callback = (event: DeviceListEvents["notify_state_changed"]) => { - if(event.newState !== "uninitialized") + if(event.newState === "uninitialized") return; this.events.off("notify_state_changed", callback); diff --git a/shared/js/connection/CommandHandler.ts b/shared/js/connection/CommandHandler.ts index f2899715..17480a71 100644 --- a/shared/js/connection/CommandHandler.ts +++ b/shared/js/connection/CommandHandler.ts @@ -576,7 +576,7 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { let self = client instanceof LocalClientEntry; let channel_to = tree.findChannel(parseInt(json["ctid"])); - let channel_from = tree.findChannel(parseInt(json["cfid"])); + let channelFrom = tree.findChannel(parseInt(json["cfid"])); if(!client) { log.error(LogCategory.NETWORKING, tr("Unknown client move (Client)!")); @@ -589,17 +589,17 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { } if(!self) { - if(!channel_from) { + if(!channelFrom) { log.error(LogCategory.NETWORKING, tr("Unknown client move (Channel from)!")); - channel_from = client.currentChannel(); - } else if(channel_from != client.currentChannel()) { + channelFrom = client.currentChannel(); + } else if(channelFrom != client.currentChannel()) { log.error(LogCategory.NETWORKING, tr("Client move from invalid source channel! Local client registered in channel %d but server send %d."), - client.currentChannel().channelId, channel_from.channelId + client.currentChannel().channelId, channelFrom.channelId ); } } else { - channel_from = client.currentChannel(); + channelFrom = client.currentChannel(); } tree.moveClient(client, channel_to); @@ -607,7 +607,7 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { if(self) { this.connection_handler.update_voice_status(channel_to); - for(const entry of client.channelTree.clientsByChannel(channel_from)) { + for(const entry of client.channelTree.clientsByChannel(channelFrom)) { if(entry !== client && entry.get_audio_handle()) { entry.get_audio_handle().abort_replay(); entry.speaking = false; @@ -616,16 +616,18 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { const side_bar = this.connection_handler.side_bar; side_bar.info_frame().update_channel_talk(); + } else { + client.speaking = false; } const own_channel = this.connection.client.getClient().currentChannel(); - const event = self ? EventType.CLIENT_VIEW_MOVE_OWN : (channel_from == own_channel || channel_to == own_channel ? EventType.CLIENT_VIEW_MOVE_OWN_CHANNEL : EventType.CLIENT_VIEW_MOVE); + const event = self ? EventType.CLIENT_VIEW_MOVE_OWN : (channelFrom == own_channel || channel_to == own_channel ? EventType.CLIENT_VIEW_MOVE_OWN_CHANNEL : EventType.CLIENT_VIEW_MOVE); this.connection_handler.log.log(event, { - channel_from: channel_from ? { - channel_id: channel_from.channelId, - channel_name: channel_from.channelName() + channel_from: channelFrom ? { + channel_id: channelFrom.channelId, + channel_name: channelFrom.channelName() } : undefined, - channel_from_own: channel_from == own_channel, + channel_from_own: channelFrom == own_channel, channel_to: channel_to ? { channel_id: channel_to.channelId, @@ -650,20 +652,20 @@ export class ConnectionCommandHandler extends AbstractCommandHandler { this.connection_handler.sound.play(Sound.USER_MOVED_SELF); else if(own_channel == channel_to) this.connection_handler.sound.play(Sound.USER_ENTERED_MOVED); - else if(own_channel == channel_from) + else if(own_channel == channelFrom) this.connection_handler.sound.play(Sound.USER_LEFT_MOVED); } else if(json["reasonid"] == ViewReasonId.VREASON_USER_ACTION) { if(self) {} //If we do an action we wait for the error response else if(own_channel == channel_to) this.connection_handler.sound.play(Sound.USER_ENTERED); - else if(own_channel == channel_from) + else if(own_channel == channelFrom) this.connection_handler.sound.play(Sound.USER_LEFT); } else if(json["reasonid"] == ViewReasonId.VREASON_CHANNEL_KICK) { if(self) { this.connection_handler.sound.play(Sound.CHANNEL_KICKED); } else if(own_channel == channel_to) this.connection_handler.sound.play(Sound.USER_ENTERED_KICKED); - else if(own_channel == channel_from) + else if(own_channel == channelFrom) this.connection_handler.sound.play(Sound.USER_LEFT_KICKED_CHANNEL); } else { console.warn(tr("Unknown reason id %o"), json["reasonid"]); diff --git a/shared/js/connection/HandshakeHandler.ts b/shared/js/connection/HandshakeHandler.ts index cccdb446..40c72302 100644 --- a/shared/js/connection/HandshakeHandler.ts +++ b/shared/js/connection/HandshakeHandler.ts @@ -100,8 +100,8 @@ export class HandshakeHandler { client_server_password: this.parameters.password ? this.parameters.password.password : undefined, client_browser_engine: navigator.product, - client_input_hardware: this.connection.client.hasInputHardware(), - client_output_hardware: false, + client_input_hardware: this.connection.client.isMicrophoneDisabled(), + client_output_hardware: this.connection.client.hasOutputHardware(), client_input_muted: this.connection.client.isMicrophoneMuted(), client_output_muted: this.connection.client.isSpeakerMuted(), }; diff --git a/shared/js/main.tsx b/shared/js/main.tsx index 86502736..07e467ef 100644 --- a/shared/js/main.tsx +++ b/shared/js/main.tsx @@ -336,27 +336,10 @@ function main() { top_menu.initialize(); const initial_handler = server_connections.spawn_server_connection(); - initial_handler.acquire_recorder(default_recorder, false); + initial_handler.acquireInputHardware().then(() => {}); cmanager.server_connections.set_active_connection(initial_handler); /** Setup the XF forum identity **/ fidentity.update_forum(); - - let _resize_timeout; - $(window).on('resize', event => { - if(event.target !== window) - return; - - if(_resize_timeout) - clearTimeout(_resize_timeout); - _resize_timeout = setTimeout(() => { - for(const connection of server_connections.all_connections()) - connection.invoke_resized_on_activate = true; - const active_connection = server_connections.active_connection(); - if(active_connection) - active_connection.resize_elements(); - $(".window-resize-listener").trigger('resize'); - }, 1000); - }); keycontrol.initialize(); stats.initialize({ diff --git a/shared/js/ui/elements/Modal.ts b/shared/js/ui/elements/Modal.ts index c4cd13ed..138f335d 100644 --- a/shared/js/ui/elements/Modal.ts +++ b/shared/js/ui/elements/Modal.ts @@ -34,7 +34,7 @@ export const ModalFunctions = { case "string": if(type == ElementType.HEADER) return $.spawn("div").addClass("modal-title").text(val); - return $("
" + val + "
"); + return $("
" + val.replace(/\n/g, "
") + "
"); case "object": return val as JQuery; case "undefined": return undefined; diff --git a/shared/js/ui/frames/connection_handlers.ts b/shared/js/ui/frames/connection_handlers.ts index 7dfe8b24..3f957548 100644 --- a/shared/js/ui/frames/connection_handlers.ts +++ b/shared/js/ui/frames/connection_handlers.ts @@ -113,9 +113,6 @@ export class ConnectionManager { this._container_channel_tree.append(handler.channelTree.tag_tree()); this._container_chat.append(handler.side_bar.html_tag()); this._container_log_server.append(handler.log.getHTMLTag()); - - if(handler.invoke_resized_on_activate) - handler.resize_elements(); } const old_handler = this.active_handler; this.active_handler = handler; diff --git a/shared/js/ui/frames/control-bar/index.tsx b/shared/js/ui/frames/control-bar/index.tsx index 3b797b2f..34288fc9 100644 --- a/shared/js/ui/frames/control-bar/index.tsx +++ b/shared/js/ui/frames/control-bar/index.tsx @@ -3,7 +3,12 @@ import {Button} from "./button"; import {DropdownEntry} from "tc-shared/ui/frames/control-bar/dropdown"; import {Translatable} from "tc-shared/ui/react-elements/i18n"; import {ReactComponentBase} from "tc-shared/ui/react-elements/ReactComponentBase"; -import {ConnectionEvents, ConnectionHandler, ConnectionStateUpdateType} from "tc-shared/ConnectionHandler"; +import { + ConnectionEvents, + ConnectionHandler, + ConnectionState as CConnectionState, + ConnectionStateUpdateType +} from "tc-shared/ConnectionHandler"; import {Event, EventHandler, ReactEventHandler, Registry} from "tc-shared/events"; import {ConnectionManagerEvents, server_connections} from "tc-shared/ui/frames/connection_handlers"; import {Settings, settings} from "tc-shared/settings"; @@ -21,6 +26,7 @@ import {createInputModal} from "tc-shared/ui/elements/Modal"; import {default_recorder} from "tc-shared/voice/RecorderProfile"; import {global_client_actions} from "tc-shared/events/GlobalEvents"; import {icon_cache_loader} from "tc-shared/file/Icons"; +import {InputState} from "tc-shared/voice/RecorderBase"; const cssStyle = require("./index.scss"); const cssButtonStyle = require("./button.scss"); @@ -704,8 +710,7 @@ function initialize(event_registry: Registry) { if(current_connection_handler) { current_connection_handler.setMicrophoneMuted(!state); - if(!current_connection_handler.getVoiceRecorder()) - current_connection_handler.acquire_recorder(default_recorder, true); /* acquire_recorder already updates the voice status */ + current_connection_handler.acquireInputHardware().then(() => {}); } }); diff --git a/shared/js/ui/modal/ModalSettings.tsx b/shared/js/ui/modal/ModalSettings.tsx index 1a221047..3d494ca7 100644 --- a/shared/js/ui/modal/ModalSettings.tsx +++ b/shared/js/ui/modal/ModalSettings.tsx @@ -12,8 +12,6 @@ import {LogCategory} from "tc-shared/log"; import * as profiles from "tc-shared/profiles/ConnectionProfile"; import {RepositoryTranslation, TranslationRepository} from "tc-shared/i18n/localize"; import {Registry} from "tc-shared/events"; -import {key_description} from "tc-shared/PPTListener"; -import {default_recorder} from "tc-shared/voice/RecorderProfile"; import {spawnYesNo} from "tc-shared/ui/modal/ModalYesNo"; import * as i18n from "tc-shared/i18n/localize"; import * as i18nc from "tc-shared/i18n/country"; @@ -22,12 +20,9 @@ import * as events from "tc-shared/events"; import * as sound from "tc-shared/sound/Sounds"; import * as forum from "tc-shared/profiles/identities/teaspeak-forum"; import {formatMessage, set_icon_size} from "tc-shared/ui/frames/chat"; -import {spawnKeySelect} from "tc-shared/ui/modal/ModalKeySelect"; import {spawnTeamSpeakIdentityImport, spawnTeamSpeakIdentityImprove} from "tc-shared/ui/modal/ModalIdentity"; import {Device} from "tc-shared/audio/player"; -import {LevelMeter} from "tc-shared/voice/RecorderBase"; import * as aplayer from "tc-backend/audio/player"; -import * as arecorder from "tc-backend/audio/recorder"; import {KeyMapSettings} from "tc-shared/ui/modal/settings/Keymap"; import * as React from "react"; import * as ReactDOM from "react-dom"; diff --git a/shared/js/ui/view.tsx b/shared/js/ui/view.tsx index 993c0713..130662df 100644 --- a/shared/js/ui/view.tsx +++ b/shared/js/ui/view.tsx @@ -542,13 +542,17 @@ export class ChannelTree { client["_channel"] = targetChannel; targetChannel?.registerClient(client); - if(oldChannel) + if(oldChannel) { this.client.side_bar.info_frame().update_channel_client_count(oldChannel); - if(targetChannel) + } + + if(targetChannel) { this.client.side_bar.info_frame().update_channel_client_count(targetChannel); - if(oldChannel && targetChannel) + } + + if(oldChannel && targetChannel) { client.events.fire("notify_client_moved", { oldChannel: oldChannel, newChannel: targetChannel }); - client.speaking = false; + } } finally { flush_batched_updates(BatchUpdateType.CHANNEL_TREE); } diff --git a/shared/js/voice/Filter.ts b/shared/js/voice/Filter.ts index 58ff5345..776e1b9c 100644 --- a/shared/js/voice/Filter.ts +++ b/shared/js/voice/Filter.ts @@ -7,41 +7,42 @@ export enum FilterType { export interface FilterBase { readonly priority: number; - set_enabled(flag: boolean) : void; - is_enabled() : boolean; + setEnabled(flag: boolean) : void; + isEnabled() : boolean; } export interface MarginedFilter { - get_margin_frames() : number; - set_margin_frames(value: number); + getMarginFrames() : number; + setMarginFrames(value: number); } export interface ThresholdFilter extends FilterBase, MarginedFilter { readonly type: FilterType.THRESHOLD; - get_threshold() : number; - set_threshold(value: number) : Promise; + getThreshold() : number; + setThreshold(value: number); - get_attack_smooth() : number; - get_release_smooth() : number; + getAttackSmooth() : number; + getReleaseSmooth() : number; - set_attack_smooth(value: number); - set_release_smooth(value: number); + setAttackSmooth(value: number); + setReleaseSmooth(value: number); - callback_level?: (value: number) => any; + registerLevelCallback(callback: (value: number) => void); + removeLevelCallback(callback: (value: number) => void); } export interface VoiceLevelFilter extends FilterBase, MarginedFilter { type: FilterType.VOICE_LEVEL; - get_level() : number; + getLevel() : number; } export interface StateFilter extends FilterBase { type: FilterType.STATE; - set_state(state: boolean) : Promise; - is_active() : boolean; /* if true the the filter allows data to pass */ + setState(state: boolean); + isActive() : boolean; /* if true the the filter allows data to pass */ } export type FilterTypeClass = diff --git a/shared/js/voice/RecorderBase.ts b/shared/js/voice/RecorderBase.ts index e6d36b29..28554024 100644 --- a/shared/js/voice/RecorderBase.ts +++ b/shared/js/voice/RecorderBase.ts @@ -7,32 +7,43 @@ export enum InputConsumerType { NODE, NATIVE } - -export interface InputConsumer { - type: InputConsumerType; -} - -export interface CallbackInputConsumer extends InputConsumer { +export interface CallbackInputConsumer { + type: InputConsumerType.CALLBACK; callback_audio?: (buffer: AudioBuffer) => any; callback_buffer?: (buffer: Float32Array, samples: number, channels: number) => any; } -export interface NodeInputConsumer extends InputConsumer { +export interface NodeInputConsumer { + type: InputConsumerType.NODE; callback_node: (source_node: AudioNode) => any; callback_disconnect: (source_node: AudioNode) => any; } +export interface NativeInputConsumer { + type: InputConsumerType.NATIVE; +} + +export type InputConsumer = CallbackInputConsumer | NodeInputConsumer | NativeInputConsumer; + export enum InputState { + /* Input recording has been paused */ PAUSED, + + /* + * Recording has been requested, and is currently initializing. + * This state may persist, when the audio context hasn't been initialized yet + */ INITIALIZING, - RECORDING, - DRY + + /* we're currently recording the input */ + RECORDING } export enum InputStartResult { EOK = "eok", EUNKNOWN = "eunknown", + EDEVICEUNKNOWN = "edeviceunknown", EBUSY = "ebusy", ENOTALLOWED = "enotallowed", ENOTSUPPORTED = "enotsupported" @@ -51,17 +62,28 @@ export interface AbstractInput { start() : Promise; stop() : Promise; - currentDevice() : IDevice | undefined; - setDevice(device: IDevice | undefined) : Promise; + /* + * Returns true if the input is currently filtered. + * If the current state isn't recording, than it will return true. + */ + isFiltered() : boolean; + + currentDeviceId() : string | undefined; + + /* + * This method should not throw! + * If the target device is unknown than it should return EDEVICEUNKNOWN on start. + * After changing the device, the input state falls to InputState.PAUSED. + */ + setDeviceId(device: string | undefined) : Promise; currentConsumer() : InputConsumer | undefined; setConsumer(consumer: InputConsumer) : Promise; supportsFilter(type: FilterType) : boolean; createFilter(type: T, priority: number) : FilterTypeClass; - removeFilter(filter: Filter); - resetFilter(); + /* resetFilter(); */ getVolume() : number; setVolume(volume: number); diff --git a/shared/js/voice/RecorderProfile.ts b/shared/js/voice/RecorderProfile.ts index b8984369..c122428d 100644 --- a/shared/js/voice/RecorderProfile.ts +++ b/shared/js/voice/RecorderProfile.ts @@ -7,7 +7,7 @@ import {ConnectionHandler} from "tc-shared/ConnectionHandler"; import * as aplayer from "tc-backend/audio/player"; import * as ppt from "tc-backend/ppt"; import {getRecorderBackend, IDevice} from "tc-shared/audio/recorder"; -import {FilterType, StateFilter} from "tc-shared/voice/Filter"; +import {FilterType, StateFilter, ThresholdFilter} from "tc-shared/voice/Filter"; export type VadType = "threshold" | "push_to_talk" | "active"; export interface RecorderProfileConfig { @@ -38,6 +38,7 @@ export let default_recorder: RecorderProfile; /* needs initialize */ export function set_default_recorder(recorder: RecorderProfile) { default_recorder = recorder; } + export class RecorderProfile { readonly name; readonly volatile; /* not saving profile */ @@ -47,7 +48,8 @@ export class RecorderProfile { current_handler: ConnectionHandler; - callback_input_change: (oldInput: AbstractInput | undefined, newInput: AbstractInput | undefined) => Promise; + /* attention: this callback will only be called when the audio input hasn't been initialized! */ + callback_input_initialized: (input: AbstractInput) => void; callback_start: () => any; callback_stop: () => any; @@ -58,7 +60,11 @@ export class RecorderProfile { private pptHookRegistered: boolean; private registeredFilter = { - "ppt-gate": undefined as StateFilter + "ppt-gate": undefined as StateFilter, + "threshold": undefined as ThresholdFilter, + + /* disable voice transmission by default, e.g. when reinitializing filters etc. */ + "default-disabled": undefined as StateFilter } constructor(name: string, volatile?: boolean) { @@ -71,7 +77,7 @@ export class RecorderProfile { clearTimeout(this.pptTimeout); this.pptTimeout = setTimeout(() => { - this.registeredFilter["ppt-gate"]?.set_state(true); + this.registeredFilter["ppt-gate"]?.setState(true); }, Math.max(this.config.vad_push_to_talk.delay, 0)); }, @@ -79,7 +85,7 @@ export class RecorderProfile { if(this.pptTimeout) clearTimeout(this.pptTimeout); - this.registeredFilter["ppt-gate"]?.set_state(false); + this.registeredFilter["ppt-gate"]?.setState(false); }, cancel: false @@ -120,15 +126,16 @@ export class RecorderProfile { } aplayer.on_ready(async () => { - await getRecorderBackend().getDeviceList().awaitHealthy(); + console.error("AWAITING DEVICE LIST"); + await getRecorderBackend().getDeviceList().awaitInitialized(); + console.error("AWAITING DEVICE LIST DONE"); - this.initialize_input(); - await this.load(); + await this.initializeInput(); await this.reinitializeFilter(); }); } - private initialize_input() { + private async initializeInput() { this.input = getRecorderBackend().createInput(); this.input.events.on("notify_voice_start", () => { @@ -143,28 +150,24 @@ export class RecorderProfile { this.callback_stop(); }); - //TODO: Await etc? - this.callback_input_change && this.callback_input_change(undefined, this.input); - } + this.registeredFilter["default-disabled"] = this.input.createFilter(FilterType.STATE, 20); + await this.registeredFilter["default-disabled"].setState(true); /* filter */ + this.registeredFilter["default-disabled"].setEnabled(true); - private async load() { - this.input.setVolume(this.config.volume / 100); + this.registeredFilter["ppt-gate"] = this.input.createFilter(FilterType.STATE, 100); + this.registeredFilter["ppt-gate"].setEnabled(false); - { - const allDevices = getRecorderBackend().getDeviceList().getDevices(); - const defaultDeviceId = getRecorderBackend().getDeviceList().getDefaultDeviceId(); - console.error("Devices: %o | Searching: %s", allDevices, this.config.device_id); + this.registeredFilter["threshold"] = this.input.createFilter(FilterType.THRESHOLD, 100); + this.registeredFilter["threshold"].setEnabled(false); - const devices = allDevices.filter(e => e.deviceId === defaultDeviceId || e.deviceId === this.config.device_id); - const device = devices.find(e => e.deviceId === this.config.device_id) || devices[0]; - - log.info(LogCategory.VOICE, tr("Loaded record profile device %s | %o (%o)"), this.config.device_id, device, allDevices); - try { - await this.input.setDevice(device); - } catch(error) { - log.error(LogCategory.VOICE, tr("Failed to set input device (%o)"), error); - } + if(this.callback_input_initialized) { + this.callback_input_initialized(this.input); } + + + /* apply initial config values */ + this.input.setVolume(this.config.volume / 100); + await this.input.setDeviceId(this.config.device_id); } private save() { @@ -172,13 +175,33 @@ export class RecorderProfile { settings.changeGlobal(Settings.FN_PROFILE_RECORD(this.name), this.config); } + private reinitializePPTHook() { + if(this.config.vad_type !== "push_to_talk") + return; + + if(this.pptHookRegistered) { + ppt.unregister_key_hook(this.pptHook); + this.pptHookRegistered = false; + } + + for(const key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) + this.pptHook[key] = this.config.vad_push_to_talk[key]; + + ppt.register_key_hook(this.pptHook); + this.pptHookRegistered = true; + + this.registeredFilter["ppt-gate"]?.setState(true); + } + private async reinitializeFilter() { if(!this.input) return; - /* TODO: Really required? If still same input we can just use the registered filters */ + /* don't let any audio pass while we initialize the other filters */ + this.registeredFilter["default-disabled"].setEnabled(true); - this.input.resetFilter(); - delete this.registeredFilter["ppt-gate"]; + /* disable all filter */ + this.registeredFilter["threshold"].setEnabled(false); + this.registeredFilter["ppt-gate"].setEnabled(false); if(this.pptHookRegistered) { ppt.unregister_key_hook(this.pptHook); @@ -186,23 +209,29 @@ export class RecorderProfile { } if(this.config.vad_type === "threshold") { - const filter = this.input.createFilter(FilterType.THRESHOLD, 100); - await filter.set_threshold(this.config.vad_threshold.threshold); + const filter = this.registeredFilter["threshold"]; + filter.setEnabled(true); + filter.setThreshold(this.config.vad_threshold.threshold); - filter.set_margin_frames(10); /* 500ms */ - filter.set_attack_smooth(.25); - filter.set_release_smooth(.9); + filter.setMarginFrames(10); /* 500ms */ + filter.setAttackSmooth(.25); + filter.setReleaseSmooth(.9); } else if(this.config.vad_type === "push_to_talk") { - const filter = this.input.createFilter(FilterType.STATE, 100); - await filter.set_state(true); - this.registeredFilter["ppt-gate"] = filter; + const filter = this.registeredFilter["ppt-gate"]; + filter.setEnabled(true); + filter.setState(true); /* by default set filtered */ for(const key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) this.pptHook[key] = this.config.vad_push_to_talk[key]; ppt.register_key_hook(this.pptHook); this.pptHookRegistered = true; - } else if(this.config.vad_type === "active") {} + } else if(this.config.vad_type === "active") { + /* we don't have to initialize any filters */ + } + + + this.registeredFilter["default-disabled"].setEnabled(false); } async unmount() : Promise { @@ -218,7 +247,7 @@ export class RecorderProfile { } } - this.callback_input_change = undefined; + this.callback_input_initialized = undefined; this.callback_start = undefined; this.callback_stop = undefined; this.callback_unmount = undefined; @@ -229,6 +258,7 @@ export class RecorderProfile { set_vad_type(type: VadType) : boolean { if(this.config.vad_type === type) return true; + if(["push_to_talk", "threshold", "active"].findIndex(e => e === type) == -1) return false; @@ -244,7 +274,7 @@ export class RecorderProfile { return; this.config.vad_threshold.threshold = value; - this.reinitializeFilter(); + this.registeredFilter["threshold"]?.setThreshold(this.config.vad_threshold.threshold); this.save(); } @@ -253,7 +283,7 @@ export class RecorderProfile { for(const _key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) this.config.vad_push_to_talk[_key] = key[_key]; - this.reinitializeFilter(); + this.reinitializePPTHook(); this.save(); } @@ -263,7 +293,6 @@ export class RecorderProfile { return; this.config.vad_push_to_talk.delay = value; - this.reinitializeFilter(); this.save(); } @@ -280,7 +309,7 @@ export class RecorderProfile { return; this.config.volume = volume; - this.input && this.input.setVolume(volume / 100); + this.input?.setVolume(volume / 100); this.save(); } } \ No newline at end of file diff --git a/shared/svg-sprites/client-icons.d.ts b/shared/svg-sprites/client-icons.d.ts index d6d53744..78ef4593 100644 --- a/shared/svg-sprites/client-icons.d.ts +++ b/shared/svg-sprites/client-icons.d.ts @@ -3,9 +3,9 @@ * * This file has been auto generated by the svg-sprite generator. * Sprite source directory: D:\TeaSpeak\web\shared\img\client-icons - * Sprite count: 202 + * Sprite count: 203 */ -export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-microphone_broken" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; +export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture-denied" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-microphone_broken" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; export enum ClientIcon { About = "client-about", @@ -31,6 +31,7 @@ export enum ClientIcon { BookmarkRemove = "client-bookmark_remove", BrokenImage = "client-broken_image", BrowseAddonOnline = "client-browse-addon-online", + CaptureDenied = "client-capture-denied", Capture = "client-capture", ChangeNickname = "client-change_nickname", Changelog = "client-changelog", diff --git a/web/app/audio/Recorder.ts b/web/app/audio/Recorder.ts index a63de9b1..0936f068 100644 --- a/web/app/audio/Recorder.ts +++ b/web/app/audio/Recorder.ts @@ -1,19 +1,10 @@ -import { - AbstractDeviceList, - AudioRecorderBacked, - DeviceList, - DeviceListEvents, - DeviceListState, - IDevice, - PermissionState -} from "tc-shared/audio/recorder"; +import {AudioRecorderBacked, DeviceList, IDevice,} from "tc-shared/audio/recorder"; import {Registry} from "tc-shared/events"; -import * as rbase from "tc-shared/voice/RecorderBase"; import { AbstractInput, - CallbackInputConsumer, InputConsumer, - InputConsumerType, InputEvents, + InputConsumerType, + InputEvents, InputStartResult, InputState, LevelMeter, @@ -23,8 +14,8 @@ import * as log from "tc-shared/log"; import {LogCategory, logWarn} from "tc-shared/log"; import * as aplayer from "./player"; import {JAbstractFilter, JStateFilter, JThresholdFilter} from "./RecorderFilter"; -import * as loader from "tc-loader"; import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; +import {inputDeviceList} from "tc-backend/web/audio/RecorderDeviceList"; declare global { interface MediaStream { @@ -36,24 +27,10 @@ export interface WebIDevice extends IDevice { groupId: string; } -function getUserMediaFunctionPromise() : (constraints: MediaStreamConstraints) => Promise { - if('mediaDevices' in navigator && 'getUserMedia' in navigator.mediaDevices) - return constraints => navigator.mediaDevices.getUserMedia(constraints); - - const _callbacked_function = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; - if(!_callbacked_function) - return undefined; - - return constraints => new Promise((resolve, reject) => _callbacked_function(constraints, resolve, reject)); -} - async function requestMicrophoneMediaStream(constraints: MediaTrackConstraints, updateDeviceList: boolean) : Promise { - const mediaFunction = getUserMediaFunctionPromise(); - if(!mediaFunction) return InputStartResult.ENOTSUPPORTED; - try { log.info(LogCategory.AUDIO, tr("Requesting a microphone stream for device %s in group %s"), constraints.deviceId, constraints.groupId); - const stream = mediaFunction({ audio: constraints }); + const stream = await navigator.mediaDevices.getUserMedia({ audio: constraints }); if(updateDeviceList && inputDeviceList.getStatus() === "no-permissions") { inputDeviceList.refresh().then(() => {}); /* added the then body to avoid a inspection warning... */ @@ -76,155 +53,37 @@ async function requestMicrophoneMediaStream(constraints: MediaTrackConstraints, } } -async function requestMicrophonePermissions() : Promise { - const begin = Date.now(); +/* request permission for devices only one per time! */ +let currentMediaStreamRequest: Promise; +async function requestMediaStream(deviceId: string, groupId: string) : Promise { + /* wait for the current media stream requests to finish */ + while(currentMediaStreamRequest) { + try { + await currentMediaStreamRequest; + } catch(error) { } + } + + const audioConstrains: MediaTrackConstraints = {}; + if(window.detectedBrowser?.name === "firefox") { + /* + * Firefox only allows to open one mic as well deciding whats the input device it. + * It does not respect the deviceId nor the groupId + */ + } else { + audioConstrains.deviceId = deviceId; + audioConstrains.groupId = groupId; + } + + audioConstrains.echoCancellation = true; + audioConstrains.autoGainControl = true; + audioConstrains.noiseSuppression = true; + + const promise = (currentMediaStreamRequest = requestMicrophoneMediaStream(audioConstrains, true)); try { - await getUserMediaFunctionPromise()({ audio: { deviceId: "default" }, video: false }); - return "granted"; - } catch (error) { - const end = Date.now(); - const isSystem = (end - begin) < 250; - log.debug(LogCategory.AUDIO, tr("Microphone device request took %d milliseconds. System answered: %s"), end - begin, isSystem); - return "denied"; - } -} - -let inputDeviceList: WebInputDeviceList; -class WebInputDeviceList extends AbstractDeviceList { - private devices: WebIDevice[]; - - private deviceListQueryPromise: Promise; - - constructor() { - super(); - - this.devices = []; - } - - getDefaultDeviceId(): string { - return "default"; - } - - getDevices(): IDevice[] { - return this.devices; - } - - getEvents(): Registry { - return this.events; - } - - getStatus(): DeviceListState { - return this.listState; - } - - isRefreshAvailable(): boolean { - return true; - } - - refresh(askPermissions?: boolean): Promise { - return this.queryDevices(askPermissions === true); - } - - async requestPermissions(): Promise { - if(this.permissionState !== "unknown") - return this.permissionState; - - let result = await requestMicrophonePermissions(); - if(result === "granted" && this.listState === "no-permissions") { - /* if called within doQueryDevices, queryDevices will just return the promise */ - this.queryDevices(false).then(() => {}); - } - this.setPermissionState(result); - return result; - } - - private queryDevices(askPermissions: boolean) : Promise { - if(this.deviceListQueryPromise) - return this.deviceListQueryPromise; - - this.deviceListQueryPromise = this.doQueryDevices(askPermissions).catch(error => { - log.error(LogCategory.AUDIO, tr("Failed to query microphone devices (%o)"), error); - - if(this.listState !== "healthy") - this.listState = "error"; - }).then(() => { - this.deviceListQueryPromise = undefined; - }); - - return this.deviceListQueryPromise || Promise.resolve(); - } - - private async doQueryDevices(askPermissions: boolean) { - let devices = await navigator.mediaDevices.enumerateDevices(); - let hasPermissions = devices.findIndex(e => e.label !== "") !== -1; - - if(!hasPermissions && askPermissions) { - this.setState("no-permissions"); - - let skipPermissionAsk = false; - if('permissions' in navigator && 'query' in navigator.permissions) { - try { - const result = await navigator.permissions.query({ name: "microphone" }); - if(result.state === "denied") { - this.setPermissionState("denied"); - skipPermissionAsk = true; - } - } catch (error) { - logWarn(LogCategory.GENERAL, tr("Failed to query for microphone permissions: %s"), error); - } - } - - if(skipPermissionAsk) { - /* request permissions */ - hasPermissions = await this.requestPermissions() === "granted"; - if(hasPermissions) { - devices = await navigator.mediaDevices.enumerateDevices(); - } - } - } - if(hasPermissions) { - this.setPermissionState("granted"); - } - - if(window.detectedBrowser?.name === "firefox") { - devices = [{ - label: tr("Default Firefox device"), - groupId: "default", - deviceId: "default", - kind: "audioinput", - - toJSON: undefined - }]; - } - - const inputDevices = devices.filter(e => e.kind === "audioinput"); - - const oldDeviceList = this.devices; - this.devices = []; - - let devicesAdded = 0; - for(const device of inputDevices) { - const oldIndex = oldDeviceList.findIndex(e => e.deviceId === device.deviceId); - if(oldIndex === -1) { - devicesAdded++; - } else { - oldDeviceList.splice(oldIndex, 1); - } - - this.devices.push({ - deviceId: device.deviceId, - driver: "WebAudio", - groupId: device.groupId, - name: device.label - }); - } - - this.events.fire("notify_list_updated", { addedDeviceCount: devicesAdded, removedDeviceCount: oldDeviceList.length }); - if(hasPermissions) { - this.setState("healthy"); - } else { - this.setState("no-permissions"); - } + return await currentMediaStreamRequest; + } finally { + if(currentMediaStreamRequest === promise) + currentMediaStreamRequest = undefined; } } @@ -234,7 +93,7 @@ export class WebAudioRecorder implements AudioRecorderBacked { } async createLevelMeter(device: IDevice): Promise { - const meter = new JavascriptLevelmeter(device as any); + const meter = new JavascriptLevelMeter(device as any); await meter.initialize(); return meter; } @@ -247,245 +106,203 @@ export class WebAudioRecorder implements AudioRecorderBacked { class JavascriptInput implements AbstractInput { public readonly events: Registry; - private _state: InputState = InputState.PAUSED; - private _current_device: WebIDevice | undefined; - private _current_consumer: InputConsumer; + private state: InputState = InputState.PAUSED; + private deviceId: string | undefined; + private consumer: InputConsumer; - private _current_stream: MediaStream; - private _current_audio_stream: MediaStreamAudioSourceNode; + private currentStream: MediaStream; + private currentAudioStream: MediaStreamAudioSourceNode; - private _audio_context: AudioContext; - private _source_node: AudioNode; /* last node which could be connected to the target; target might be the _consumer_node */ - private _consumer_callback_node: ScriptProcessorNode; - private readonly _consumer_audio_callback; - private _volume_node: GainNode; - private _mute_node: GainNode; + private audioContext: AudioContext; + private sourceNode: AudioNode; /* last node which could be connected to the target; target might be the _consumer_node */ + private audioNodeCallbackConsumer: ScriptProcessorNode; + private readonly audioScriptProcessorCallback; + private audioNodeVolume: GainNode; + + /* The node is connected to the audio context. Used for the script processor so it has a sink */ + private audioNodeMute: GainNode; private registeredFilters: (Filter & JAbstractFilter)[] = []; - private _filter_active: boolean = false; + private inputFiltered: boolean = false; - private _volume: number = 1; + private startPromise: Promise; - callback_begin: () => any = undefined; - callback_end: () => any = undefined; + private volumeModifier: number = 1; constructor() { this.events = new Registry(); - aplayer.on_ready(() => this._audio_initialized()); - this._consumer_audio_callback = this._audio_callback.bind(this); + aplayer.on_ready(() => this.handleAudioInitialized()); + this.audioScriptProcessorCallback = this.handleAudio.bind(this); } - private _audio_initialized() { - this._audio_context = aplayer.context(); - if(!this._audio_context) - return; + private handleAudioInitialized() { + this.audioContext = aplayer.context(); + this.audioNodeMute = this.audioContext.createGain(); + this.audioNodeMute.gain.value = 0; + this.audioNodeMute.connect(this.audioContext.destination); - this._mute_node = this._audio_context.createGain(); - this._mute_node.gain.value = 0; - this._mute_node.connect(this._audio_context.destination); + this.audioNodeCallbackConsumer = this.audioContext.createScriptProcessor(1024 * 4); + this.audioNodeCallbackConsumer.connect(this.audioNodeMute); - this._consumer_callback_node = this._audio_context.createScriptProcessor(1024 * 4); - this._consumer_callback_node.connect(this._mute_node); - - this._volume_node = this._audio_context.createGain(); - this._volume_node.gain.value = this._volume; + this.audioNodeVolume = this.audioContext.createGain(); + this.audioNodeVolume.gain.value = this.volumeModifier; this.initializeFilters(); - if(this._state === InputState.INITIALIZING) - this.start(); + if(this.state === InputState.INITIALIZING) { + this.start().catch(error => { + logWarn(LogCategory.AUDIO, tr("Failed to automatically start audio recording: %s"), error); + }); + } } private initializeFilters() { - for(const filter of this.registeredFilters) { - if(filter.is_enabled()) - filter.finalize(); - } - + this.registeredFilters.forEach(e => e.finalize()); this.registeredFilters.sort((a, b) => a.priority - b.priority); - if(this._audio_context && this._volume_node) { - const active_filter = this.registeredFilters.filter(e => e.is_enabled()); - let stream: AudioNode = this._volume_node; - for(const f of active_filter) { - f.initialize(this._audio_context, stream); - stream = f.audio_node; + + if(this.audioContext && this.audioNodeVolume) { + const activeFilters = this.registeredFilters.filter(e => e.isEnabled()); + + let chain = "output <- "; + let currentSource: AudioNode = this.audioNodeVolume; + for(const f of activeFilters) { + f.initialize(this.audioContext, currentSource); + f.setPaused(false); + + currentSource = f.audioNode; + chain += FilterType[f.type] + " <- "; } - this._switch_source_node(stream); + chain += "input"; + console.error("Filter chain: %s", chain); + + this.switchSourceNode(currentSource); } } - private _audio_callback(event: AudioProcessingEvent) { - if(!this._current_consumer || this._current_consumer.type !== InputConsumerType.CALLBACK) + private handleAudio(event: AudioProcessingEvent) { + if(this.consumer?.type !== InputConsumerType.CALLBACK) { return; + } - const callback = this._current_consumer as CallbackInputConsumer; - if(callback.callback_audio) - callback.callback_audio(event.inputBuffer); + if(this.consumer.callback_audio) { + this.consumer.callback_audio(event.inputBuffer); + } - if(callback.callback_buffer) { + if(this.consumer.callback_buffer) { log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!")); } } - current_state() : InputState { return this._state; }; - - private _start_promise: Promise; async start() : Promise { - if(this._start_promise) { + while(this.startPromise) { try { - await this._start_promise; - if(this._state != InputState.PAUSED) - return; - } catch(error) { - log.debug(LogCategory.AUDIO, tr("JavascriptInput:start() Start promise await resulted in an error: %o"), error); - } + await this.startPromise; + } catch {} } - return await (this._start_promise = this._start()); + if(this.state != InputState.PAUSED) + return; + + return await (this.startPromise = this.doStart()); } - /* request permission for devices only one per time! */ - private static _running_request: Promise; - static async request_media_stream(device_id: string, group_id: string) : Promise { - while(this._running_request) { - try { - await this._running_request; - } catch(error) { } - } - - const audio_constrains: MediaTrackConstraints = {}; - if(window.detectedBrowser?.name === "firefox") { - /* - * Firefox only allows to open one mic as well deciding whats the input device it. - * It does not respect the deviceId nor the groupId - */ - } else { - audio_constrains.deviceId = device_id; - audio_constrains.groupId = group_id; - } - - audio_constrains.echoCancellation = true; - audio_constrains.autoGainControl = true; - audio_constrains.noiseSuppression = true; - - const promise = (this._running_request = requestMicrophoneMediaStream(audio_constrains, true)); + private async doStart() : Promise { try { - return await this._running_request; - } finally { - if(this._running_request === promise) - this._running_request = undefined; - } - } - - private async _start() : Promise { - try { - if(this._state != InputState.PAUSED) + if(this.state != InputState.PAUSED) throw tr("recorder already started"); - this._state = InputState.INITIALIZING; - if(!this._current_device) + this.state = InputState.INITIALIZING; + if(!this.deviceId) { throw tr("invalid device"); - - if(!this._audio_context) { - debugger; - throw tr("missing audio context"); } - const _result = await JavascriptInput.request_media_stream(this._current_device.deviceId, this._current_device.groupId); - if(!(_result instanceof MediaStream)) { - this._state = InputState.PAUSED; - return _result; + if(!this.audioContext) { + /* Awaiting the audio context to be initialized */ + return; } - this._current_stream = _result; - for(const f of this.registeredFilters) { - if(f.is_enabled()) { - f.set_pause(false); + const requestResult = await requestMediaStream(this.deviceId, undefined); + if(!(requestResult instanceof MediaStream)) { + this.state = InputState.PAUSED; + return requestResult; + } + this.currentStream = requestResult; + + for(const filter of this.registeredFilters) { + if(filter.isEnabled()) { + filter.setPaused(false); } } - this._consumer_callback_node.addEventListener('audioprocess', this._consumer_audio_callback); + /* TODO: Only add if we're really having a callback consumer */ + this.audioNodeCallbackConsumer.addEventListener('audioprocess', this.audioScriptProcessorCallback); + + this.currentAudioStream = this.audioContext.createMediaStreamSource(this.currentStream); + this.currentAudioStream.connect(this.audioNodeVolume); + + this.state = InputState.RECORDING; + this.recalculateFilterStatus(true); - this._current_audio_stream = this._audio_context.createMediaStreamSource(this._current_stream); - this._current_audio_stream.connect(this._volume_node); - this._state = InputState.RECORDING; return InputStartResult.EOK; } catch(error) { - if(this._state == InputState.INITIALIZING) { - this._state = InputState.PAUSED; + if(this.state == InputState.INITIALIZING) { + this.state = InputState.PAUSED; } + throw error; } finally { - this._start_promise = undefined; + this.startPromise = undefined; } } async stop() { - /* await all starts */ - try { - if(this._start_promise) - await this._start_promise; - } catch(error) {} - - this._state = InputState.PAUSED; - if(this._current_audio_stream) { - this._current_audio_stream.disconnect(); + /* await the start */ + if(this.startPromise) { + try { + await this.startPromise; + } catch {} } - if(this._current_stream) { - if(this._current_stream.stop) { - this._current_stream.stop(); + this.state = InputState.PAUSED; + if(this.currentAudioStream) { + this.currentAudioStream.disconnect(); + } + + if(this.currentStream) { + if(this.currentStream.stop) { + this.currentStream.stop(); } else { - this._current_stream.getTracks().forEach(value => { + this.currentStream.getTracks().forEach(value => { value.stop(); }); } } - this._current_stream = undefined; - this._current_audio_stream = undefined; + this.currentStream = undefined; + this.currentAudioStream = undefined; for(const f of this.registeredFilters) { - if(f.is_enabled()) { - f.set_pause(true); + if(f.isEnabled()) { + f.setPaused(true); } } - if(this._consumer_callback_node) { - this._consumer_callback_node.removeEventListener('audioprocess', this._consumer_audio_callback); + if(this.audioNodeCallbackConsumer) { + this.audioNodeCallbackConsumer.removeEventListener('audioprocess', this.audioScriptProcessorCallback); } return undefined; } - current_device(): IDevice | undefined { - return this._current_device; - } - - async set_device(device: IDevice | undefined) { - if(this._current_device === device) + async setDeviceId(deviceId: string | undefined) { + if(this.deviceId === deviceId) return; - const savedState = this._state; try { await this.stop(); } catch(error) { log.warn(LogCategory.AUDIO, tr("Failed to stop previous record session (%o)"), error); } - this._current_device = device as any; - if(!device) { - this._state = savedState === InputState.PAUSED ? InputState.PAUSED : InputState.DRY; - return; - } - - if(savedState !== InputState.PAUSED) { - try { - await this.start() - } catch(error) { - log.warn(LogCategory.AUDIO, tr("Failed to start new recording stream (%o)"), error); - throw "failed to start record"; - } - } - return; + this.deviceId = deviceId; } @@ -507,10 +324,12 @@ class JavascriptInput implements AbstractInput { throw tr("unknown filter type"); } - filter.callback_active_change = () => this._recalculate_filter_status(); + filter.callback_active_change = () => this.recalculateFilterStatus(false); + filter.callback_enabled_change = () => this.initializeFilters(); + this.registeredFilters.push(filter); this.initializeFilters(); - this._recalculate_filter_status(); + this.recalculateFilterStatus(false); return filter as any; } @@ -532,7 +351,7 @@ class JavascriptInput implements AbstractInput { this.registeredFilters = []; this.initializeFilters(); - this._recalculate_filter_status(); + this.recalculateFilterStatus(false); } removeFilter(filterInstance: Filter) { @@ -544,85 +363,104 @@ class JavascriptInput implements AbstractInput { filter.enabled = false; this.initializeFilters(); - this._recalculate_filter_status(); + this.recalculateFilterStatus(false); } - private _recalculate_filter_status() { - let filtered = this.registeredFilters.filter(e => e.is_enabled()).filter(e => (e as JAbstractFilter).active).length > 0; - if(filtered === this._filter_active) + private recalculateFilterStatus(forceUpdate: boolean) { + let filtered = this.registeredFilters.filter(e => e.isEnabled()).filter(e => e.active).length > 0; + if(filtered === this.inputFiltered && !forceUpdate) return; - this._filter_active = filtered; + this.inputFiltered = filtered; if(filtered) { - if(this.callback_end) - this.callback_end(); + this.events.fire("notify_voice_end"); } else { - if(this.callback_begin) - this.callback_begin(); + this.events.fire("notify_voice_start"); } } - current_consumer(): InputConsumer | undefined { - return this._current_consumer; + isRecording(): boolean { + return !this.inputFiltered; } - async set_consumer(consumer: InputConsumer) { - if(this._current_consumer) { - if(this._current_consumer.type == InputConsumerType.NODE) { - if(this._source_node) - (this._current_consumer as NodeInputConsumer).callback_disconnect(this._source_node) - } else if(this._current_consumer.type === InputConsumerType.CALLBACK) { - if(this._source_node) - this._source_node.disconnect(this._consumer_callback_node); + async setConsumer(consumer: InputConsumer) { + if(this.consumer) { + if(this.consumer.type == InputConsumerType.NODE) { + if(this.sourceNode) + (this.consumer as NodeInputConsumer).callback_disconnect(this.sourceNode) + } else if(this.consumer.type === InputConsumerType.CALLBACK) { + if(this.sourceNode) + this.sourceNode.disconnect(this.audioNodeCallbackConsumer); } } if(consumer) { if(consumer.type == InputConsumerType.CALLBACK) { - if(this._source_node) - this._source_node.connect(this._consumer_callback_node); + if(this.sourceNode) + this.sourceNode.connect(this.audioNodeCallbackConsumer); } else if(consumer.type == InputConsumerType.NODE) { - if(this._source_node) - (consumer as NodeInputConsumer).callback_node(this._source_node); + if(this.sourceNode) + (consumer as NodeInputConsumer).callback_node(this.sourceNode); } else { throw "native callback consumers are not supported!"; } } - this._current_consumer = consumer; + this.consumer = consumer; } - private _switch_source_node(new_node: AudioNode) { - if(this._current_consumer) { - if(this._current_consumer.type == InputConsumerType.NODE) { - const node_consumer = this._current_consumer as NodeInputConsumer; - if(this._source_node) - node_consumer.callback_disconnect(this._source_node); - if(new_node) - node_consumer.callback_node(new_node); - } else if(this._current_consumer.type == InputConsumerType.CALLBACK) { - this._source_node.disconnect(this._consumer_callback_node); - if(new_node) - new_node.connect(this._consumer_callback_node); + private switchSourceNode(newNode: AudioNode) { + if(this.consumer) { + if(this.consumer.type == InputConsumerType.NODE) { + const node_consumer = this.consumer as NodeInputConsumer; + if(this.sourceNode) { + node_consumer.callback_disconnect(this.sourceNode); + } + + if(newNode) { + node_consumer.callback_node(newNode); + } + } else if(this.consumer.type == InputConsumerType.CALLBACK) { + this.sourceNode.disconnect(this.audioNodeCallbackConsumer); + if(newNode) { + newNode.connect(this.audioNodeCallbackConsumer); + } } } - this._source_node = new_node; + + this.sourceNode = newNode; } - get_volume(): number { - return this._volume; + currentConsumer(): InputConsumer | undefined { + return this.consumer; } - set_volume(volume: number) { - if(volume === this._volume) + currentDeviceId(): string | undefined { + return this.deviceId; + } + + currentState(): InputState { + return this.state; + } + + getVolume(): number { + return this.volumeModifier; + } + + setVolume(volume: number) { + if(volume === this.volumeModifier) return; - this._volume = volume; - this._volume_node.gain.value = volume; + this.volumeModifier = volume; + this.audioNodeVolume.gain.value = volume; + } + + isFiltered(): boolean { + return this.state === InputState.RECORDING ? this.inputFiltered : true; } } -class JavascriptLevelmeter implements LevelMeter { - private static _instances: JavascriptLevelmeter[] = []; - private static _update_task: number; +class JavascriptLevelMeter implements LevelMeter { + private static meterInstances: JavascriptLevelMeter[] = []; + private static meterUpdateTask: number; readonly _device: WebIDevice; @@ -671,7 +509,7 @@ class JavascriptLevelmeter implements LevelMeter { this._analyse_buffer = new Uint8Array(this._analyser_node.fftSize); /* starting stream */ - const _result = await JavascriptInput.request_media_stream(this._device.deviceId, this._device.groupId); + const _result = await requestMediaStream(this._device.deviceId, this._device.groupId); if(!(_result instanceof MediaStream)){ if(_result === InputStartResult.ENOTALLOWED) throw tr("No permissions"); @@ -690,18 +528,18 @@ class JavascriptLevelmeter implements LevelMeter { this._analyser_node.connect(this._gain_node); this._gain_node.connect(this._context.destination); - JavascriptLevelmeter._instances.push(this); - if(JavascriptLevelmeter._instances.length == 1) { - clearInterval(JavascriptLevelmeter._update_task); - JavascriptLevelmeter._update_task = setInterval(() => JavascriptLevelmeter._analyse_all(), JThresholdFilter.update_task_interval) as any; + JavascriptLevelMeter.meterInstances.push(this); + if(JavascriptLevelMeter.meterInstances.length == 1) { + clearInterval(JavascriptLevelMeter.meterUpdateTask); + JavascriptLevelMeter.meterUpdateTask = setInterval(() => JavascriptLevelMeter._analyse_all(), JThresholdFilter.update_task_interval) as any; } } destroy() { - JavascriptLevelmeter._instances.remove(this); - if(JavascriptLevelmeter._instances.length == 0) { - clearInterval(JavascriptLevelmeter._update_task); - JavascriptLevelmeter._update_task = 0; + JavascriptLevelMeter.meterInstances.remove(this); + if(JavascriptLevelMeter.meterInstances.length == 0) { + clearInterval(JavascriptLevelMeter.meterUpdateTask); + JavascriptLevelMeter.meterUpdateTask = 0; } if(this._source_node) { @@ -736,31 +574,15 @@ class JavascriptLevelmeter implements LevelMeter { } private static _analyse_all() { - for(const instance of [...this._instances]) + for(const instance of [...this.meterInstances]) instance._analyse(); } private _analyse() { this._analyser_node.getByteTimeDomainData(this._analyse_buffer); - this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser_node.fftSize, this._current_level, .75); + this._current_level = JThresholdFilter.calculateAudioLevel(this._analyse_buffer, this._analyser_node.fftSize, this._current_level, .75); if(this._callback) this._callback(this._current_level); } -} - -loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { - function: async () => { - inputDeviceList = new WebInputDeviceList(); - }, - priority: 80, - name: "initialize media devices" -}); - -loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { - function: async () => { - inputDeviceList.refresh().then(() => {}); - }, - priority: 10, - name: "query media devices" -}); \ No newline at end of file +} \ No newline at end of file diff --git a/web/app/audio/RecorderDeviceList.ts b/web/app/audio/RecorderDeviceList.ts new file mode 100644 index 00000000..3cd41b7a --- /dev/null +++ b/web/app/audio/RecorderDeviceList.ts @@ -0,0 +1,190 @@ +import { + AbstractDeviceList, + DeviceListEvents, + DeviceListState, + IDevice, + PermissionState +} from "tc-shared/audio/recorder"; +import * as log from "tc-shared/log"; +import {LogCategory, logWarn} from "tc-shared/log"; +import {Registry} from "tc-shared/events"; +import {WebIDevice} from "tc-backend/web/audio/Recorder"; +import * as loader from "tc-loader"; + +async function requestMicrophonePermissions() : Promise { + const begin = Date.now(); + try { + await navigator.mediaDevices.getUserMedia({ audio: { deviceId: "default" }, video: false }); + return "granted"; + } catch (error) { + const end = Date.now(); + const isSystem = (end - begin) < 250; + log.debug(LogCategory.AUDIO, tr("Microphone device request took %d milliseconds. System answered: %s"), end - begin, isSystem); + return "denied"; + } +} + +export let inputDeviceList: WebInputDeviceList; +class WebInputDeviceList extends AbstractDeviceList { + private devices: WebIDevice[]; + + private deviceListQueryPromise: Promise; + + constructor() { + super(); + + this.devices = []; + } + + async initialize() { + if('permissions' in navigator && 'query' in navigator.permissions) { + try { + const result = await navigator.permissions.query({ name: "microphone" }); + switch (result.state) { + case "denied": + this.setPermissionState("denied"); + break; + + case "granted": + this.setPermissionState("granted"); + break; + + default: + return "unknown"; + } + } catch (error) { + logWarn(LogCategory.GENERAL, tr("Failed to query for microphone permissions: %s"), error); + } + } + } + + getDefaultDeviceId(): string { + return "default"; + } + + getDevices(): IDevice[] { + return this.devices; + } + + getEvents(): Registry { + return this.events; + } + + getStatus(): DeviceListState { + return this.listState; + } + + isRefreshAvailable(): boolean { + return true; + } + + refresh(askPermissions?: boolean): Promise { + return this.queryDevices(askPermissions === true); + } + + async requestPermissions(): Promise { + if(this.permissionState !== "unknown") + return this.permissionState; + + let result = await requestMicrophonePermissions(); + if(result === "granted" && this.listState === "no-permissions") { + /* if called within doQueryDevices, queryDevices will just return the promise */ + this.queryDevices(false).then(() => {}); + } + this.setPermissionState(result); + return result; + } + + private queryDevices(askPermissions: boolean) : Promise { + if(this.deviceListQueryPromise) { + return this.deviceListQueryPromise; + } + + this.deviceListQueryPromise = this.doQueryDevices(askPermissions).catch(error => { + log.error(LogCategory.AUDIO, tr("Failed to query microphone devices (%o)"), error); + + if(this.listState !== "healthy") { + this.setState("error"); + } + }).then(() => { + this.deviceListQueryPromise = undefined; + }); + + return this.deviceListQueryPromise; + } + + private async doQueryDevices(askPermissions: boolean) { + let devices = await navigator.mediaDevices.enumerateDevices(); + let hasPermissions = devices.findIndex(e => e.label !== "") !== -1; + + if(!hasPermissions && askPermissions) { + this.setState("no-permissions"); + + /* request permissions */ + hasPermissions = await this.requestPermissions() === "granted"; + if(hasPermissions) { + devices = await navigator.mediaDevices.enumerateDevices(); + } + } + if(hasPermissions) { + this.setPermissionState("granted"); + } + + if(window.detectedBrowser?.name === "firefox") { + devices = [{ + label: tr("Default Firefox device"), + groupId: "default", + deviceId: "default", + kind: "audioinput", + + toJSON: undefined + }]; + } + + const inputDevices = devices.filter(e => e.kind === "audioinput"); + + const oldDeviceList = this.devices; + this.devices = []; + + let devicesAdded = 0; + for(const device of inputDevices) { + const oldIndex = oldDeviceList.findIndex(e => e.deviceId === device.deviceId); + if(oldIndex === -1) { + devicesAdded++; + } else { + oldDeviceList.splice(oldIndex, 1); + } + + this.devices.push({ + deviceId: device.deviceId, + driver: "WebAudio", + groupId: device.groupId, + name: device.label + }); + } + + this.events.fire("notify_list_updated", { addedDeviceCount: devicesAdded, removedDeviceCount: oldDeviceList.length }); + if(hasPermissions) { + this.setState("healthy"); + } else { + this.setState("no-permissions"); + } + } +} + +loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { + function: async () => { + inputDeviceList = new WebInputDeviceList(); + await inputDeviceList.initialize(); + }, + priority: 80, + name: "initialize media devices" +}); + +loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { + function: async () => { + inputDeviceList.refresh(false).then(() => {}); + }, + priority: 10, + name: "query media devices" +}); \ No newline at end of file diff --git a/web/app/audio/RecorderFilter.ts b/web/app/audio/RecorderFilter.ts index cde5ffbc..90c33702 100644 --- a/web/app/audio/RecorderFilter.ts +++ b/web/app/audio/RecorderFilter.ts @@ -4,13 +4,15 @@ export abstract class JAbstractFilter { readonly priority: number; source_node: AudioNode; - audio_node: NodeType; + audioNode: NodeType; context: AudioContext; enabled: boolean = false; active: boolean = false; /* if true the filter filters! */ + callback_active_change: (new_state: boolean) => any; + callback_enabled_change: () => any; paused: boolean = true; @@ -18,18 +20,24 @@ export abstract class JAbstractFilter { this.priority = priority; } - abstract initialize(context: AudioContext, source_node: AudioNode); + /* Attention: After initialized, paused is the default state */ + abstract initialize(context: AudioContext, sourceNode: AudioNode); abstract finalize(); /* whatever the input has been paused and we don't expect any input */ - abstract set_pause(flag: boolean); + abstract setPaused(flag: boolean); + abstract isPaused() : boolean; - is_enabled(): boolean { + isEnabled(): boolean { return this.enabled; } - set_enabled(flag: boolean) { + setEnabled(flag: boolean) { this.enabled = flag; + + if(this.callback_enabled_change) { + this.callback_enabled_change(); + } } } @@ -37,99 +45,100 @@ export class JThresholdFilter extends JAbstractFilter implements Thres public static update_task_interval = 20; /* 20ms */ readonly type = FilterType.THRESHOLD; - callback_level?: (value: number) => any; - private _threshold = 50; + private threshold = 50; - private _update_task: any; - private _analyser: AnalyserNode; - private _analyse_buffer: Uint8Array; + private analyzeTask: any; + private audioAnalyserNode: AnalyserNode; + private analyseBuffer: Uint8Array; - private _silence_count = 0; - private _margin_frames = 5; + private silenceCount = 0; + private marginFrames = 5; - private _current_level = 0; - private _smooth_release = 0; - private _smooth_attack = 0; + private currentLevel = 0; + private smoothRelease = 0; + private smoothAttack = 0; + + private levelCallbacks: ((level: number) => void)[] = []; finalize() { - this.set_pause(true); + this.paused = true; + this.shutdownAnalyzer(); if(this.source_node) { - try { this.source_node.disconnect(this._analyser) } catch (error) {} - try { this.source_node.disconnect(this.audio_node) } catch (error) {} + try { this.source_node.disconnect(this.audioAnalyserNode) } catch (error) {} + try { this.source_node.disconnect(this.audioNode) } catch (error) {} } - this._analyser = undefined; + this.audioAnalyserNode = undefined; this.source_node = undefined; - this.audio_node = undefined; + this.audioNode = undefined; this.context = undefined; } initialize(context: AudioContext, source_node: AudioNode) { + this.paused = true; + this.context = context; this.source_node = source_node; - this.audio_node = context.createGain(); - this._analyser = context.createAnalyser(); + this.audioNode = context.createGain(); + this.audioAnalyserNode = context.createAnalyser(); const optimal_ftt_size = Math.ceil((source_node.context || context).sampleRate * (JThresholdFilter.update_task_interval / 1000)); const base2_ftt = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); - this._analyser.fftSize = base2_ftt; + this.audioAnalyserNode.fftSize = base2_ftt; - if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser.fftSize) - this._analyse_buffer = new Uint8Array(this._analyser.fftSize); + if(!this.analyseBuffer || this.analyseBuffer.length < this.audioAnalyserNode.fftSize) + this.analyseBuffer = new Uint8Array(this.audioAnalyserNode.fftSize); this.active = false; - this.audio_node.gain.value = 1; + this.audioNode.gain.value = 0; /* silence by default */ - this.source_node.connect(this.audio_node); - this.source_node.connect(this._analyser); - - /* force update paused state */ - this.set_pause(!(this.paused = !this.paused)); + this.source_node.connect(this.audioNode); + this.source_node.connect(this.audioAnalyserNode); } - get_margin_frames(): number { return this._margin_frames; } - set_margin_frames(value: number) { - this._margin_frames = value; + getMarginFrames(): number { return this.marginFrames; } + setMarginFrames(value: number) { + this.marginFrames = value; } - get_attack_smooth(): number { - return this._smooth_attack; + getAttackSmooth(): number { + return this.smoothAttack; } - get_release_smooth(): number { - return this._smooth_release; + getReleaseSmooth(): number { + return this.smoothRelease; } - set_attack_smooth(value: number) { - this._smooth_attack = value; + setAttackSmooth(value: number) { + this.smoothAttack = value; } - set_release_smooth(value: number) { - this._smooth_release = value; + setReleaseSmooth(value: number) { + this.smoothRelease = value; } - get_threshold(): number { - return this._threshold; + getThreshold(): number { + return this.threshold; } - set_threshold(value: number): Promise { - this._threshold = value; - return Promise.resolve(); + setThreshold(value: number) { + this.threshold = value; + this.updateGainNode(false); } - public static process(buffer: Uint8Array, ftt_size: number, previous: number, smooth: number) { + public static calculateAudioLevel(buffer: Uint8Array, fttSize: number, previous: number, smooth: number) : number { let level; { let total = 0, float, rms; - for(let index = 0; index < ftt_size; index++) { + for(let index = 0; index < fttSize; index++) { float = ( buffer[index++] / 0x7f ) - 1; total += (float * float); } - rms = Math.sqrt(total / ftt_size); + rms = Math.sqrt(total / fttSize); let db = 20 * ( Math.log(rms) / Math.log(10) ); // sanity check @@ -140,38 +149,44 @@ export class JThresholdFilter extends JAbstractFilter implements Thres return previous * smooth + level * (1 - smooth); } - private _analyse() { - this._analyser.getByteTimeDomainData(this._analyse_buffer); + private analyzeAnalyseBuffer() { + if(!this.audioNode || !this.audioAnalyserNode) + return; + + this.audioAnalyserNode.getByteTimeDomainData(this.analyseBuffer); let smooth; - if(this._silence_count == 0) - smooth = this._smooth_release; + if(this.silenceCount == 0) + smooth = this.smoothRelease; else - smooth = this._smooth_attack; + smooth = this.smoothAttack; - this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser.fftSize, this._current_level, smooth); + this.currentLevel = JThresholdFilter.calculateAudioLevel(this.analyseBuffer, this.audioAnalyserNode.fftSize, this.currentLevel, smooth); - this._update_gain_node(); - if(this.callback_level) - this.callback_level(this._current_level); + this.updateGainNode(true); + for(const callback of this.levelCallbacks) + callback(this.currentLevel); } - private _update_gain_node() { + private updateGainNode(increaseSilenceCount: boolean) { let state; - if(this._current_level > this._threshold) { - this._silence_count = 0; + if(this.currentLevel > this.threshold) { + this.silenceCount = 0; state = true; } else { - state = this._silence_count++ < this._margin_frames; + state = this.silenceCount < this.marginFrames; + if(increaseSilenceCount) + this.silenceCount++; } + if(state) { - this.audio_node.gain.value = 1; + this.audioNode.gain.value = 1; if(this.active) { this.active = false; this.callback_active_change(false); } } else { - this.audio_node.gain.value = 0; + this.audioNode.gain.value = 0; if(!this.active) { this.active = true; this.callback_active_change(true); @@ -179,22 +194,42 @@ export class JThresholdFilter extends JAbstractFilter implements Thres } } - set_pause(flag: boolean) { - if(flag === this.paused) return; - this.paused = flag; + isPaused(): boolean { + return this.paused; + } - if(this.paused) { - clearInterval(this._update_task); - this._update_task = undefined; - - if(this.active) { - this.active = false; - this.callback_active_change(false); - } - } else { - if(!this._update_task && this._analyser) - this._update_task = setInterval(() => this._analyse(), JThresholdFilter.update_task_interval); + setPaused(flag: boolean) { + if(flag === this.paused) { + return; } + + this.paused = flag; + this.initializeAnalyzer(); + } + + registerLevelCallback(callback: (value: number) => void) { + this.levelCallbacks.push(callback); + } + + removeLevelCallback(callback: (value: number) => void) { + this.levelCallbacks.remove(callback); + } + + private initializeAnalyzer() { + if(this.analyzeTask) { + return; + } + + /* by default we're consuming the input */ + this.active = true; + this.audioNode.gain.value = 0; + + this.analyzeTask = setInterval(() => this.analyzeAnalyseBuffer(), JThresholdFilter.update_task_interval); + } + + private shutdownAnalyzer() { + clearInterval(this.analyzeTask); + this.analyzeTask = undefined; } } @@ -203,11 +238,11 @@ export class JStateFilter extends JAbstractFilter implements StateFilt finalize() { if(this.source_node) { - try { this.source_node.disconnect(this.audio_node) } catch (error) {} + try { this.source_node.disconnect(this.audioNode) } catch (error) {} } this.source_node = undefined; - this.audio_node = undefined; + this.audioNode = undefined; this.context = undefined; } @@ -215,28 +250,31 @@ export class JStateFilter extends JAbstractFilter implements StateFilt this.context = context; this.source_node = source_node; - this.audio_node = context.createGain(); - this.audio_node.gain.value = this.active ? 0 : 1; + this.audioNode = context.createGain(); + this.audioNode.gain.value = this.active ? 0 : 1; - this.source_node.connect(this.audio_node); + this.source_node.connect(this.audioNode); } - is_active(): boolean { + isActive(): boolean { return this.active; } - set_state(state: boolean): Promise { + setState(state: boolean) { if(this.active === state) - return Promise.resolve(); + return; this.active = state; - if(this.audio_node) - this.audio_node.gain.value = state ? 0 : 1; + if(this.audioNode) + this.audioNode.gain.value = state ? 0 : 1; this.callback_active_change(state); - return Promise.resolve(); } - set_pause(flag: boolean) { + isPaused(): boolean { + return this.paused; + } + + setPaused(flag: boolean) { this.paused = flag; } } \ No newline at end of file diff --git a/web/app/connection/WrappedWebSocket.ts b/web/app/connection/WrappedWebSocket.ts index c6fd6c70..7f052a9b 100644 --- a/web/app/connection/WrappedWebSocket.ts +++ b/web/app/connection/WrappedWebSocket.ts @@ -103,7 +103,7 @@ export class WrappedWebSocket { try { if(this.socket.readyState === WebSocket.OPEN) { - this.socket.close(); + this.socket.close(3000); } else if(this.socket.readyState === WebSocket.CONNECTING) { if(kPreventOpeningWebSocketClosing) { /* to prevent the "WebSocket is closed before the connection is established." warning in the console */ diff --git a/web/app/voice/VoiceHandler.ts b/web/app/voice/VoiceHandler.ts index 693a0ac1..df75b411 100644 --- a/web/app/voice/VoiceHandler.ts +++ b/web/app/voice/VoiceHandler.ts @@ -88,16 +88,15 @@ export class VoiceConnection extends AbstractVoiceConnection { if(this.currentAudioSource === recorder && !enforce) return; - if(recorder) { - await recorder.unmount(); - } - if(this.currentAudioSource) { await this.voiceBridge?.setInput(undefined); this.currentAudioSource.callback_unmount = undefined; await this.currentAudioSource.unmount(); } + /* unmount our target recorder */ + await recorder?.unmount(); + this.handleRecorderStop(); this.currentAudioSource = recorder; @@ -108,18 +107,24 @@ export class VoiceConnection extends AbstractVoiceConnection { recorder.callback_start = this.handleRecorderStart.bind(this); recorder.callback_stop = this.handleRecorderStop.bind(this); - recorder.callback_input_change = async (oldInput, newInput) => { + recorder.callback_input_initialized = async input => { if(!this.voiceBridge) return; - if(this.voiceBridge.getInput() && this.voiceBridge.getInput() !== oldInput) { - logWarn(LogCategory.VOICE, - tr("Having a recorder input change, but our voice bridge still has another input (Having: %o, Expecting: %o)!"), - this.voiceBridge.getInput(), oldInput); - } - - await this.voiceBridge.setInput(newInput); + await this.voiceBridge.setInput(input); }; + + if(recorder.input && this.voiceBridge) { + await this.voiceBridge.setInput(recorder.input); + } + + if(!recorder.input || recorder.input.isFiltered()) { + this.handleRecorderStop(); + } else { + this.handleRecorderStart(); + } + } else { + await this.voiceBridge.setInput(undefined); } this.events.fire("notify_recorder_changed"); diff --git a/web/app/voice/bridge/NativeWebRTCVoiceBridge.ts b/web/app/voice/bridge/NativeWebRTCVoiceBridge.ts index ddb69fd8..2552a29f 100644 --- a/web/app/voice/bridge/NativeWebRTCVoiceBridge.ts +++ b/web/app/voice/bridge/NativeWebRTCVoiceBridge.ts @@ -61,11 +61,12 @@ export class NativeWebRTCVoiceBridge extends WebRTCVoiceBridge { } async setInput(input: AbstractInput | undefined) { + console.error("SET INPUT: %o", input); if (this.currentInput === input) return; if (this.currentInput) { - await this.currentInput.set_consumer(undefined); + await this.currentInput.setConsumer(undefined); this.currentInput = undefined; } @@ -73,7 +74,7 @@ export class NativeWebRTCVoiceBridge extends WebRTCVoiceBridge { if (this.currentInput) { try { - await this.currentInput.set_consumer({ + await this.currentInput.setConsumer({ type: InputConsumerType.NODE, callback_node: node => node.connect(this.localAudioDestinationNode), callback_disconnect: node => node.disconnect(this.localAudioDestinationNode) @@ -91,6 +92,7 @@ export class NativeWebRTCVoiceBridge extends WebRTCVoiceBridge { packet[2] = (this.voicePacketId >> 8) & 0xFF; //HIGHT (voiceID) packet[3] = (this.voicePacketId >> 0) & 0xFF; //LOW (voiceID) packet[4] = codec; //Codec + this.voicePacketId++; } sendStopSignal(codec: number) { From 07c1f6ff15eaea72d5a437cf12559af18cabd1e6 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 19:36:17 +0200 Subject: [PATCH 3/9] Removed some debug code --- shared/js/ConnectionHandler.ts | 6 +++--- shared/js/ui/modal/settings/Microphone.tsx | 3 ++- shared/js/voice/RecorderProfile.ts | 2 -- web/app/audio/Recorder.ts | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts index dfc97e10..64f86f47 100644 --- a/shared/js/ConnectionHandler.ts +++ b/shared/js/ConnectionHandler.ts @@ -244,7 +244,7 @@ export class ConnectionHandler { this.update_voice_status(); this.setSubscribeToAllChannels(source ? source.client_status.channel_subscribe_all : settings.global(Settings.KEY_CLIENT_STATE_SUBSCRIBE_ALL_CHANNELS)); - this.setAway_(source ? source.client_status.away : (settings.global(Settings.KEY_CLIENT_STATE_AWAY) ? settings.global(Settings.KEY_CLIENT_AWAY_MESSAGE) : false), false); + this.doSetAway(source ? source.client_status.away : (settings.global(Settings.KEY_CLIENT_STATE_AWAY) ? settings.global(Settings.KEY_CLIENT_AWAY_MESSAGE) : false), false); this.setQueriesShown(source ? source.client_status.queries_visible : settings.global(Settings.KEY_CLIENT_STATE_QUERY_SHOWN)); } @@ -1061,10 +1061,10 @@ export class ConnectionHandler { isSubscribeToAllChannels() : boolean { return this.client_status.channel_subscribe_all; } setAway(state: boolean | string) { - this.setAway_(state, true); + this.doSetAway(state, true); } - private setAway_(state: boolean | string, play_sound: boolean) { + private doSetAway(state: boolean | string, play_sound: boolean) { if(this.client_status.away === state) return; diff --git a/shared/js/ui/modal/settings/Microphone.tsx b/shared/js/ui/modal/settings/Microphone.tsx index 77bdb194..d6afcc79 100644 --- a/shared/js/ui/modal/settings/Microphone.tsx +++ b/shared/js/ui/modal/settings/Microphone.tsx @@ -310,7 +310,7 @@ export function initialize_audio_microphone_controller(events: Registry { @@ -342,3 +342,4 @@ loader.register_task(Stage.LOADED, { }, priority: -2 }) +*/ \ No newline at end of file diff --git a/shared/js/voice/RecorderProfile.ts b/shared/js/voice/RecorderProfile.ts index c122428d..7120d41e 100644 --- a/shared/js/voice/RecorderProfile.ts +++ b/shared/js/voice/RecorderProfile.ts @@ -126,9 +126,7 @@ export class RecorderProfile { } aplayer.on_ready(async () => { - console.error("AWAITING DEVICE LIST"); await getRecorderBackend().getDeviceList().awaitInitialized(); - console.error("AWAITING DEVICE LIST DONE"); await this.initializeInput(); await this.reinitializeFilter(); diff --git a/web/app/audio/Recorder.ts b/web/app/audio/Recorder.ts index 0936f068..5b91f774 100644 --- a/web/app/audio/Recorder.ts +++ b/web/app/audio/Recorder.ts @@ -11,7 +11,7 @@ import { NodeInputConsumer } from "tc-shared/voice/RecorderBase"; import * as log from "tc-shared/log"; -import {LogCategory, logWarn} from "tc-shared/log"; +import {LogCategory, logDebug, logWarn} from "tc-shared/log"; import * as aplayer from "./player"; import {JAbstractFilter, JStateFilter, JThresholdFilter} from "./RecorderFilter"; import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; @@ -173,7 +173,7 @@ class JavascriptInput implements AbstractInput { chain += FilterType[f.type] + " <- "; } chain += "input"; - console.error("Filter chain: %s", chain); + logDebug(LogCategory.AUDIO, tr("Input filter chain: %s"), chain); this.switchSourceNode(currentSource); } From 14fb1fda540a7895ca5ec4f2d820f0e51d25a123 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 19:41:05 +0200 Subject: [PATCH 4/9] Fixed a minor issue related to the teaclient --- shared/js/ui/react-elements/external-modal/PopoutEntrypoint.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared/js/ui/react-elements/external-modal/PopoutEntrypoint.ts b/shared/js/ui/react-elements/external-modal/PopoutEntrypoint.ts index 67879ecf..aa0a5664 100644 --- a/shared/js/ui/react-elements/external-modal/PopoutEntrypoint.ts +++ b/shared/js/ui/react-elements/external-modal/PopoutEntrypoint.ts @@ -2,8 +2,6 @@ import * as loader from "tc-loader"; import * as ipc from "../../../ipc/BrowserIPC"; import * as i18n from "../../../i18n/localize"; -import "tc-shared/proto"; - import {Stage} from "tc-loader"; import {AbstractModal, ModalRenderer} from "tc-shared/ui/react-elements/ModalDefinitions"; import {Settings, SettingsKey} from "tc-shared/settings"; @@ -26,6 +24,7 @@ loader.register_task(Stage.JAVASCRIPT_INITIALIZING, { name: "setup", priority: 110, function: async () => { + await import("tc-shared/proto"); await i18n.initialize(); ipc.setup(); } From 8e5525b201edccc11431cbe220691edbad94d2a2 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 22:27:03 +0200 Subject: [PATCH 5/9] Added the ability to insert new lines within a translate able message --- shared/js/ui/react-elements/i18n/index.tsx | 27 ++++++++++++++++++---- tools/trgen/ts_generator.ts | 25 +++++++++++++++----- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/shared/js/ui/react-elements/i18n/index.tsx b/shared/js/ui/react-elements/i18n/index.tsx index ab77d642..db51cd91 100644 --- a/shared/js/ui/react-elements/i18n/index.tsx +++ b/shared/js/ui/react-elements/i18n/index.tsx @@ -1,19 +1,38 @@ import * as React from "react"; import {parseMessageWithArguments} from "tc-shared/ui/frames/chat"; -import {cloneElement} from "react"; +import {cloneElement, ReactNode} from "react"; let instances = []; -export class Translatable extends React.Component<{ children: string, __cacheKey?: string, trIgnore?: boolean, enforceTextOnly?: boolean }, { translated: string }> { +export class Translatable extends React.Component<{ + children: string | (string | React.ReactElement)[], + __cacheKey?: string, + trIgnore?: boolean, + enforceTextOnly?: boolean +}, { translated: string }> { + protected renderElementIndex = 0; + constructor(props) { super(props); + let text; + if(Array.isArray(this.props.children)) { + text = (this.props.children as any[]).map(e => typeof e === "string" ? e : "\n").join(""); + } else { + text = this.props.children; + } + this.state = { - translated: /* @tr-ignore */ tr(typeof this.props.children === "string" ? this.props.children : (this.props as any).message) + translated: /* @tr-ignore */ tr(text) } } render() { - return this.state.translated; + return this.state.translated.split("\n").reduce((previousValue, currentValue, currentIndex, array) => { + previousValue.push({currentValue}); + if(currentIndex + 1 !== array.length) + previousValue.push(
); + return previousValue; + }, []); } componentDidMount(): void { diff --git a/tools/trgen/ts_generator.ts b/tools/trgen/ts_generator.ts index 9291222b..ee2d8bf4 100644 --- a/tools/trgen/ts_generator.ts +++ b/tools/trgen/ts_generator.ts @@ -370,16 +370,29 @@ export function replace_processor(config: Configuration, cache: VolatileTransfor throw new Error(source_location(ignoreAttribute) + ": Invalid attribute value of type " + SyntaxKind[ignoreAttribute.expression.kind]); } - if(element.children.length !== 1) - throw new Error(source_location(element) + ": Element has been called with an invalid arguments (" + (element.children.length === 0 ? "too few" : "too many") + ")"); + if(element.children.length < 1) { + throw new Error(source_location(element) + ": Element has been called with an invalid arguments (too few)"); + } - const text = element.children[0] as ts.JsxText; - if(text.kind != SyntaxKind.JsxText) - throw new Error(source_location(element) + ": Element has invalid children. Expected JsxText but got " + SyntaxKind[text.kind]); + let text = element.children.map(element => { + if(element.kind === SyntaxKind.JsxText) { + return element.text; + } else if(element.kind === SyntaxKind.JsxSelfClosingElement) { + if(element.tagName.kind !== SyntaxKind.Identifier) { + throw new Error(source_location(element.tagName) + ": Expected a JsxSelfClosingElement, but received " + SyntaxKind[element.tagName.kind]); + } + + if(element.tagName.escapedText !== "br") { + throw new Error(source_location(element.tagName) + ": Expected a br element, but received " + element.tagName.escapedText); + } + + return "\n"; + } + }).join(""); let { line, character } = source_file.getLineAndCharacterOfPosition(node.getStart()); cache.translations.push({ - message: text.text, + message: text, line: line, character: character, filename: (source_file || {fileName: "unknown"}).fileName, From c0f09e4d54c9d73f5448b7a621be383700c96c91 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 22:40:40 +0200 Subject: [PATCH 6/9] Fixed the microphone selection for the newcomer modal --- shared/css/static/modal-newcomer.scss | 10 +- shared/html/templates/modal/newcomer.html | 4 +- .../{ModalNewcomer.ts => ModalNewcomer.tsx} | 163 +++--------------- shared/js/ui/modal/settings/Heighlight.scss | 115 ++++++++++++ shared/js/ui/modal/settings/Heighlight.tsx | 38 ++++ shared/js/ui/modal/settings/Microphone.scss | 15 ++ shared/js/ui/modal/settings/Microphone.tsx | 11 +- .../ui/modal/settings/MicrophoneRenderer.tsx | 144 ++++++++++++---- shared/svg-sprites/client-icons.d.ts | 5 +- 9 files changed, 316 insertions(+), 189 deletions(-) rename shared/js/ui/modal/{ModalNewcomer.ts => ModalNewcomer.tsx} (64%) create mode 100644 shared/js/ui/modal/settings/Heighlight.scss create mode 100644 shared/js/ui/modal/settings/Heighlight.tsx diff --git a/shared/css/static/modal-newcomer.scss b/shared/css/static/modal-newcomer.scss index 03632a1b..b6da49eb 100644 --- a/shared/css/static/modal-newcomer.scss +++ b/shared/css/static/modal-newcomer.scss @@ -14,7 +14,7 @@ html:root { padding: 0!important; min-width: 20em; - width: 50em; + width: 60em; @include user-select(none); @@ -70,6 +70,8 @@ html:root { @include chat-scrollbar-horizontal(); @include chat-scrollbar-vertical(); + background-color: #19191b; + .body { display: flex; flex-direction: column; @@ -86,7 +88,7 @@ html:root { &.step-welcome, &.step-finish { display: flex; flex-direction: row; - justify-content: stretch; + justify-content: center; .text { align-self: center; @@ -119,7 +121,7 @@ html:root { } /* for step-identity or step-microphone */ - .container-settings-identity-profile, .container-settings-audio-microphone { + .container-settings-identity-profile { padding: .5em; .left .body { @@ -136,8 +138,6 @@ html:root { &.step-identity { } - &.step-microphone { } - &.hidden { display: none; } diff --git a/shared/html/templates/modal/newcomer.html b/shared/html/templates/modal/newcomer.html index 005cd09d..5ae1f1cf 100644 --- a/shared/html/templates/modal/newcomer.html +++ b/shared/html/templates/modal/newcomer.html @@ -37,9 +37,7 @@ {{tr "It is save to exit this guide at any point and directly jump ahead using the client." /}}
-
- {{include tmpl="tmpl_settings-microphone" /}} -
+
{{include tmpl="tmpl_settings-profiles" /}}
diff --git a/shared/js/ui/modal/ModalNewcomer.ts b/shared/js/ui/modal/ModalNewcomer.tsx similarity index 64% rename from shared/js/ui/modal/ModalNewcomer.ts rename to shared/js/ui/modal/ModalNewcomer.tsx index cefea293..4da7189a 100644 --- a/shared/js/ui/modal/ModalNewcomer.ts +++ b/shared/js/ui/modal/ModalNewcomer.tsx @@ -5,6 +5,10 @@ import { modal as emodal } from "tc-shared/events"; import {modal_settings} from "tc-shared/ui/modal/ModalSettings"; import {profiles} from "tc-shared/profiles/ConnectionProfile"; import {spawnYesNo} from "tc-shared/ui/modal/ModalYesNo"; +import {initialize_audio_microphone_controller, MicrophoneSettingsEvents} from "tc-shared/ui/modal/settings/Microphone"; +import {MicrophoneSettings} from "tc-shared/ui/modal/settings/MicrophoneRenderer"; +import * as React from "react"; +import * as ReactDOM from "react-dom"; const next_step: {[key: string]:string} = { "welcome": "microphone", @@ -69,7 +73,7 @@ function initializeBasicFunctionality(tag: JQuery, event_registry: Registry { + button_last_step.on('click', () => { if(last_step[current_step]) event_registry.fire("show_step", { step: last_step[current_step] as any }); else @@ -100,8 +104,8 @@ function initializeBasicFunctionality(tag: JQuery, event_registry: Registry button_next_step.prop("disabled", true)); - event_registry.on("show_step", event => button_last_step.prop("disabled", true)); + event_registry.on("show_step", () => button_next_step.prop("disabled", true)); + event_registry.on("show_step", () => button_last_step.prop("disabled", true)); event_registry.on("step-status", event => button_next_step.prop("disabled", !event.next_button)); event_registry.on("step-status", event => button_last_step.prop("disabled", !event.previous_button)); @@ -292,145 +296,28 @@ function initializeStepIdentity(tag: JQuery, event_registry: Registry, modal: Modal) { - const microphone_events = new Registry(); - //microphone_events.enable_debug("settings-microphone"); - //modal_settings.initialize_audio_microphone_controller(microphone_events); - //modal_settings.initialize_audio_microphone_view(tag, microphone_events); - modal.close_listener.push(() => microphone_events.fire_async("deinitialize")); + let helpStep = 0; - let help_animation_done = false; - const update_step_status = () => event_registry.fire_async("step-status", { next_button: help_animation_done, previous_button: help_animation_done }); - event_registry.on("show_step", e => { - if(e.step !== "microphone") return; + const settingEvents = new Registry(); + settingEvents.on("query_help", () => settingEvents.fire_async("notify_highlight", { field: helpStep <= 2 ? ("hs-" + helpStep) as any : undefined })); + settingEvents.on("action_help_click", () => { + helpStep++; + settingEvents.fire("query_help"); - update_step_status(); + event_registry.fire_async("step-status", { next_button: helpStep > 2, previous_button: helpStep > 2 }) }); - /* the help sequence */ - { - const container = tag.find(".container-settings-audio-microphone"); - const container_help_text = tag.find(".container-help-text"); + initialize_audio_microphone_controller(settingEvents); + ReactDOM.render(, tag[0]); - const container_profile_list = tag.find(".highlight-microphone-list"); - const container_profile_settings = tag.find(".highlight-microphone-settings"); + modal.close_listener.push(() => { + settingEvents.fire("notify_destroy"); + ReactDOM.unmountComponentAtNode(tag[0]); + }); - let is_first_show = true; - event_registry.on("show_step", event => { - if(!is_first_show || event.step !== "microphone") return; - is_first_show = false; - container.addClass("help-shown"); - const text = tr( /* @tr-ignore */ - "Firstly we need to setup a microphone.\n" + - "Let me guide you thru the basic UI elements.\n" + - "\n" + - "To continue click anywhere on the screen." - ); - set_help_text(text); - $("body").one('mousedown', event => show_microphone_list_help()); - }); - - const set_help_text = text => { - container_help_text.empty(); - text.split("\n").forEach(e => container_help_text.append(e == "" ? $.spawn("br") : $.spawn("a").text(e))); - }; - - const show_microphone_list_help = () => { - container.find(".highlighted").removeClass("highlighted"); - container_profile_list.addClass("highlighted"); - - const update_position = () => { - const font_size = parseFloat(getComputedStyle(container_help_text[0]).fontSize); - - const offset = container_profile_list.offset(); - const abs = container.offset(); - - container_help_text.css({ - top: offset.top - abs.top, - left: ((offset.left - abs.left) + container_profile_list.outerWidth() + font_size) + "px", - right: "1em", - bottom: "1em" - }); - }; - update_position(); - container_help_text.off('resize').on('resize', update_position); - - const text = tr( /* @tr-ignore */ - "All your available microphones are listed within this box.\n" + - "\n" + - "The currently selected microphone\n" + - "is marked with a green checkmark. To change the selected microphone\n" + - "just click on the new one.\n" + - "\n" + - "To continue click anywhere on the screen." - ); - set_help_text(text); - $("body").one('mousedown', event => show_microphone_settings_help()); - }; - - const show_microphone_settings_help = () => { - container.find(".highlighted").removeClass("highlighted"); - container_profile_settings.addClass("highlighted"); - - const update_position = () => { - const font_size = parseFloat(getComputedStyle(container_help_text[0]).fontSize); - const container_settings_offset = container_profile_settings.offset(); - const right = container_profile_settings.outerWidth() + font_size * 2; - container_help_text.css({ - top: container_settings_offset.top - container.offset().top, - left: "1em", - right: right + "px", - bottom: "1em" - }); - }; - - container_help_text.empty(); - container_help_text.append($.spawn("div").addClass("help-microphone-settings").append( - $.spawn("a").text(tr("On the right side you'll find all microphone settings.")), - $.spawn("br"), - $.spawn("a").text("TeaSpeak has three voice activity detection types:"), - $.spawn("ol").append( - $.spawn("li").addClass("vad-type").append( - $.spawn("a").addClass("title").text(tr("Push to Talk")), - $.spawn("a").addClass("description").html(tr( /* @tr-ignore */ - "To transmit audio data you'll have to
" + - "press a key. The key could be selected " + - "via the button right to the radio button." - )) - ), - $.spawn("li").addClass("vad-type").append( - $.spawn("a").addClass("title").text(tr("Voice activity detection")), - $.spawn("a").addClass("description").html(tr( /* @tr-ignore */ - "In this mode, TeaSpeak will continuously analyze your microphone input. " + - "If the audio level is grater than a certain threshold, " + - "the audio will be transmitted. " + - "The threshold is changeable via the \"Sensitivity Settings\" slider." - )) - ), - $.spawn("li").addClass("vad-type").append( - $.spawn("a").addClass("title").html(tr("Always active")), - $.spawn("a").addClass("description").text(tr( /* @tr-ignore */ - "Continuously transmit any audio data.\n" - )) - ) - ), - $.spawn("br"), - $.spawn("a").text(tr("Now you're ready to configure your microphone. Just click anywhere on the screen.")) - )); - update_position(); - container_help_text.off('resize').on('resize', update_position); - - $("body").one('mousedown', event => hide_help()); - }; - - const hide_help = () => { - container.find(".highlighted").removeClass("highlighted"); - container.addClass("hide-help"); - setTimeout(() => container.removeClass("help-shown"), 1000); - container_help_text.off('resize'); - - help_animation_done = true; - update_step_status(); - }; - } + event_registry.on("show_step", event => { + if(event.step !== "microphone") return; + event_registry.fire_async("step-status", { next_button: helpStep > 2, previous_button: helpStep > 2 }); + }); } \ No newline at end of file diff --git a/shared/js/ui/modal/settings/Heighlight.scss b/shared/js/ui/modal/settings/Heighlight.scss new file mode 100644 index 00000000..9a3a0ef7 --- /dev/null +++ b/shared/js/ui/modal/settings/Heighlight.scss @@ -0,0 +1,115 @@ +@import "../../../../css/static/mixin.scss"; +@import "../../../../css/static/properties.scss"; + +.container { + $highlight-time: .5s; + $backdrop-color: rgba(0, 0, 0, .9); + + display: flex; + position: relative; + + padding: .5em; + + background-color: inherit; + + .background { + position: absolute; + + top: 0; + left: 0; + right: 0; + bottom: 0; + + display: none; + background-color: $backdrop-color; + border-radius: .15em; + + padding: .5em; + } + + /* + .highlightable { + display: flex; + } + */ + + .helpText { + opacity: 0; + z-index: 20; + + pointer-events: none; + + display: block; + + overflow: auto; + @include chat-scrollbar(); + @include transition($highlight-time ease-in-out); + + a { + display: block; + } + + ol { + margin-top: .5em; + margin-bottom: 0; + } + + li { + margin-bottom: .5em; + + .title { + font-weight: bold; + } + } + + &.shown { + opacity: 1; + pointer-events: initial; + + @include transition($highlight-time ease-in-out); + } + } + + &.shown { + .background { + display: flex; + z-index: 1; + + opacity: 1; + } + + .highlightable { + border-radius: .1em; + position: relative; + z-index: 10; + + background-color: inherit; + + @include transition($highlight-time ease-in-out); + + &::after { + content: ' '; + + z-index: 5; + position: absolute; + + top: 0; + left: 0; + right: 0; + bottom: 0; + + background-color: $backdrop-color; + + @include transition($highlight-time ease-in-out); + } + + &.highlighted { + padding: .5em; + + &::after { + background-color: #00000000; + } + } + } + } +} \ No newline at end of file diff --git a/shared/js/ui/modal/settings/Heighlight.tsx b/shared/js/ui/modal/settings/Heighlight.tsx new file mode 100644 index 00000000..3f7da0e2 --- /dev/null +++ b/shared/js/ui/modal/settings/Heighlight.tsx @@ -0,0 +1,38 @@ +import * as React from "react"; +import {useContext} from "react"; + +const cssStyle = require("./Heighlight.scss"); + +const HighlightContext = React.createContext(undefined); +export const HighlightContainer = (props: { children: React.ReactNode | React.ReactNode[], highlightedId?: string, onClick?: () => void }) => { + return ( + +
+ {props.children} +
+
+ + ); +}; + +export const HighlightRegion = (props: React.HTMLProps & { highlightId: string } ) => { + const wProps = Object.assign({}, props); + delete wProps["highlightId"]; + + const highlightedId = useContext(HighlightContext); + const highlighted = highlightedId === props.highlightId; + + wProps.className = (props.className || "") + " " + cssStyle.highlightable + " " + (highlighted ? cssStyle.highlighted : ""); + return React.createElement("div", wProps); +}; + +export const HighlightText = (props: { highlightId: string, className?: string, children?: React.ReactNode | React.ReactNode[] } ) => { + const highlightedId = useContext(HighlightContext); + const highlighted = highlightedId === props.highlightId; + + return ( +
+ {props.children} +
+ ) +}; \ No newline at end of file diff --git a/shared/js/ui/modal/settings/Microphone.scss b/shared/js/ui/modal/settings/Microphone.scss index d1b4446e..317a7343 100644 --- a/shared/js/ui/modal/settings/Microphone.scss +++ b/shared/js/ui/modal/settings/Microphone.scss @@ -11,6 +11,7 @@ min-width: 43em; min-height: 41em; + background-color: inherit; position: relative; .left, .right { @@ -588,6 +589,20 @@ filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#70407e', endColorstr='#45407e',GradientType=1 ); /* IE6-9 */ } +/* The help overlays */ +.help { + position: absolute; + + top: 0; + left: 0; + right: 0; + bottom: 0; + + &.paddingTop { + padding-top: 3.6em; + } +} + @-moz-keyframes spin { 100% { -moz-transform: rotate(360deg); } } @-webkit-keyframes spin { 100% { -webkit-transform: rotate(360deg); } } @keyframes spin { 100% { -webkit-transform: rotate(360deg); transform:rotate(360deg); } } \ No newline at end of file diff --git a/shared/js/ui/modal/settings/Microphone.tsx b/shared/js/ui/modal/settings/Microphone.tsx index d6afcc79..bf98037d 100644 --- a/shared/js/ui/modal/settings/Microphone.tsx +++ b/shared/js/ui/modal/settings/Microphone.tsx @@ -24,11 +24,12 @@ export type MicrophoneDevice = { export interface MicrophoneSettingsEvents { "query_devices": { refresh_list: boolean }, - + "query_help": {}, "query_setting": { setting: MicrophoneSetting }, + "action_help_click": {}, "action_request_permissions": {}, "action_set_selected_device": { deviceId: string }, "action_set_selected_device_result": { @@ -70,6 +71,10 @@ export interface MicrophoneSettingsEvents { status: Exclude }, + notify_highlight: { + field: "hs-0" | "hs-1" | "hs-2" | undefined + } + notify_destroy: {} } @@ -310,7 +315,6 @@ export function initialize_audio_microphone_controller(events: Registry { @@ -341,5 +345,4 @@ loader.register_task(Stage.LOADED, { }); }, priority: -2 -}) -*/ \ No newline at end of file +}) \ No newline at end of file diff --git a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx index 2ad2fb7c..ce7bd58a 100644 --- a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx +++ b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx @@ -1,7 +1,7 @@ import * as React from "react"; import {Translatable} from "tc-shared/ui/react-elements/i18n"; import {Button} from "tc-shared/ui/react-elements/Button"; -import {modal, Registry} from "tc-shared/events"; +import {Registry} from "tc-shared/events"; import {MicrophoneDevice, MicrophoneSettingsEvents} from "tc-shared/ui/modal/settings/Microphone"; import {useEffect, useRef, useState} from "react"; import {ClientIconRenderer} from "tc-shared/ui/react-elements/Icons"; @@ -16,6 +16,7 @@ import {spawnKeySelect} from "tc-shared/ui/modal/ModalKeySelect"; import {Checkbox} from "tc-shared/ui/react-elements/Checkbox"; import {BoxedInputField} from "tc-shared/ui/react-elements/InputField"; import {IDevice} from "tc-shared/audio/recorder"; +import {HighlightContainer, HighlightRegion, HighlightText} from "./Heighlight"; const cssStyle = require("./Microphone.scss"); @@ -157,7 +158,6 @@ const MicrophoneList = (props: { events: Registry }) = }); const [ selectedDevice, setSelectedDevice ] = useState<{ deviceId: string, mode: "selected" | "selecting" }>(); const [ deviceList, setDeviceList ] = useState([]); - const [ error, setError ] = useState(undefined); props.events.reactUse("notify_devices", event => { setSelectedDevice(undefined); @@ -202,7 +202,7 @@ const MicrophoneList = (props: { events: Registry }) =
Please grant access to your microphone. @@ -506,37 +506,109 @@ const ThresholdSelector = (props: { events: Registry } ) }; -export const MicrophoneSettings = (props: { events: Registry }) => ( -
-
-
- Select your Microphone Device - +const HelpText0 = () => ( + + + Firstly we need to setup a microphone.
+ Let me guide you thru the basic UI elements.
+
+ To continue click anywhere on the screen. + + +); + +const HelpText1 = () => ( + + + All your available microphones are listed within this box.
+
+ The currently selected microphone
+ is marked with a green checkmark. To change the selected microphone
+ just click on the new one.
+
+ To continue click anywhere on the screen. + + +); + +const HelpText2 = () => ( + + + TeaSpeak has three voice activity detection types: + +
    +
  1. + + To transmit audio data you'll have to
    + press a key. The key could be selected via the button right to the radio button +     +
    2. +
  2. + + In this mode, TeaSpeak will continuously analyze your microphone input. + If the audio level is grater than a certain threshold, + the audio will be transmitted. + The threshold is changeable via the "Sensitivity Settings" slider + +
    3. +
  3. + Continuously transmit any audio data. +
    4. +
+ + + Now you're ready to configure your microphone.
+ Just click anywhere on the screen. + + + +); + +export const MicrophoneSettings = (props: { events: Registry }) => { + const [ highlighted, setHighlighted ] = useState(() => { + props.events.fire("query_help"); + return undefined; + }); + + props.events.reactUse("notify_highlight", event => setHighlighted(event.field)); + + return ( + props.events.fire("action_help_click")}> +
+ + + + + + + + + +
+ + +
+ +
+ +
+ +
+
+ +
+
+
- -
-
- -
- - -
- -
- -
- -
-
- -
-
-
-
-) \ No newline at end of file + + ); +} \ No newline at end of file diff --git a/shared/svg-sprites/client-icons.d.ts b/shared/svg-sprites/client-icons.d.ts index 78ef4593..d6d53744 100644 --- a/shared/svg-sprites/client-icons.d.ts +++ b/shared/svg-sprites/client-icons.d.ts @@ -3,9 +3,9 @@ * * This file has been auto generated by the svg-sprite generator. * Sprite source directory: D:\TeaSpeak\web\shared\img\client-icons - * Sprite count: 203 + * Sprite count: 202 */ -export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture-denied" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-microphone_broken" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; +export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-microphone_broken" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; export enum ClientIcon { About = "client-about", @@ -31,7 +31,6 @@ export enum ClientIcon { BookmarkRemove = "client-bookmark_remove", BrokenImage = "client-broken_image", BrowseAddonOnline = "client-browse-addon-online", - CaptureDenied = "client-capture-denied", Capture = "client-capture", ChangeNickname = "client-change_nickname", Changelog = "client-changelog", From 90e662a9c80809ccdfac1d1961af5d1657609aac Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 22:53:53 +0200 Subject: [PATCH 7/9] Removed old jquery $ reference --- loader/app/loader/template_loader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/app/loader/template_loader.ts b/loader/app/loader/template_loader.ts index 6bf25e93..02ba3646 100644 --- a/loader/app/loader/template_loader.ts +++ b/loader/app/loader/template_loader.ts @@ -8,7 +8,7 @@ function load_template_url(url: string) : Promise { return _template_promises[url]; return (_template_promises[url] = (async () => { - const response = await $.ajax(config.baseUrl + url); + const response = await (await fetch(config.baseUrl + url)).text(); let node = document.createElement("html"); node.innerHTML = response; From e77b9e1184b2293af23f39cd8d070bacd49fd8ee Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 22:55:43 +0200 Subject: [PATCH 8/9] Some minor code cleanups --- .../profiles/identities/TeamSpeakIdentity.ts | 7 ++-- shared/js/ui/modal/settings/Microphone.tsx | 17 +++++---- .../external-modal/Controller.ts | 5 ++- web/css/static/main.css | 35 ------------------- web/css/static/main.css.map | 1 - 5 files changed, 17 insertions(+), 48 deletions(-) delete mode 100644 web/css/static/main.css delete mode 100644 web/css/static/main.css.map diff --git a/shared/js/profiles/identities/TeamSpeakIdentity.ts b/shared/js/profiles/identities/TeamSpeakIdentity.ts index d4ac0b86..ca950edc 100644 --- a/shared/js/profiles/identities/TeamSpeakIdentity.ts +++ b/shared/js/profiles/identities/TeamSpeakIdentity.ts @@ -9,7 +9,6 @@ import { IdentitifyType, Identity } from "tc-shared/profiles/Identity"; -import {settings} from "tc-shared/settings"; import {arrayBufferBase64, base64_encode_ab, str2ab8} from "tc-shared/utils/buffers"; import {AbstractServerConnection} from "tc-shared/connection/ConnectionBase"; import {CommandResult} from "tc-shared/connection/ServerConnectionDeclaration"; @@ -259,7 +258,7 @@ export class TeaSpeakHandshakeHandler extends AbstractHandshakeIdentityHandler { error = error.extra_message || error.message; this.trigger_fail("failed to execute proof (" + error + ")"); }).then(() => this.trigger_success()); - }).catch(error => { + }).catch(() => { this.trigger_fail("failed to sign message"); }); } @@ -702,10 +701,10 @@ export class TeaSpeakIdentity implements Identity { const exit = () => { const timeout = setTimeout(() => resolve(true), 1000); - Promise.all(worker_promise).then(result => { + Promise.all(worker_promise).then(() => { clearTimeout(timeout); resolve(true); - }).catch(error => resolve(true)); + }).catch(() => resolve(true)); active = false; }; diff --git a/shared/js/ui/modal/settings/Microphone.tsx b/shared/js/ui/modal/settings/Microphone.tsx index bf98037d..b7dd03e9 100644 --- a/shared/js/ui/modal/settings/Microphone.tsx +++ b/shared/js/ui/modal/settings/Microphone.tsx @@ -5,12 +5,6 @@ import {LevelMeter} from "tc-shared/voice/RecorderBase"; import * as log from "tc-shared/log"; import {LogCategory, logWarn} from "tc-shared/log"; import {default_recorder} from "tc-shared/voice/RecorderProfile"; -import * as loader from "tc-loader"; -import {Stage} from "tc-loader"; -import {spawnReactModal} from "tc-shared/ui/react-elements/Modal"; -import {InternalModal} from "tc-shared/ui/react-elements/internal-modal/Controller"; -import {Translatable} from "tc-shared/ui/react-elements/i18n"; -import {MicrophoneSettings} from "tc-shared/ui/modal/settings/MicrophoneRenderer"; import {DeviceListState, getRecorderBackend, IDevice} from "tc-shared/audio/recorder"; export type MicrophoneSetting = "volume" | "vad-type" | "ppt-key" | "ppt-release-delay" | "ppt-release-delay-active" | "threshold-threshold"; @@ -315,6 +309,14 @@ export function initialize_audio_microphone_controller(events: Registry { @@ -345,4 +347,5 @@ loader.register_task(Stage.LOADED, { }); }, priority: -2 -}) \ No newline at end of file +}) +*/ \ No newline at end of file diff --git a/shared/js/ui/react-elements/external-modal/Controller.ts b/shared/js/ui/react-elements/external-modal/Controller.ts index 18a0a18b..e5460aa6 100644 --- a/shared/js/ui/react-elements/external-modal/Controller.ts +++ b/shared/js/ui/react-elements/external-modal/Controller.ts @@ -77,7 +77,10 @@ export abstract class AbstractExternalModalController extends EventControllerBas }); } catch (e) { this.modalState = ModalState.DESTROYED; - this.doDestroyWindow(); + if(__build.mode !== "debug") { + /* do not destroy the window in debug mode in order to debug what happened */ + this.doDestroyWindow(); + } throw e; } diff --git a/web/css/static/main.css b/web/css/static/main.css deleted file mode 100644 index fd4e4c5b..00000000 --- a/web/css/static/main.css +++ /dev/null @@ -1,35 +0,0 @@ -html, body { - overflow-y: hidden; - height: 100%; - width: 100%; - position: fixed; -} - -.app-container { - display: flex; - justify-content: stretch; - position: absolute; - top: 1.5em !important; - bottom: 0; - transition: all 0.5s linear; -} -.app-container .app { - width: 100%; - height: 100%; - margin: 0; - display: flex; - flex-direction: column; - resize: both; -} - -@media only screen and (max-width: 650px) { - html, body { - padding: 0 !important; - } - - .app-container { - bottom: 0; - } -} - -/*# sourceMappingURL=main.css.map */ diff --git a/web/css/static/main.css.map b/web/css/static/main.css.map deleted file mode 100644 index daab01e0..00000000 --- a/web/css/static/main.css.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"sourceRoot":"","sources":["main.scss"],"names":[],"mappings":"AAAA;EACC;EAEG;EACA;EACA;;;AAGJ;EACC;EACA;EACA;EAEA;EACG;EAEA;;AAEH;EACC;EACA;EACA;EAEA;EAAe;EAAwB;;;AAKzC;EACC;IACC;;;EAGD;IACC","file":"main.css"} \ No newline at end of file From a319633b368f5d789db4a0e1cf9b6746e4987d31 Mon Sep 17 00:00:00 2001 From: WolverinDEV Date: Wed, 19 Aug 2020 22:58:40 +0200 Subject: [PATCH 9/9] Fixed minimal microphone UI visual bug and updated the ChangeLog.md --- ChangeLog.md | 5 ++++- shared/js/ui/modal/settings/Heighlight.tsx | 4 ++-- shared/js/ui/modal/settings/Microphone.scss | 5 +++++ shared/js/ui/modal/settings/MicrophoneRenderer.tsx | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index 5159bc25..15218c11 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,7 +1,10 @@ # Changelog: * **11.08.20** - Fixed the voice push to talk delay - /* FIXME: Newcomer modal with the microphone */ + - Improved the microphone setting controller + - Heavily reworked the input recorder API + - Improved denied audio permission handling + * **09.08.20** - Added a "watch to gather" context menu entry for clients - Disassembled the current client icon sprite into his icons diff --git a/shared/js/ui/modal/settings/Heighlight.tsx b/shared/js/ui/modal/settings/Heighlight.tsx index 3f7da0e2..d7c3f0a0 100644 --- a/shared/js/ui/modal/settings/Heighlight.tsx +++ b/shared/js/ui/modal/settings/Heighlight.tsx @@ -4,10 +4,10 @@ import {useContext} from "react"; const cssStyle = require("./Heighlight.scss"); const HighlightContext = React.createContext(undefined); -export const HighlightContainer = (props: { children: React.ReactNode | React.ReactNode[], highlightedId?: string, onClick?: () => void }) => { +export const HighlightContainer = (props: { children: React.ReactNode | React.ReactNode[], classList?: string, highlightedId?: string, onClick?: () => void }) => { return ( -
+
{props.children}
diff --git a/shared/js/ui/modal/settings/Microphone.scss b/shared/js/ui/modal/settings/Microphone.scss index 317a7343..42163c7b 100644 --- a/shared/js/ui/modal/settings/Microphone.scss +++ b/shared/js/ui/modal/settings/Microphone.scss @@ -1,6 +1,11 @@ @import "../../../../css/static/properties"; @import "../../../../css/static/mixin"; +.highlightContainer { + height: 100%; + width: 100%; +} + .container { display: flex; flex-direction: row; diff --git a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx index ce7bd58a..b6af7b4e 100644 --- a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx +++ b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx @@ -573,7 +573,7 @@ export const MicrophoneSettings = (props: { events: Registry setHighlighted(event.field)); return ( - props.events.fire("action_help_click")}> + props.events.fire("action_help_click")} classList={cssStyle.highlightContainer}>