diff --git a/ChangeLog.md b/ChangeLog.md index f56b5f25..5159bc25 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,7 +1,7 @@ # Changelog: * **11.08.20** - Fixed the voice push to talk delay - + /* FIXME: Newcomer modal with the microphone */ * **09.08.20** - Added a "watch to gather" context menu entry for clients - Disassembled the current client icon sprite into his icons diff --git a/loader/app/targets/shared.ts b/loader/app/targets/shared.ts index 834a934a..3ca6cf57 100644 --- a/loader/app/targets/shared.ts +++ b/loader/app/targets/shared.ts @@ -1,6 +1,15 @@ import * as loader from "../loader/loader"; import {Stage} from "../loader/loader"; -import {detect as detectBrowser} from "detect-browser"; +import { + BrowserInfo, + detect as detectBrowser, +} from "detect-browser"; + +declare global { + interface Window { + detectedBrowser: BrowserInfo + } +} if(__build.target === "web") { loader.register_task(Stage.SETUP, { @@ -13,14 +22,15 @@ if(__build.target === "web") { return; console.log("Resolved browser manufacturer to \"%s\" version \"%s\" on %s", browser.name, browser.version, browser.os); - if(browser.type && browser.type !== "browser") { + if(browser.type !== "browser") { loader.critical_error("Your device isn't supported.", "User agent type " + browser.type + " isn't supported."); throw "unsupported user type"; } + window.detectedBrowser = browser; + switch (browser?.name) { case "aol": - case "bot": case "crios": case "ie": loader.critical_error("Browser not supported", "We're sorry, but your browser isn't supported."); diff --git a/shared/backend.d/audio/recorder.d.ts b/shared/backend.d/audio/recorder.d.ts deleted file mode 100644 index a5fdd45d..00000000 --- a/shared/backend.d/audio/recorder.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {AbstractInput, InputDevice, LevelMeter} from "tc-shared/voice/RecorderBase"; - -export function devices() : InputDevice[]; - -export function device_refresh_available() : boolean; -export function refresh_devices() : Promise; - -export function create_input() : AbstractInput; -export function create_levelmeter(device: InputDevice) : Promise; \ No newline at end of file diff --git a/shared/img/client-icons/microphone_broken.svg b/shared/img/client-icons/microphone_broken.svg new file mode 100644 index 00000000..c632b55d --- /dev/null +++ b/shared/img/client-icons/microphone_broken.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/shared/js/ConnectionHandler.ts b/shared/js/ConnectionHandler.ts index 2c8f2a5a..a307cf25 100644 --- a/shared/js/ConnectionHandler.ts +++ b/shared/js/ConnectionHandler.ts @@ -38,6 +38,7 @@ import {PluginCmdRegistry} from "tc-shared/connection/PluginCmdHandler"; import {W2GPluginCmdHandler} from "tc-shared/video-viewer/W2GPlugin"; import {VoiceConnectionStatus} from "tc-shared/connection/VoiceConnection"; import {getServerConnectionFactory} from "tc-shared/connection/ConnectionFactory"; +import {getRecorderBackend} from "tc-shared/audio/recorder"; export enum DisconnectReason { HANDLER_DESTROYED, @@ -738,6 +739,8 @@ export class ConnectionHandler { const support_record = basic_voice_support && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); const support_playback = basic_voice_support && (!targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec)); + const hasInputDevice = getRecorderBackend().getDeviceList().getPermissionState() === "granted" && !!vconnection.voice_recorder(); + const property_update = { client_input_muted: this.client_status.input_muted, client_output_muted: this.client_status.output_muted @@ -749,12 +752,11 @@ export class ConnectionHandler { if(!this.serverConnection.connected() || vconnection.getConnectionState() !== VoiceConnectionStatus.Connected) { property_update["client_input_hardware"] = false; property_update["client_output_hardware"] = false; - this.client_status.input_hardware = true; /* IDK if we have input hardware or not, but it dosn't matter at all so */ + this.client_status.input_hardware = hasInputDevice; /* no icons are shown so no update at all */ } else { - const audio_source = vconnection.voice_recorder(); - const recording_supported = typeof(audio_source) !== "undefined" && audio_source.record_supported && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); + const recording_supported = hasInputDevice && (!targetChannel || vconnection.encoding_supported(targetChannel.properties.channel_codec)); const playback_supported = !targetChannel || vconnection.decoding_supported(targetChannel.properties.channel_codec); property_update["client_input_hardware"] = recording_supported; @@ -807,7 +809,7 @@ export class ConnectionHandler { this.client_status.sound_record_supported = support_record; this.client_status.sound_playback_supported = support_playback; - if(vconnection && vconnection.voice_recorder() && vconnection.voice_recorder().record_supported) { + if(vconnection && vconnection.voice_recorder()) { const active = !this.client_status.input_muted && !this.client_status.output_muted; /* No need to start the microphone when we're not even connected */ diff --git a/shared/js/audio/recorder.ts b/shared/js/audio/recorder.ts new file mode 100644 index 00000000..c70f8085 --- /dev/null +++ b/shared/js/audio/recorder.ts @@ -0,0 +1,157 @@ +import {AbstractInput, LevelMeter} from "tc-shared/voice/RecorderBase"; +import {Registry} from "tc-shared/events"; + +export type DeviceQueryResult = {} + +export interface AudioRecorderBacked { + createInput() : AbstractInput; + createLevelMeter(device: IDevice) : Promise; + + getDeviceList() : DeviceList; +} + +export interface DeviceListEvents { + /* + * Should only trigger if the list really changed. + */ + notify_list_updated: { + removedDeviceCount: number, + addedDeviceCount: number + }, + + notify_state_changed: { + oldState: DeviceListState; + newState: DeviceListState; + }, + + notify_permissions_changed: { + oldState: PermissionState, + newState: PermissionState + } +} + +export type DeviceListState = "healthy" | "uninitialized" | "no-permissions" | "error"; + +export interface IDevice { + deviceId: string; + + driver: string; + name: string; +} +export namespace IDevice { + export const NoDeviceId = "none"; +} + +export type PermissionState = "granted" | "denied" | "unknown"; + +export interface DeviceList { + getEvents() : Registry; + + isRefreshAvailable() : boolean; + refresh() : Promise; + + /* implicitly update our own permission state */ + requestPermissions() : Promise; + getPermissionState() : PermissionState; + + getStatus() : DeviceListState; + getDevices() : IDevice[]; + + getDefaultDeviceId() : string; + + awaitHealthy(): Promise; + awaitInitialized() : Promise; +} + +export abstract class AbstractDeviceList implements DeviceList { + protected readonly events: Registry; + protected listState: DeviceListState; + protected permissionState: PermissionState; + + protected constructor() { + this.events = new Registry(); + this.permissionState = "unknown"; + this.listState = "uninitialized"; + } + + getStatus(): DeviceListState { + return this.listState; + } + + getPermissionState(): PermissionState { + return this.permissionState; + } + + protected setState(state: DeviceListState) { + if(this.listState === state) + return; + + const oldState = this.listState; + this.listState = state; + this.events.fire("notify_state_changed", { oldState: oldState, newState: state }); + } + + protected setPermissionState(state: PermissionState) { + if(this.permissionState === state) + return; + + const oldState = this.permissionState; + this.permissionState = state; + this.events.fire("notify_permissions_changed", { oldState: oldState, newState: state }); + } + + awaitInitialized(): Promise { + if(this.listState !== "uninitialized") + return Promise.resolve(); + + return new Promise(resolve => { + const callback = (event: DeviceListEvents["notify_state_changed"]) => { + if(event.newState !== "uninitialized") + return; + + this.events.off("notify_state_changed", callback); + resolve(); + }; + this.events.on("notify_state_changed", callback); + }); + } + + awaitHealthy(): Promise { + if(this.listState === "healthy") + return Promise.resolve(); + + return new Promise(resolve => { + const callback = (event: DeviceListEvents["notify_state_changed"]) => { + if(event.newState !== "healthy") + return; + + this.events.off("notify_state_changed", callback); + resolve(); + }; + this.events.on("notify_state_changed", callback); + }); + } + + abstract getDefaultDeviceId(): string; + abstract getDevices(): IDevice[]; + abstract getEvents(): Registry; + abstract isRefreshAvailable(): boolean; + abstract refresh(): Promise; + abstract requestPermissions(): Promise; +} + +let recorderBackend: AudioRecorderBacked; + +export function getRecorderBackend() : AudioRecorderBacked { + if(typeof recorderBackend === "undefined") + throw tr("the recorder backend hasn't been set yet"); + + return recorderBackend; +} + +export function setRecorderBackend(instance: AudioRecorderBacked) { + if(typeof recorderBackend !== "undefined") + throw tr("a recorder backend has already been initialized"); + + recorderBackend = instance; +} diff --git a/shared/js/main.tsx b/shared/js/main.tsx index f36c7c2d..86502736 100644 --- a/shared/js/main.tsx +++ b/shared/js/main.tsx @@ -21,7 +21,6 @@ import {spawnYesNo} from "tc-shared/ui/modal/ModalYesNo"; import {formatMessage} from "tc-shared/ui/frames/chat"; import {openModalNewcomer} from "tc-shared/ui/modal/ModalNewcomer"; import * as aplayer from "tc-backend/audio/player"; -import * as arecorder from "tc-backend/audio/recorder"; import * as ppt from "tc-backend/ppt"; import * as keycontrol from "./KeyControl"; import * as React from "react"; @@ -182,8 +181,6 @@ async function initialize_app() { aplayer.on_ready(() => aplayer.set_master_volume(settings.global(Settings.KEY_SOUND_MASTER) / 100)); else log.warn(LogCategory.GENERAL, tr("Client does not support aplayer.set_master_volume()... May client is too old?")); - if(arecorder.device_refresh_available()) - arecorder.refresh_devices(); }); set_default_recorder(new RecorderProfile("default")); @@ -512,7 +509,7 @@ const task_teaweb_starter: loader.Task = { if(!aplayer.initializeFromGesture) { console.error(tr("Missing aplayer.initializeFromGesture")); } else - $(document).one('click', event => aplayer.initializeFromGesture()); + $(document).one('click', () => aplayer.initializeFromGesture()); } } catch (ex) { console.error(ex.stack); diff --git a/shared/js/ui/modal/settings/Microphone.scss b/shared/js/ui/modal/settings/Microphone.scss index 99de55c1..d1b4446e 100644 --- a/shared/js/ui/modal/settings/Microphone.scss +++ b/shared/js/ui/modal/settings/Microphone.scss @@ -241,7 +241,7 @@ } .header { - height: 2.6em; + height: 3em; flex-grow: 0; flex-shrink: 0; @@ -266,7 +266,7 @@ white-space: nowrap; } - .btn { + button { flex-shrink: 0; flex-grow: 0; @@ -452,10 +452,24 @@ text-align: center; } + button { + width: 10em; + align-self: center; + margin-top: 2em; + } + &.hidden { pointer-events: none; opacity: 0; } + + :global(.icon_em) { + align-self: center; + font-size: 10em; + + margin-bottom: .25em; + margin-top: -.25em; + } } } diff --git a/shared/js/ui/modal/settings/Microphone.tsx b/shared/js/ui/modal/settings/Microphone.tsx index 3d8449a6..77bdb194 100644 --- a/shared/js/ui/modal/settings/Microphone.tsx +++ b/shared/js/ui/modal/settings/Microphone.tsx @@ -2,7 +2,6 @@ import * as aplayer from "tc-backend/audio/player"; import * as React from "react"; import {Registry} from "tc-shared/events"; import {LevelMeter} from "tc-shared/voice/RecorderBase"; -import * as arecorder from "tc-backend/audio/recorder"; import * as log from "tc-shared/log"; import {LogCategory, logWarn} from "tc-shared/log"; import {default_recorder} from "tc-shared/voice/RecorderProfile"; @@ -12,6 +11,7 @@ import {spawnReactModal} from "tc-shared/ui/react-elements/Modal"; import {InternalModal} from "tc-shared/ui/react-elements/internal-modal/Controller"; import {Translatable} from "tc-shared/ui/react-elements/i18n"; import {MicrophoneSettings} from "tc-shared/ui/modal/settings/MicrophoneRenderer"; +import {DeviceListState, getRecorderBackend, IDevice} from "tc-shared/audio/recorder"; export type MicrophoneSetting = "volume" | "vad-type" | "ppt-key" | "ppt-release-delay" | "ppt-release-delay-active" | "threshold-threshold"; @@ -29,6 +29,7 @@ export interface MicrophoneSettingsEvents { setting: MicrophoneSetting }, + "action_request_permissions": {}, "action_set_selected_device": { deviceId: string }, "action_set_selected_device_result": { deviceId: string, /* on error it will contain the current selected device */ @@ -48,9 +49,11 @@ export interface MicrophoneSettingsEvents { } "notify_devices": { - status: "success" | "error" | "audio-not-initialized", + status: "success" | "error" | "audio-not-initialized" | "no-permissions", error?: string, + shouldAsk?: boolean, + devices?: MicrophoneDevice[] selectedDevice?: string; }, @@ -62,13 +65,17 @@ export interface MicrophoneSettingsEvents { level?: number, error?: string - }} + }}, + + status: Exclude }, notify_destroy: {} } export function initialize_audio_microphone_controller(events: Registry) { + const recorderBackend = getRecorderBackend(); + /* level meters */ { const level_meters: {[key: string]:Promise} = {}; @@ -80,7 +87,7 @@ export function initialize_audio_microphone_controller(events: Registry e.destory()); + meter.then(e => e.destroy()); }); Object.keys(level_info).forEach(e => delete level_info[e]); }; @@ -88,37 +95,42 @@ export function initialize_audio_microphone_controller(events: Registry { destroy_meters(); - for(const device of arecorder.devices()) { - let promise = arecorder.create_levelmeter(device).then(meter => { - meter.set_observer(level => { - if(level_meters[device.unique_id] !== promise) return; /* old level meter */ + level_info["none"] = { deviceId: "none", status: "success", level: 0 }; - level_info[device.unique_id] = { - deviceId: device.unique_id, + for(const device of recorderBackend.getDeviceList().getDevices()) { + let promise = recorderBackend.createLevelMeter(device).then(meter => { + meter.set_observer(level => { + if(level_meters[device.deviceId] !== promise) return; /* old level meter */ + + level_info[device.deviceId] = { + deviceId: device.deviceId, status: "success", level: level }; }); return Promise.resolve(meter); }).catch(error => { - if(level_meters[device.unique_id] !== promise) return; /* old level meter */ - level_info[device.unique_id] = { - deviceId: device.unique_id, + if(level_meters[device.deviceId] !== promise) return; /* old level meter */ + level_info[device.deviceId] = { + deviceId: device.deviceId, status: "error", error: error }; - log.warn(LogCategory.AUDIO, tr("Failed to initialize a level meter for device %s (%s): %o"), device.unique_id, device.driver + ":" + device.name, error); + log.warn(LogCategory.AUDIO, tr("Failed to initialize a level meter for device %s (%s): %o"), device.deviceId, device.driver + ":" + device.name, error); return Promise.reject(error); }); - level_meters[device.unique_id] = promise; + level_meters[device.deviceId] = promise; } }; level_update_task = setInterval(() => { + const deviceListStatus = recorderBackend.getDeviceList().getStatus(); + events.fire("notify_device_level", { - level: level_info + level: level_info, + status: deviceListStatus === "error" ? "uninitialized" : deviceListStatus }); }, 50); @@ -142,34 +154,43 @@ export function initialize_audio_microphone_controller(events: Registry { - return arecorder.device_refresh_available() && event.refresh_list ? arecorder.refresh_devices() : Promise.resolve(); - }).catch(error => { - log.warn(LogCategory.AUDIO, tr("Failed to refresh device list: %o"), error); - return Promise.resolve(); - }).then(() => { - const devices = arecorder.devices(); + const deviceList = recorderBackend.getDeviceList(); + switch (deviceList.getStatus()) { + case "no-permissions": + events.fire_async("notify_devices", { status: "no-permissions", shouldAsk: deviceList.getPermissionState() === "denied" }); + return; + + case "uninitialized": + events.fire_async("notify_devices", { status: "audio-not-initialized" }); + return; + } + + if(event.refresh_list && deviceList.isRefreshAvailable()) { + /* will automatically trigger a device list changed event if something has changed */ + deviceList.refresh().then(() => {}); + } else { + const devices = deviceList.getDevices(); events.fire_async("notify_devices", { status: "success", - selectedDevice: default_recorder.current_device() ? default_recorder.current_device().unique_id : "none", - devices: devices.map(e => { return { id: e.unique_id, name: e.name, driver: e.driver }}) + selectedDevice: default_recorder.getDeviceId(), + devices: devices.map(e => { return { id: e.deviceId, name: e.name, driver: e.driver }}) }); - }); + } }); events.on("action_set_selected_device", event => { - const device = arecorder.devices().find(e => e.unique_id === event.deviceId); - if(!device && event.deviceId !== "none") { - events.fire_async("action_set_selected_device_result", { status: "error", error: tr("Invalid device id"), deviceId: default_recorder.current_device().unique_id }); + const device = recorderBackend.getDeviceList().getDevices().find(e => e.deviceId === event.deviceId); + if(!device && event.deviceId !== IDevice.NoDeviceId) { + events.fire_async("action_set_selected_device_result", { status: "error", error: tr("Invalid device id"), deviceId: default_recorder.getDeviceId() }); return; } default_recorder.set_device(device).then(() => { - console.debug(tr("Changed default microphone device")); + console.debug(tr("Changed default microphone device to %s"), event.deviceId); events.fire_async("action_set_selected_device_result", { status: "success", deviceId: event.deviceId }); }).catch((error) => { - log.warn(LogCategory.AUDIO, tr("Failed to change microphone to device %s: %o"), device ? device.unique_id : "none", error); + log.warn(LogCategory.AUDIO, tr("Failed to change microphone to device %s: %o"), device ? device.deviceId : IDevice.NoDeviceId, error); events.fire_async("action_set_selected_device_result", { status: "success", deviceId: event.deviceId }); }); }); @@ -265,7 +286,59 @@ export function initialize_audio_microphone_controller(events: Registry recorderBackend.getDeviceList().requestPermissions().then(result => { + console.error("Permission request result: %o", result); + + if(result === "granted") { + /* we've nothing to do, the device change event will already update out list */ + } else { + events.fire_async("notify_devices", { status: "no-permissions", shouldAsk: result === "denied" }); + return; + } + })); + + events.on("notify_destroy", recorderBackend.getDeviceList().getEvents().on("notify_list_updated", () => { + events.fire("query_devices"); + })); + + events.on("notify_destroy", recorderBackend.getDeviceList().getEvents().on("notify_state_changed", () => { + events.fire("query_devices"); + })); + if(!aplayer.initialized()) { aplayer.on_ready(() => { events.fire_async("query_devices"); }); } -} \ No newline at end of file +} + + +loader.register_task(Stage.LOADED, { + name: "test", + function: async () => { + aplayer.on_ready(() => { + const modal = spawnReactModal(class extends InternalModal { + settings = new Registry(); + constructor() { + super(); + + initialize_audio_microphone_controller(this.settings); + } + + renderBody(): React.ReactElement { + return
+ +
; + } + + title(): string | React.ReactElement { + return "test"; + } + }); + + modal.show(); + }); + }, + priority: -2 +}) diff --git a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx index 89369a89..2ad2fb7c 100644 --- a/shared/js/ui/modal/settings/MicrophoneRenderer.tsx +++ b/shared/js/ui/modal/settings/MicrophoneRenderer.tsx @@ -9,13 +9,13 @@ import {ClientIcon} from "svg-sprites/client-icons"; import {LoadingDots} from "tc-shared/ui/react-elements/LoadingDots"; import {createErrorModal} from "tc-shared/ui/elements/Modal"; import {Slider} from "tc-shared/ui/react-elements/Slider"; -import MicrophoneSettings = modal.settings.MicrophoneSettings; import {RadioButton} from "tc-shared/ui/react-elements/RadioButton"; import {VadType} from "tc-shared/voice/RecorderProfile"; import {key_description, KeyDescriptor} from "tc-shared/PPTListener"; import {spawnKeySelect} from "tc-shared/ui/modal/ModalKeySelect"; import {Checkbox} from "tc-shared/ui/react-elements/Checkbox"; import {BoxedInputField} from "tc-shared/ui/react-elements/InputField"; +import {IDevice} from "tc-shared/audio/recorder"; const cssStyle = require("./Microphone.scss"); @@ -37,28 +37,41 @@ const MicrophoneStatus = (props: { state: MicrophoneSelectedState }) => { } } -type ActivityBarStatus = { mode: "success" } | { mode: "error", message: string } | { mode: "loading" }; +type ActivityBarStatus = { mode: "success" } | { mode: "error", message: string } | { mode: "loading" } | { mode: "uninitialized" }; const ActivityBar = (props: { events: Registry, deviceId: string, disabled?: boolean }) => { const refHider = useRef(); const [ status, setStatus ] = useState({ mode: "loading" }); props.events.reactUse("notify_device_level", event => { - const device = event.level[props.deviceId]; - if(!device) { - if(status.mode === "loading") + if(event.status === "uninitialized") { + if(status.mode === "uninitialized") return; - setStatus({ mode: "loading" }); - } else if(device.status === "success") { - if(status.mode !== "success") { - setStatus({ mode: "success" }); - } - refHider.current.style.width = (100 - device.level) + "%"; + setStatus({ mode: "uninitialized" }); + } else if(event.status === "no-permissions") { + const noPermissionsMessage = tr("no permissions"); + if(status.mode === "error" && status.message === noPermissionsMessage) + return; + + setStatus({ mode: "error", message: noPermissionsMessage }); } else { - if(status.mode === "error" && status.message === device.error) - return; + const device = event.level[props.deviceId]; + if(!device) { + if(status.mode === "loading") + return; - setStatus({ mode: "error", message: device.error }); + setStatus({ mode: "loading" }); + } else if(device.status === "success") { + if(status.mode !== "success") { + setStatus({ mode: "success" }); + } + refHider.current.style.width = (100 - device.level) + "%"; + } else { + if(status.mode === "error" && status.message === device.error) + return; + + setStatus({ mode: "error", message: device.error + "" }); + } } }); @@ -96,16 +109,51 @@ const Microphone = (props: { events: Registry, device:
{props.device.name}
- + {props.device.id === IDevice.NoDeviceId ? undefined : + + }
); }; +type MicrophoneListState = { + type: "normal" | "loading" | "audio-not-initialized" +} | { + type: "error", + message: string +} | { + type: "no-permissions", + bySystem: boolean +} + +const PermissionDeniedOverlay = (props: { bySystem: boolean, shown: boolean, onRequestPermission: () => void }) => { + if(props.bySystem) { + return ( + + ); + } else { + return ( + + ); + } +} + const MicrophoneList = (props: { events: Registry }) => { - const [ state, setState ] = useState<"normal" | "loading" | "error" | "audio-not-initialized">(() => { + const [ state, setState ] = useState(() => { props.events.fire("query_devices"); - return "loading"; + return { type: "loading" }; }); const [ selectedDevice, setSelectedDevice ] = useState<{ deviceId: string, mode: "selected" | "selecting" }>(); const [ deviceList, setDeviceList ] = useState([]); @@ -116,17 +164,20 @@ const MicrophoneList = (props: { events: Registry }) = switch (event.status) { case "success": setDeviceList(event.devices.slice(0)); - setState("normal"); + setState({ type: "normal" }); setSelectedDevice({ mode: "selected", deviceId: event.selectedDevice }); break; case "error": - setError(event.error || tr("Unknown error")); - setState("error"); + setState({ type: "error", message: event.error || tr("Unknown error") }); break; case "audio-not-initialized": - setState("audio-not-initialized"); + setState({ type: "audio-not-initialized" }); + break; + + case "no-permissions": + setState({ type: "no-permissions", bySystem: event.shouldAsk }); break; } }); @@ -144,25 +195,50 @@ const MicrophoneList = (props: { events: Registry }) = return (
-
+ -
+ -
+
+ Please grant access to your microphone. + +
+ props.events.fire("action_request_permissions")} + /> + + { + if(state.type !== "normal" || selectedDevice?.mode === "selecting") + return; + + props.events.fire("action_set_selected_device", { deviceId: IDevice.NoDeviceId }); + }} + /> + {deviceList.map(e => { - if(state !== "normal" || selectedDevice?.mode === "selecting") + if(state.type !== "normal" || selectedDevice?.mode === "selecting") return; props.events.fire("action_set_selected_device", { deviceId: e.id }); @@ -187,7 +263,7 @@ const ListRefreshButton = (props: { events: Registry } props.events.reactUse("query_devices", () => setUpdateTimeout(Date.now() + 2000)); - return ; } @@ -203,7 +279,6 @@ const VolumeSettings = (props: { events: Registry }) = if(event.setting !== "volume") return; - console.error("Set value: %o", event.value); refSlider.current?.setState({ value: event.value }); setValue(event.value); }); @@ -386,6 +461,7 @@ const ThresholdSelector = (props: { events: Registry } return "loading"; }); + const [ currentDevice, setCurrentDevice ] = useState(undefined); const [ isActive, setActive ] = useState(false); props.events.reactUse("notify_setting", event => { @@ -397,10 +473,18 @@ const ThresholdSelector = (props: { events: Registry } } }); + props.events.reactUse("notify_devices", event => { + setCurrentDevice(event.selectedDevice); + }); + + props.events.reactUse("action_set_selected_device_result", event => { + setCurrentDevice(event.deviceId); + }); + return (
- +
} disabled={value === "loading" || !isActive} - onChange={value => {}} + onChange={value => { props.events.fire("action_set_setting", { setting: "threshold-threshold", value: value })}} />
) diff --git a/shared/js/voice/Filter.ts b/shared/js/voice/Filter.ts new file mode 100644 index 00000000..58ff5345 --- /dev/null +++ b/shared/js/voice/Filter.ts @@ -0,0 +1,53 @@ +export enum FilterType { + THRESHOLD, + VOICE_LEVEL, + STATE +} + +export interface FilterBase { + readonly priority: number; + + set_enabled(flag: boolean) : void; + is_enabled() : boolean; +} + +export interface MarginedFilter { + get_margin_frames() : number; + set_margin_frames(value: number); +} + +export interface ThresholdFilter extends FilterBase, MarginedFilter { + readonly type: FilterType.THRESHOLD; + + get_threshold() : number; + set_threshold(value: number) : Promise; + + get_attack_smooth() : number; + get_release_smooth() : number; + + set_attack_smooth(value: number); + set_release_smooth(value: number); + + callback_level?: (value: number) => any; +} + +export interface VoiceLevelFilter extends FilterBase, MarginedFilter { + type: FilterType.VOICE_LEVEL; + + get_level() : number; +} + +export interface StateFilter extends FilterBase { + type: FilterType.STATE; + + set_state(state: boolean) : Promise; + is_active() : boolean; /* if true the the filter allows data to pass */ +} + +export type FilterTypeClass = + T extends FilterType.STATE ? StateFilter : + T extends FilterType.VOICE_LEVEL ? VoiceLevelFilter : + T extends FilterType.THRESHOLD ? ThresholdFilter : + never; + +export type Filter = ThresholdFilter | VoiceLevelFilter | StateFilter; \ No newline at end of file diff --git a/shared/js/voice/RecorderBase.ts b/shared/js/voice/RecorderBase.ts index deab0e8f..e6d36b29 100644 --- a/shared/js/voice/RecorderBase.ts +++ b/shared/js/voice/RecorderBase.ts @@ -1,14 +1,6 @@ -export interface InputDevice { - unique_id: string; - driver: string; - name: string; - default_input: boolean; - - supported: boolean; - - sample_rate: number; - channels: number; -} +import {IDevice} from "tc-shared/audio/recorder"; +import {Registry} from "tc-shared/events"; +import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; export enum InputConsumerType { CALLBACK, @@ -31,47 +23,6 @@ export interface NodeInputConsumer extends InputConsumer { } -export namespace filter { - export enum Type { - THRESHOLD, - VOICE_LEVEL, - STATE - } - - export interface Filter { - type: Type; - - is_enabled() : boolean; - } - - export interface MarginedFilter { - get_margin_frames() : number; - set_margin_frames(value: number); - } - - export interface ThresholdFilter extends Filter, MarginedFilter { - get_threshold() : number; - set_threshold(value: number) : Promise; - - get_attack_smooth() : number; - get_release_smooth() : number; - - set_attack_smooth(value: number); - set_release_smooth(value: number); - - callback_level?: (value: number) => any; - } - - export interface VoiceLevelFilter extends Filter, MarginedFilter { - get_level() : number; - } - - export interface StateFilter extends Filter { - set_state(state: boolean) : Promise; - is_active() : boolean; /* if true the the filter allows data to pass */ - } -} - export enum InputState { PAUSED, INITIALIZING, @@ -87,36 +38,39 @@ export enum InputStartResult { ENOTSUPPORTED = "enotsupported" } -export interface AbstractInput { - callback_begin: () => any; - callback_end: () => any; +export interface InputEvents { + notify_voice_start: {}, + notify_voice_end: {} +} - current_state() : InputState; +export interface AbstractInput { + readonly events: Registry; + + currentState() : InputState; start() : Promise; stop() : Promise; - current_device() : InputDevice | undefined; - set_device(device: InputDevice | undefined) : Promise; + currentDevice() : IDevice | undefined; + setDevice(device: IDevice | undefined) : Promise; - current_consumer() : InputConsumer | undefined; - set_consumer(consumer: InputConsumer) : Promise; + currentConsumer() : InputConsumer | undefined; + setConsumer(consumer: InputConsumer) : Promise; - get_filter(type: filter.Type) : filter.Filter | undefined; - supports_filter(type: filter.Type) : boolean; + supportsFilter(type: FilterType) : boolean; + createFilter(type: T, priority: number) : FilterTypeClass; - clear_filter(); - disable_filter(type: filter.Type); - enable_filter(type: filter.Type); + removeFilter(filter: Filter); + resetFilter(); - get_volume() : number; - set_volume(volume: number); + getVolume() : number; + setVolume(volume: number); } export interface LevelMeter { - device() : InputDevice; + device() : IDevice; set_observer(callback: (value: number) => any); - destory(); + destroy(); } \ No newline at end of file diff --git a/shared/js/voice/RecorderProfile.ts b/shared/js/voice/RecorderProfile.ts index c8d97bc9..b8984369 100644 --- a/shared/js/voice/RecorderProfile.ts +++ b/shared/js/voice/RecorderProfile.ts @@ -1,12 +1,13 @@ import * as log from "tc-shared/log"; -import {AbstractInput, filter, InputDevice} from "tc-shared/voice/RecorderBase"; +import {LogCategory, logWarn} from "tc-shared/log"; +import {AbstractInput} from "tc-shared/voice/RecorderBase"; import {KeyDescriptor, KeyHook} from "tc-shared/PPTListener"; -import {LogCategory} from "tc-shared/log"; import {Settings, settings} from "tc-shared/settings"; import {ConnectionHandler} from "tc-shared/ConnectionHandler"; import * as aplayer from "tc-backend/audio/player"; -import * as arecorder from "tc-backend/audio/recorder"; import * as ppt from "tc-backend/ppt"; +import {getRecorderBackend, IDevice} from "tc-shared/audio/recorder"; +import {FilterType, StateFilter} from "tc-shared/voice/Filter"; export type VadType = "threshold" | "push_to_talk" | "active"; export interface RecorderProfileConfig { @@ -46,18 +47,20 @@ export class RecorderProfile { current_handler: ConnectionHandler; - callback_input_change: (old_input: AbstractInput, new_input: AbstractInput) => Promise; + callback_input_change: (oldInput: AbstractInput | undefined, newInput: AbstractInput | undefined) => Promise; callback_start: () => any; callback_stop: () => any; callback_unmount: () => any; /* called if somebody else takes the ownership */ - record_supported: boolean; - - private pptHook: KeyHook; + private readonly pptHook: KeyHook; private pptTimeout: number; private pptHookRegistered: boolean; + private registeredFilter = { + "ppt-gate": undefined as StateFilter + } + constructor(name: string, volatile?: boolean) { this.name = name; this.volatile = typeof(volatile) === "boolean" ? volatile : false; @@ -68,84 +71,96 @@ export class RecorderProfile { clearTimeout(this.pptTimeout); this.pptTimeout = setTimeout(() => { - const f = this.input.get_filter(filter.Type.STATE) as filter.StateFilter; - if(f) f.set_state(true); + this.registeredFilter["ppt-gate"]?.set_state(true); }, Math.max(this.config.vad_push_to_talk.delay, 0)); }, + callback_press: () => { if(this.pptTimeout) clearTimeout(this.pptTimeout); - const f = this.input.get_filter(filter.Type.STATE) as filter.StateFilter; - if(f) f.set_state(false); + this.registeredFilter["ppt-gate"]?.set_state(false); }, cancel: false } as KeyHook; this.pptHookRegistered = false; - this.record_supported = true; } async initialize() : Promise { + { + let config = {}; + try { + config = settings.static_global(Settings.FN_PROFILE_RECORD(this.name), {}) as RecorderProfileConfig; + } catch (error) { + logWarn(LogCategory.AUDIO, tr("Failed to load old recorder profile config for %s"), this.name); + } + + /* default values */ + this.config = { + version: 1, + device_id: undefined, + volume: 100, + + vad_threshold: { + threshold: 25 + }, + vad_type: "threshold", + vad_push_to_talk: { + delay: 300, + key_alt: false, + key_ctrl: false, + key_shift: false, + key_windows: false, + key_code: 't' + } + }; + + Object.assign(this.config, config || {}); + } + aplayer.on_ready(async () => { + await getRecorderBackend().getDeviceList().awaitHealthy(); + this.initialize_input(); await this.load(); - await this.reinitialize_filter(); + await this.reinitializeFilter(); }); } private initialize_input() { - this.input = arecorder.create_input(); - this.input.callback_begin = () => { + this.input = getRecorderBackend().createInput(); + + this.input.events.on("notify_voice_start", () => { log.debug(LogCategory.VOICE, "Voice start"); if(this.callback_start) this.callback_start(); - }; + }); - this.input.callback_end = () => { + this.input.events.on("notify_voice_end", () => { log.debug(LogCategory.VOICE, "Voice end"); if(this.callback_stop) this.callback_stop(); - }; + }); //TODO: Await etc? this.callback_input_change && this.callback_input_change(undefined, this.input); } private async load() { - const config = settings.static_global(Settings.FN_PROFILE_RECORD(this.name), {}) as RecorderProfileConfig; - - /* default values */ - this.config = { - version: 1, - device_id: undefined, - volume: 100, - - vad_threshold: { - threshold: 25 - }, - vad_type: "threshold", - vad_push_to_talk: { - delay: 300, - key_alt: false, - key_ctrl: false, - key_shift: false, - key_windows: false, - key_code: 't' - } - }; - - Object.assign(this.config, config || {}); - this.input.set_volume(this.config.volume / 100); + this.input.setVolume(this.config.volume / 100); { - const all_devices = arecorder.devices(); - const devices = all_devices.filter(e => e.default_input || e.unique_id === this.config.device_id); - const device = devices.find(e => e.unique_id === this.config.device_id) || devices[0]; + const allDevices = getRecorderBackend().getDeviceList().getDevices(); + const defaultDeviceId = getRecorderBackend().getDeviceList().getDefaultDeviceId(); + console.error("Devices: %o | Searching: %s", allDevices, this.config.device_id); - log.info(LogCategory.VOICE, tr("Loaded record profile device %s | %o (%o)"), this.config.device_id, device, all_devices); + const devices = allDevices.filter(e => e.deviceId === defaultDeviceId || e.deviceId === this.config.device_id); + const device = devices.find(e => e.deviceId === this.config.device_id) || devices[0]; + + log.info(LogCategory.VOICE, tr("Loaded record profile device %s | %o (%o)"), this.config.device_id, device, allDevices); try { - await this.input.set_device(device); + await this.input.setDevice(device); } catch(error) { log.error(LogCategory.VOICE, tr("Failed to set input device (%o)"), error); } @@ -157,38 +172,36 @@ export class RecorderProfile { settings.changeGlobal(Settings.FN_PROFILE_RECORD(this.name), this.config); } - private async reinitialize_filter() { + private async reinitializeFilter() { if(!this.input) return; - this.input.clear_filter(); + /* TODO: Really required? If still same input we can just use the registered filters */ + + this.input.resetFilter(); + delete this.registeredFilter["ppt-gate"]; + if(this.pptHookRegistered) { ppt.unregister_key_hook(this.pptHook); this.pptHookRegistered = false; } if(this.config.vad_type === "threshold") { - const filter_ = this.input.get_filter(filter.Type.THRESHOLD) as filter.ThresholdFilter; - await filter_.set_threshold(this.config.vad_threshold.threshold); - await filter_.set_margin_frames(10); /* 500ms */ + const filter = this.input.createFilter(FilterType.THRESHOLD, 100); + await filter.set_threshold(this.config.vad_threshold.threshold); - /* legacy client support */ - if('set_attack_smooth' in filter_) - filter_.set_attack_smooth(.25); - - if('set_release_smooth' in filter_) - filter_.set_release_smooth(.9); - - this.input.enable_filter(filter.Type.THRESHOLD); + filter.set_margin_frames(10); /* 500ms */ + filter.set_attack_smooth(.25); + filter.set_release_smooth(.9); } else if(this.config.vad_type === "push_to_talk") { - const filter_ = this.input.get_filter(filter.Type.STATE) as filter.StateFilter; - await filter_.set_state(true); + const filter = this.input.createFilter(FilterType.STATE, 100); + await filter.set_state(true); + this.registeredFilter["ppt-gate"] = filter; for(const key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) this.pptHook[key] = this.config.vad_push_to_talk[key]; + ppt.register_key_hook(this.pptHook); this.pptHookRegistered = true; - - this.input.enable_filter(filter.Type.STATE); } else if(this.config.vad_type === "active") {} } @@ -199,7 +212,7 @@ export class RecorderProfile { if(this.input) { try { - await this.input.set_consumer(undefined); + await this.input.setConsumer(undefined); } catch(error) { log.warn(LogCategory.VOICE, tr("Failed to unmount input consumer for profile (%o)"), error); } @@ -220,7 +233,7 @@ export class RecorderProfile { return false; this.config.vad_type = type; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); return true; } @@ -231,7 +244,7 @@ export class RecorderProfile { return; this.config.vad_threshold.threshold = value; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); } @@ -240,7 +253,7 @@ export class RecorderProfile { for(const _key of ["key_alt", "key_ctrl", "key_shift", "key_windows", "key_code"]) this.config.vad_push_to_talk[_key] = key[_key]; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); } @@ -250,25 +263,24 @@ export class RecorderProfile { return; this.config.vad_push_to_talk.delay = value; - this.reinitialize_filter(); + this.reinitializeFilter(); this.save(); } - - current_device() : InputDevice | undefined { return this.input?.current_device(); } - set_device(device: InputDevice | undefined) : Promise { - this.config.device_id = device ? device.unique_id : undefined; + getDeviceId() : string { return this.config.device_id; } + set_device(device: IDevice | undefined) : Promise { + this.config.device_id = device ? device.deviceId : IDevice.NoDeviceId; this.save(); - return this.input.set_device(device); + return this.input?.setDevice(device) || Promise.resolve(); } - get_volume() : number { return this.input ? (this.input.get_volume() * 100) : this.config.volume; } + get_volume() : number { return this.input ? (this.input.getVolume() * 100) : this.config.volume; } set_volume(volume: number) { if(this.config.volume === volume) return; this.config.volume = volume; - this.input && this.input.set_volume(volume / 100); + this.input && this.input.setVolume(volume / 100); this.save(); } } \ No newline at end of file diff --git a/shared/svg-sprites/client-icons.d.ts b/shared/svg-sprites/client-icons.d.ts index 2d574769..d6d53744 100644 --- a/shared/svg-sprites/client-icons.d.ts +++ b/shared/svg-sprites/client-icons.d.ts @@ -3,9 +3,9 @@ * * This file has been auto generated by the svg-sprite generator. * Sprite source directory: D:\TeaSpeak\web\shared\img\client-icons - * Sprite count: 201 + * Sprite count: 202 */ -export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; +export type ClientIconClass = "client-about" | "client-activate_microphone" | "client-add" | "client-add_foe" | "client-add_folder" | "client-add_friend" | "client-addon-collection" | "client-addon" | "client-apply" | "client-arrow_down" | "client-arrow_left" | "client-arrow_right" | "client-arrow_up" | "client-away" | "client-ban_client" | "client-ban_list" | "client-bookmark_add" | "client-bookmark_add_folder" | "client-bookmark_duplicate" | "client-bookmark_manager" | "client-bookmark_remove" | "client-broken_image" | "client-browse-addon-online" | "client-capture" | "client-change_nickname" | "client-changelog" | "client-channel_chat" | "client-channel_collapse_all" | "client-channel_commander" | "client-channel_create" | "client-channel_create_sub" | "client-channel_default" | "client-channel_delete" | "client-channel_edit" | "client-channel_expand_all" | "client-channel_green" | "client-channel_green_subscribed" | "client-channel_green_subscribed2" | "client-channel_private" | "client-channel_red" | "client-channel_red_subscribed" | "client-channel_switch" | "client-channel_unsubscribed" | "client-channel_yellow" | "client-channel_yellow_subscribed" | "client-check_update" | "client-client_hide" | "client-client_show" | "client-close_button" | "client-complaint_list" | "client-conflict-icon" | "client-connect" | "client-contact" | "client-copy" | "client-copy_url" | "client-d_sound" | "client-d_sound_me" | "client-d_sound_user" | "client-default" | "client-default_for_all_bookmarks" | "client-delete" | "client-delete_avatar" | "client-disconnect" | "client-down" | "client-download" | "client-edit" | "client-edit_friend_foe_status" | "client-emoticon" | "client-error" | "client-file_home" | "client-file_refresh" | "client-filetransfer" | "client-find" | "client-folder" | "client-folder_up" | "client-group_100" | "client-group_200" | "client-group_300" | "client-group_500" | "client-group_600" | "client-guisetup" | "client-hardware_input_muted" | "client-hardware_output_muted" | "client-home" | "client-hoster_button" | "client-hotkeys" | "client-icon-pack" | "client-iconsview" | "client-iconviewer" | "client-identity_default" | "client-identity_export" | "client-identity_import" | "client-identity_manager" | "client-info" | "client-input_muted" | "client-input_muted_local" | "client-invite_buddy" | "client-is_talker" | "client-kick_channel" | "client-kick_server" | "client-listview" | "client-loading_image" | "client-message_incoming" | "client-message_info" | "client-message_outgoing" | "client-messages" | "client-microphone_broken" | "client-minimize_button" | "client-moderated" | "client-move_client_to_own_channel" | "client-music" | "client-new_chat" | "client-notifications" | "client-offline_messages" | "client-on_whisperlist" | "client-output_muted" | "client-permission_channel" | "client-permission_client" | "client-permission_overview" | "client-permission_server_groups" | "client-phoneticsnickname" | "client-ping_1" | "client-ping_2" | "client-ping_3" | "client-ping_4" | "client-ping_calculating" | "client-ping_disconnected" | "client-play" | "client-player_chat" | "client-player_commander_off" | "client-player_commander_on" | "client-player_off" | "client-player_on" | "client-player_whisper" | "client-plugins" | "client-poke" | "client-present" | "client-recording_start" | "client-recording_stop" | "client-refresh" | "client-register" | "client-reload" | "client-remove_foe" | "client-remove_friend" | "client-security" | "client-selectfolder" | "client-send_complaint" | "client-server_green" | "client-server_log" | "client-server_query" | "client-settings" | "client-sort_by_name" | "client-sound-pack" | "client-soundpack" | "client-stop" | "client-subscribe_mode" | "client-subscribe_to_all_channels" | "client-subscribe_to_channel" | "client-subscribe_to_channel_family" | "client-switch_advanced" | "client-switch_standard" | "client-sync-disable" | "client-sync-enable" | "client-sync-icon" | "client-tab_close_button" | "client-talk_power_grant" | "client-talk_power_grant_next" | "client-talk_power_request" | "client-talk_power_request_cancel" | "client-talk_power_revoke" | "client-talk_power_revoke_all_grant_next" | "client-temp_server_password" | "client-temp_server_password_add" | "client-textformat" | "client-textformat_bold" | "client-textformat_foreground" | "client-textformat_italic" | "client-textformat_underline" | "client-theme" | "client-toggle_server_query_clients" | "client-toggle_whisper" | "client-token" | "client-token_use" | "client-translation" | "client-unsubscribe_from_all_channels" | "client-unsubscribe_from_channel_family" | "client-unsubscribe_mode" | "client-up" | "client-upload" | "client-upload_avatar" | "client-urlcatcher" | "client-user-account" | "client-virtualserver_edit" | "client-volume" | "client-w2g" | "client-warning" | "client-warning_external_link" | "client-warning_info" | "client-warning_question" | "client-weblist" | "client-whisper" | "client-whisperlists"; export enum ClientIcon { About = "client-about", @@ -114,6 +114,7 @@ export enum ClientIcon { MessageInfo = "client-message_info", MessageOutgoing = "client-message_outgoing", Messages = "client-messages", + MicrophoneBroken = "client-microphone_broken", MinimizeButton = "client-minimize_button", Moderated = "client-moderated", MoveClientToOwnChannel = "client-move_client_to_own_channel", diff --git a/web/app/audio/Recorder.ts b/web/app/audio/Recorder.ts new file mode 100644 index 00000000..a63de9b1 --- /dev/null +++ b/web/app/audio/Recorder.ts @@ -0,0 +1,766 @@ +import { + AbstractDeviceList, + AudioRecorderBacked, + DeviceList, + DeviceListEvents, + DeviceListState, + IDevice, + PermissionState +} from "tc-shared/audio/recorder"; +import {Registry} from "tc-shared/events"; +import * as rbase from "tc-shared/voice/RecorderBase"; +import { + AbstractInput, + CallbackInputConsumer, + InputConsumer, + InputConsumerType, InputEvents, + InputStartResult, + InputState, + LevelMeter, + NodeInputConsumer +} from "tc-shared/voice/RecorderBase"; +import * as log from "tc-shared/log"; +import {LogCategory, logWarn} from "tc-shared/log"; +import * as aplayer from "./player"; +import {JAbstractFilter, JStateFilter, JThresholdFilter} from "./RecorderFilter"; +import * as loader from "tc-loader"; +import {Filter, FilterType, FilterTypeClass} from "tc-shared/voice/Filter"; + +declare global { + interface MediaStream { + stop(); + } +} + +export interface WebIDevice extends IDevice { + groupId: string; +} + +function getUserMediaFunctionPromise() : (constraints: MediaStreamConstraints) => Promise { + if('mediaDevices' in navigator && 'getUserMedia' in navigator.mediaDevices) + return constraints => navigator.mediaDevices.getUserMedia(constraints); + + const _callbacked_function = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; + if(!_callbacked_function) + return undefined; + + return constraints => new Promise((resolve, reject) => _callbacked_function(constraints, resolve, reject)); +} + +async function requestMicrophoneMediaStream(constraints: MediaTrackConstraints, updateDeviceList: boolean) : Promise { + const mediaFunction = getUserMediaFunctionPromise(); + if(!mediaFunction) return InputStartResult.ENOTSUPPORTED; + + try { + log.info(LogCategory.AUDIO, tr("Requesting a microphone stream for device %s in group %s"), constraints.deviceId, constraints.groupId); + const stream = mediaFunction({ audio: constraints }); + + if(updateDeviceList && inputDeviceList.getStatus() === "no-permissions") { + inputDeviceList.refresh().then(() => {}); /* added the then body to avoid a inspection warning... */ + } + + return stream; + } catch(error) { + if('name' in error) { + if(error.name === "NotAllowedError") { + log.warn(LogCategory.AUDIO, tr("Microphone request failed (No permissions). Browser message: %o"), error.message); + return InputStartResult.ENOTALLOWED; + } else { + log.warn(LogCategory.AUDIO, tr("Microphone request failed. Request resulted in error: %o: %o"), error.name, error); + } + } else { + log.warn(LogCategory.AUDIO, tr("Failed to initialize recording stream (%o)"), error); + } + + return InputStartResult.EUNKNOWN; + } +} + +async function requestMicrophonePermissions() : Promise { + const begin = Date.now(); + try { + await getUserMediaFunctionPromise()({ audio: { deviceId: "default" }, video: false }); + return "granted"; + } catch (error) { + const end = Date.now(); + const isSystem = (end - begin) < 250; + log.debug(LogCategory.AUDIO, tr("Microphone device request took %d milliseconds. System answered: %s"), end - begin, isSystem); + return "denied"; + } +} + +let inputDeviceList: WebInputDeviceList; +class WebInputDeviceList extends AbstractDeviceList { + private devices: WebIDevice[]; + + private deviceListQueryPromise: Promise; + + constructor() { + super(); + + this.devices = []; + } + + getDefaultDeviceId(): string { + return "default"; + } + + getDevices(): IDevice[] { + return this.devices; + } + + getEvents(): Registry { + return this.events; + } + + getStatus(): DeviceListState { + return this.listState; + } + + isRefreshAvailable(): boolean { + return true; + } + + refresh(askPermissions?: boolean): Promise { + return this.queryDevices(askPermissions === true); + } + + async requestPermissions(): Promise { + if(this.permissionState !== "unknown") + return this.permissionState; + + let result = await requestMicrophonePermissions(); + if(result === "granted" && this.listState === "no-permissions") { + /* if called within doQueryDevices, queryDevices will just return the promise */ + this.queryDevices(false).then(() => {}); + } + this.setPermissionState(result); + return result; + } + + private queryDevices(askPermissions: boolean) : Promise { + if(this.deviceListQueryPromise) + return this.deviceListQueryPromise; + + this.deviceListQueryPromise = this.doQueryDevices(askPermissions).catch(error => { + log.error(LogCategory.AUDIO, tr("Failed to query microphone devices (%o)"), error); + + if(this.listState !== "healthy") + this.listState = "error"; + }).then(() => { + this.deviceListQueryPromise = undefined; + }); + + return this.deviceListQueryPromise || Promise.resolve(); + } + + private async doQueryDevices(askPermissions: boolean) { + let devices = await navigator.mediaDevices.enumerateDevices(); + let hasPermissions = devices.findIndex(e => e.label !== "") !== -1; + + if(!hasPermissions && askPermissions) { + this.setState("no-permissions"); + + let skipPermissionAsk = false; + if('permissions' in navigator && 'query' in navigator.permissions) { + try { + const result = await navigator.permissions.query({ name: "microphone" }); + if(result.state === "denied") { + this.setPermissionState("denied"); + skipPermissionAsk = true; + } + } catch (error) { + logWarn(LogCategory.GENERAL, tr("Failed to query for microphone permissions: %s"), error); + } + } + + if(skipPermissionAsk) { + /* request permissions */ + hasPermissions = await this.requestPermissions() === "granted"; + if(hasPermissions) { + devices = await navigator.mediaDevices.enumerateDevices(); + } + } + } + if(hasPermissions) { + this.setPermissionState("granted"); + } + + if(window.detectedBrowser?.name === "firefox") { + devices = [{ + label: tr("Default Firefox device"), + groupId: "default", + deviceId: "default", + kind: "audioinput", + + toJSON: undefined + }]; + } + + const inputDevices = devices.filter(e => e.kind === "audioinput"); + + const oldDeviceList = this.devices; + this.devices = []; + + let devicesAdded = 0; + for(const device of inputDevices) { + const oldIndex = oldDeviceList.findIndex(e => e.deviceId === device.deviceId); + if(oldIndex === -1) { + devicesAdded++; + } else { + oldDeviceList.splice(oldIndex, 1); + } + + this.devices.push({ + deviceId: device.deviceId, + driver: "WebAudio", + groupId: device.groupId, + name: device.label + }); + } + + this.events.fire("notify_list_updated", { addedDeviceCount: devicesAdded, removedDeviceCount: oldDeviceList.length }); + if(hasPermissions) { + this.setState("healthy"); + } else { + this.setState("no-permissions"); + } + } +} + +export class WebAudioRecorder implements AudioRecorderBacked { + createInput(): AbstractInput { + return new JavascriptInput(); + } + + async createLevelMeter(device: IDevice): Promise { + const meter = new JavascriptLevelmeter(device as any); + await meter.initialize(); + return meter; + } + + getDeviceList(): DeviceList { + return inputDeviceList; + } +} + +class JavascriptInput implements AbstractInput { + public readonly events: Registry; + + private _state: InputState = InputState.PAUSED; + private _current_device: WebIDevice | undefined; + private _current_consumer: InputConsumer; + + private _current_stream: MediaStream; + private _current_audio_stream: MediaStreamAudioSourceNode; + + private _audio_context: AudioContext; + private _source_node: AudioNode; /* last node which could be connected to the target; target might be the _consumer_node */ + private _consumer_callback_node: ScriptProcessorNode; + private readonly _consumer_audio_callback; + private _volume_node: GainNode; + private _mute_node: GainNode; + + private registeredFilters: (Filter & JAbstractFilter)[] = []; + private _filter_active: boolean = false; + + private _volume: number = 1; + + callback_begin: () => any = undefined; + callback_end: () => any = undefined; + + constructor() { + this.events = new Registry(); + + aplayer.on_ready(() => this._audio_initialized()); + this._consumer_audio_callback = this._audio_callback.bind(this); + } + + private _audio_initialized() { + this._audio_context = aplayer.context(); + if(!this._audio_context) + return; + + this._mute_node = this._audio_context.createGain(); + this._mute_node.gain.value = 0; + this._mute_node.connect(this._audio_context.destination); + + this._consumer_callback_node = this._audio_context.createScriptProcessor(1024 * 4); + this._consumer_callback_node.connect(this._mute_node); + + this._volume_node = this._audio_context.createGain(); + this._volume_node.gain.value = this._volume; + + this.initializeFilters(); + if(this._state === InputState.INITIALIZING) + this.start(); + } + + private initializeFilters() { + for(const filter of this.registeredFilters) { + if(filter.is_enabled()) + filter.finalize(); + } + + this.registeredFilters.sort((a, b) => a.priority - b.priority); + if(this._audio_context && this._volume_node) { + const active_filter = this.registeredFilters.filter(e => e.is_enabled()); + let stream: AudioNode = this._volume_node; + for(const f of active_filter) { + f.initialize(this._audio_context, stream); + stream = f.audio_node; + } + this._switch_source_node(stream); + } + } + + private _audio_callback(event: AudioProcessingEvent) { + if(!this._current_consumer || this._current_consumer.type !== InputConsumerType.CALLBACK) + return; + + const callback = this._current_consumer as CallbackInputConsumer; + if(callback.callback_audio) + callback.callback_audio(event.inputBuffer); + + if(callback.callback_buffer) { + log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!")); + } + } + + current_state() : InputState { return this._state; }; + + private _start_promise: Promise; + async start() : Promise { + if(this._start_promise) { + try { + await this._start_promise; + if(this._state != InputState.PAUSED) + return; + } catch(error) { + log.debug(LogCategory.AUDIO, tr("JavascriptInput:start() Start promise await resulted in an error: %o"), error); + } + } + + return await (this._start_promise = this._start()); + } + + /* request permission for devices only one per time! */ + private static _running_request: Promise; + static async request_media_stream(device_id: string, group_id: string) : Promise { + while(this._running_request) { + try { + await this._running_request; + } catch(error) { } + } + + const audio_constrains: MediaTrackConstraints = {}; + if(window.detectedBrowser?.name === "firefox") { + /* + * Firefox only allows to open one mic as well deciding whats the input device it. + * It does not respect the deviceId nor the groupId + */ + } else { + audio_constrains.deviceId = device_id; + audio_constrains.groupId = group_id; + } + + audio_constrains.echoCancellation = true; + audio_constrains.autoGainControl = true; + audio_constrains.noiseSuppression = true; + + const promise = (this._running_request = requestMicrophoneMediaStream(audio_constrains, true)); + try { + return await this._running_request; + } finally { + if(this._running_request === promise) + this._running_request = undefined; + } + } + + private async _start() : Promise { + try { + if(this._state != InputState.PAUSED) + throw tr("recorder already started"); + + this._state = InputState.INITIALIZING; + if(!this._current_device) + throw tr("invalid device"); + + if(!this._audio_context) { + debugger; + throw tr("missing audio context"); + } + + const _result = await JavascriptInput.request_media_stream(this._current_device.deviceId, this._current_device.groupId); + if(!(_result instanceof MediaStream)) { + this._state = InputState.PAUSED; + return _result; + } + this._current_stream = _result; + + for(const f of this.registeredFilters) { + if(f.is_enabled()) { + f.set_pause(false); + } + } + this._consumer_callback_node.addEventListener('audioprocess', this._consumer_audio_callback); + + this._current_audio_stream = this._audio_context.createMediaStreamSource(this._current_stream); + this._current_audio_stream.connect(this._volume_node); + this._state = InputState.RECORDING; + return InputStartResult.EOK; + } catch(error) { + if(this._state == InputState.INITIALIZING) { + this._state = InputState.PAUSED; + } + throw error; + } finally { + this._start_promise = undefined; + } + } + + async stop() { + /* await all starts */ + try { + if(this._start_promise) + await this._start_promise; + } catch(error) {} + + this._state = InputState.PAUSED; + if(this._current_audio_stream) { + this._current_audio_stream.disconnect(); + } + + if(this._current_stream) { + if(this._current_stream.stop) { + this._current_stream.stop(); + } else { + this._current_stream.getTracks().forEach(value => { + value.stop(); + }); + } + } + + this._current_stream = undefined; + this._current_audio_stream = undefined; + for(const f of this.registeredFilters) { + if(f.is_enabled()) { + f.set_pause(true); + } + } + + if(this._consumer_callback_node) { + this._consumer_callback_node.removeEventListener('audioprocess', this._consumer_audio_callback); + } + return undefined; + } + + + current_device(): IDevice | undefined { + return this._current_device; + } + + async set_device(device: IDevice | undefined) { + if(this._current_device === device) + return; + + const savedState = this._state; + try { + await this.stop(); + } catch(error) { + log.warn(LogCategory.AUDIO, tr("Failed to stop previous record session (%o)"), error); + } + + this._current_device = device as any; + if(!device) { + this._state = savedState === InputState.PAUSED ? InputState.PAUSED : InputState.DRY; + return; + } + + if(savedState !== InputState.PAUSED) { + try { + await this.start() + } catch(error) { + log.warn(LogCategory.AUDIO, tr("Failed to start new recording stream (%o)"), error); + throw "failed to start record"; + } + } + return; + } + + + createFilter(type: T, priority: number): FilterTypeClass { + let filter: JAbstractFilter & Filter; + switch (type) { + case FilterType.STATE: + filter = new JStateFilter(priority); + break; + + case FilterType.THRESHOLD: + filter = new JThresholdFilter(priority); + break; + + case FilterType.VOICE_LEVEL: + throw tr("voice filter isn't supported!"); + + default: + throw tr("unknown filter type"); + } + + filter.callback_active_change = () => this._recalculate_filter_status(); + this.registeredFilters.push(filter); + this.initializeFilters(); + this._recalculate_filter_status(); + return filter as any; + } + + supportsFilter(type: FilterType): boolean { + switch (type) { + case FilterType.THRESHOLD: + case FilterType.STATE: + return true; + default: + return false; + } + } + + resetFilter() { + for(const filter of this.registeredFilters) { + filter.finalize(); + filter.enabled = false; + } + + this.registeredFilters = []; + this.initializeFilters(); + this._recalculate_filter_status(); + } + + removeFilter(filterInstance: Filter) { + const index = this.registeredFilters.indexOf(filterInstance as any); + if(index === -1) return; + + const [ filter ] = this.registeredFilters.splice(index, 1); + filter.finalize(); + filter.enabled = false; + + this.initializeFilters(); + this._recalculate_filter_status(); + } + + private _recalculate_filter_status() { + let filtered = this.registeredFilters.filter(e => e.is_enabled()).filter(e => (e as JAbstractFilter).active).length > 0; + if(filtered === this._filter_active) + return; + + this._filter_active = filtered; + if(filtered) { + if(this.callback_end) + this.callback_end(); + } else { + if(this.callback_begin) + this.callback_begin(); + } + } + + current_consumer(): InputConsumer | undefined { + return this._current_consumer; + } + + async set_consumer(consumer: InputConsumer) { + if(this._current_consumer) { + if(this._current_consumer.type == InputConsumerType.NODE) { + if(this._source_node) + (this._current_consumer as NodeInputConsumer).callback_disconnect(this._source_node) + } else if(this._current_consumer.type === InputConsumerType.CALLBACK) { + if(this._source_node) + this._source_node.disconnect(this._consumer_callback_node); + } + } + + if(consumer) { + if(consumer.type == InputConsumerType.CALLBACK) { + if(this._source_node) + this._source_node.connect(this._consumer_callback_node); + } else if(consumer.type == InputConsumerType.NODE) { + if(this._source_node) + (consumer as NodeInputConsumer).callback_node(this._source_node); + } else { + throw "native callback consumers are not supported!"; + } + } + this._current_consumer = consumer; + } + + private _switch_source_node(new_node: AudioNode) { + if(this._current_consumer) { + if(this._current_consumer.type == InputConsumerType.NODE) { + const node_consumer = this._current_consumer as NodeInputConsumer; + if(this._source_node) + node_consumer.callback_disconnect(this._source_node); + if(new_node) + node_consumer.callback_node(new_node); + } else if(this._current_consumer.type == InputConsumerType.CALLBACK) { + this._source_node.disconnect(this._consumer_callback_node); + if(new_node) + new_node.connect(this._consumer_callback_node); + } + } + this._source_node = new_node; + } + + get_volume(): number { + return this._volume; + } + + set_volume(volume: number) { + if(volume === this._volume) + return; + this._volume = volume; + this._volume_node.gain.value = volume; + } +} + +class JavascriptLevelmeter implements LevelMeter { + private static _instances: JavascriptLevelmeter[] = []; + private static _update_task: number; + + readonly _device: WebIDevice; + + private _callback: (num: number) => any; + + private _context: AudioContext; + private _gain_node: GainNode; + private _source_node: MediaStreamAudioSourceNode; + private _analyser_node: AnalyserNode; + + private _media_stream: MediaStream; + + private _analyse_buffer: Uint8Array; + + private _current_level = 0; + + constructor(device: WebIDevice) { + this._device = device; + } + + async initialize() { + try { + await new Promise((resolve, reject) => { + const timeout = setTimeout(reject, 5000); + aplayer.on_ready(() => { + clearTimeout(timeout); + resolve(); + }); + }); + } catch(error) { + throw tr("audio context timeout"); + } + this._context = aplayer.context(); + if(!this._context) throw tr("invalid context"); + + this._gain_node = this._context.createGain(); + this._gain_node.gain.setValueAtTime(0, 0); + + /* analyser node */ + this._analyser_node = this._context.createAnalyser(); + + const optimal_ftt_size = Math.ceil(this._context.sampleRate * (JThresholdFilter.update_task_interval / 1000)); + this._analyser_node.fftSize = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); + + if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser_node.fftSize) + this._analyse_buffer = new Uint8Array(this._analyser_node.fftSize); + + /* starting stream */ + const _result = await JavascriptInput.request_media_stream(this._device.deviceId, this._device.groupId); + if(!(_result instanceof MediaStream)){ + if(_result === InputStartResult.ENOTALLOWED) + throw tr("No permissions"); + if(_result === InputStartResult.ENOTSUPPORTED) + throw tr("Not supported"); + if(_result === InputStartResult.EBUSY) + throw tr("Device busy"); + if(_result === InputStartResult.EUNKNOWN) + throw tr("an error occurred"); + throw _result; + } + this._media_stream = _result; + + this._source_node = this._context.createMediaStreamSource(this._media_stream); + this._source_node.connect(this._analyser_node); + this._analyser_node.connect(this._gain_node); + this._gain_node.connect(this._context.destination); + + JavascriptLevelmeter._instances.push(this); + if(JavascriptLevelmeter._instances.length == 1) { + clearInterval(JavascriptLevelmeter._update_task); + JavascriptLevelmeter._update_task = setInterval(() => JavascriptLevelmeter._analyse_all(), JThresholdFilter.update_task_interval) as any; + } + } + + destroy() { + JavascriptLevelmeter._instances.remove(this); + if(JavascriptLevelmeter._instances.length == 0) { + clearInterval(JavascriptLevelmeter._update_task); + JavascriptLevelmeter._update_task = 0; + } + + if(this._source_node) { + this._source_node.disconnect(); + this._source_node = undefined; + } + if(this._media_stream) { + if(this._media_stream.stop) + this._media_stream.stop(); + else + this._media_stream.getTracks().forEach(value => { + value.stop(); + }); + this._media_stream = undefined; + } + if(this._gain_node) { + this._gain_node.disconnect(); + this._gain_node = undefined; + } + if(this._analyser_node) { + this._analyser_node.disconnect(); + this._analyser_node = undefined; + } + } + + device(): IDevice { + return this._device; + } + + set_observer(callback: (value: number) => any) { + this._callback = callback; + } + + private static _analyse_all() { + for(const instance of [...this._instances]) + instance._analyse(); + } + + private _analyse() { + this._analyser_node.getByteTimeDomainData(this._analyse_buffer); + + this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser_node.fftSize, this._current_level, .75); + if(this._callback) + this._callback(this._current_level); + } +} + +loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { + function: async () => { + inputDeviceList = new WebInputDeviceList(); + }, + priority: 80, + name: "initialize media devices" +}); + +loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { + function: async () => { + inputDeviceList.refresh().then(() => {}); + }, + priority: 10, + name: "query media devices" +}); \ No newline at end of file diff --git a/web/app/audio/RecorderFilter.ts b/web/app/audio/RecorderFilter.ts new file mode 100644 index 00000000..cde5ffbc --- /dev/null +++ b/web/app/audio/RecorderFilter.ts @@ -0,0 +1,242 @@ +import {FilterType, StateFilter, ThresholdFilter} from "tc-shared/voice/Filter"; + +export abstract class JAbstractFilter { + readonly priority: number; + + source_node: AudioNode; + audio_node: NodeType; + + context: AudioContext; + enabled: boolean = false; + + active: boolean = false; /* if true the filter filters! */ + callback_active_change: (new_state: boolean) => any; + + paused: boolean = true; + + constructor(priority: number) { + this.priority = priority; + } + + abstract initialize(context: AudioContext, source_node: AudioNode); + abstract finalize(); + + /* whatever the input has been paused and we don't expect any input */ + abstract set_pause(flag: boolean); + + is_enabled(): boolean { + return this.enabled; + } + + set_enabled(flag: boolean) { + this.enabled = flag; + } +} + +export class JThresholdFilter extends JAbstractFilter implements ThresholdFilter { + public static update_task_interval = 20; /* 20ms */ + + readonly type = FilterType.THRESHOLD; + callback_level?: (value: number) => any; + + private _threshold = 50; + + private _update_task: any; + private _analyser: AnalyserNode; + private _analyse_buffer: Uint8Array; + + private _silence_count = 0; + private _margin_frames = 5; + + private _current_level = 0; + private _smooth_release = 0; + private _smooth_attack = 0; + + finalize() { + this.set_pause(true); + + if(this.source_node) { + try { this.source_node.disconnect(this._analyser) } catch (error) {} + try { this.source_node.disconnect(this.audio_node) } catch (error) {} + } + + this._analyser = undefined; + this.source_node = undefined; + this.audio_node = undefined; + this.context = undefined; + } + + initialize(context: AudioContext, source_node: AudioNode) { + this.context = context; + this.source_node = source_node; + + this.audio_node = context.createGain(); + this._analyser = context.createAnalyser(); + + const optimal_ftt_size = Math.ceil((source_node.context || context).sampleRate * (JThresholdFilter.update_task_interval / 1000)); + const base2_ftt = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); + this._analyser.fftSize = base2_ftt; + + if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser.fftSize) + this._analyse_buffer = new Uint8Array(this._analyser.fftSize); + + this.active = false; + this.audio_node.gain.value = 1; + + this.source_node.connect(this.audio_node); + this.source_node.connect(this._analyser); + + /* force update paused state */ + this.set_pause(!(this.paused = !this.paused)); + } + + get_margin_frames(): number { return this._margin_frames; } + set_margin_frames(value: number) { + this._margin_frames = value; + } + + get_attack_smooth(): number { + return this._smooth_attack; + } + + get_release_smooth(): number { + return this._smooth_release; + } + + set_attack_smooth(value: number) { + this._smooth_attack = value; + } + + set_release_smooth(value: number) { + this._smooth_release = value; + } + + get_threshold(): number { + return this._threshold; + } + + set_threshold(value: number): Promise { + this._threshold = value; + return Promise.resolve(); + } + + public static process(buffer: Uint8Array, ftt_size: number, previous: number, smooth: number) { + let level; + { + let total = 0, float, rms; + + for(let index = 0; index < ftt_size; index++) { + float = ( buffer[index++] / 0x7f ) - 1; + total += (float * float); + } + rms = Math.sqrt(total / ftt_size); + let db = 20 * ( Math.log(rms) / Math.log(10) ); + // sanity check + + db = Math.max(-192, Math.min(db, 0)); + level = 100 + ( db * 1.92 ); + } + + return previous * smooth + level * (1 - smooth); + } + + private _analyse() { + this._analyser.getByteTimeDomainData(this._analyse_buffer); + + let smooth; + if(this._silence_count == 0) + smooth = this._smooth_release; + else + smooth = this._smooth_attack; + + this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser.fftSize, this._current_level, smooth); + + this._update_gain_node(); + if(this.callback_level) + this.callback_level(this._current_level); + } + + private _update_gain_node() { + let state; + if(this._current_level > this._threshold) { + this._silence_count = 0; + state = true; + } else { + state = this._silence_count++ < this._margin_frames; + } + if(state) { + this.audio_node.gain.value = 1; + if(this.active) { + this.active = false; + this.callback_active_change(false); + } + } else { + this.audio_node.gain.value = 0; + if(!this.active) { + this.active = true; + this.callback_active_change(true); + } + } + } + + set_pause(flag: boolean) { + if(flag === this.paused) return; + this.paused = flag; + + if(this.paused) { + clearInterval(this._update_task); + this._update_task = undefined; + + if(this.active) { + this.active = false; + this.callback_active_change(false); + } + } else { + if(!this._update_task && this._analyser) + this._update_task = setInterval(() => this._analyse(), JThresholdFilter.update_task_interval); + } + } +} + +export class JStateFilter extends JAbstractFilter implements StateFilter { + public readonly type = FilterType.STATE; + + finalize() { + if(this.source_node) { + try { this.source_node.disconnect(this.audio_node) } catch (error) {} + } + + this.source_node = undefined; + this.audio_node = undefined; + this.context = undefined; + } + + initialize(context: AudioContext, source_node: AudioNode) { + this.context = context; + this.source_node = source_node; + + this.audio_node = context.createGain(); + this.audio_node.gain.value = this.active ? 0 : 1; + + this.source_node.connect(this.audio_node); + } + + is_active(): boolean { + return this.active; + } + + set_state(state: boolean): Promise { + if(this.active === state) + return Promise.resolve(); + + this.active = state; + if(this.audio_node) + this.audio_node.gain.value = state ? 0 : 1; + this.callback_active_change(state); + return Promise.resolve(); + } + + set_pause(flag: boolean) { + this.paused = flag; + } +} \ No newline at end of file diff --git a/web/app/audio/player.ts b/web/app/audio/player.ts index 684524b2..e8cca985 100644 --- a/web/app/audio/player.ts +++ b/web/app/audio/player.ts @@ -1,25 +1,3 @@ -/* -import {Device} from "tc-shared/audio/player"; - -export function initialize() : boolean; -export function initialized() : boolean; - -export function context() : AudioContext; -export function get_master_volume() : number; -export function set_master_volume(volume: number); - -export function destination() : AudioNode; - -export function on_ready(cb: () => any); - -export function available_devices() : Promise; -export function set_device(device_id: string) : Promise; - -export function current_device() : Device; - -export function initializeFromGesture(); -*/ - import {Device} from "tc-shared/audio/player"; import * as log from "tc-shared/log"; import {LogCategory} from "tc-shared/log"; @@ -52,6 +30,10 @@ function fire_initialized() { function createNewContext() { audioContextInstance = new (window.webkitAudioContext || window.AudioContext)(); + audioContextInstance.onstatechange = () => { + if(audioContextInstance.state === "running") + fire_initialized(); + }; audioContextInitializeCallbacks.unshift(() => { globalAudioGainInstance = audioContextInstance.createGain(); @@ -128,9 +110,7 @@ export function current_device() : Device { export function initializeFromGesture() { if(audioContextInstance) { if(audioContextInstance.state !== "running") { - audioContextInstance.resume().then(() => { - fire_initialized(); - }).catch(error => { + audioContextInstance.resume().catch(error => { log.error(LogCategory.AUDIO, tr("Failed to initialize audio context instance from gesture: %o"), error); }); } diff --git a/web/app/audio/recorder.ts b/web/app/audio/recorder.ts deleted file mode 100644 index 66375a9a..00000000 --- a/web/app/audio/recorder.ts +++ /dev/null @@ -1,877 +0,0 @@ -import { - AbstractInput, CallbackInputConsumer, - InputConsumer, - InputConsumerType, - InputDevice, InputStartResult, - InputState, - LevelMeter, NodeInputConsumer -} from "tc-shared/voice/RecorderBase"; -import * as log from "tc-shared/log"; -import * as loader from "tc-loader"; -import {LogCategory} from "tc-shared/log"; -import * as aplayer from "./player"; -import * as rbase from "tc-shared/voice/RecorderBase"; - -declare global { - interface MediaStream { - stop(); - } -} - -let _queried_devices: JavascriptInputDevice[]; -let _queried_permissioned: boolean = false; - -export interface JavascriptInputDevice extends InputDevice { - device_id: string; - group_id: string; -} - -function getUserMediaFunctionPromise() : (constraints: MediaStreamConstraints) => Promise { - if('mediaDevices' in navigator && 'getUserMedia' in navigator.mediaDevices) - return constraints => navigator.mediaDevices.getUserMedia(constraints); - - const _callbacked_function = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia; - if(!_callbacked_function) - return undefined; - - return constraints => new Promise((resolve, reject) => _callbacked_function(constraints, resolve, reject)); -} - -async function query_devices() { - const general_supported = !!getUserMediaFunctionPromise(); - - try { - const context = aplayer.context(); - const devices = await navigator.mediaDevices.enumerateDevices(); - - _queried_permissioned = false; - if(devices.filter(e => !!e.label).length > 0) - _queried_permissioned = true; - - _queried_devices = devices.filter(e => e.kind === "audioinput").map((e: MediaDeviceInfo): JavascriptInputDevice => { - return { - channels: context ? context.destination.channelCount : 2, - sample_rate: context ? context.sampleRate : 44100, - - default_input: e.deviceId == "default", - - driver: "WebAudio", - name: e.label || "device-id{" + e.deviceId+ "}", - - supported: general_supported, - - device_id: e.deviceId, - group_id: e.groupId, - - unique_id: e.deviceId - } - }); - if(_queried_devices.length > 0 && _queried_devices.filter(e => e.default_input).length == 0) - _queried_devices[0].default_input = true; - } catch(error) { - log.error(LogCategory.AUDIO, tr("Failed to query microphone devices (%o)"), error); - _queried_devices = []; - } -} - -export function devices() : InputDevice[] { - if(typeof(_queried_devices) === "undefined") - query_devices(); - - return _queried_devices || []; -} - - -export function device_refresh_available() : boolean { return true; } -export function refresh_devices() : Promise { return query_devices(); } - -export function create_input() : AbstractInput { return new JavascriptInput(); } - -export async function create_levelmeter(device: InputDevice) : Promise { - const meter = new JavascriptLevelmeter(device as any); - await meter.initialize(); - return meter; -} - -loader.register_task(loader.Stage.JAVASCRIPT_INITIALIZING, { - function: async () => { query_devices(); }, /* May wait for it? */ - priority: 10, - name: "query media devices" -}); - -export namespace filter { - export abstract class JAbstractFilter implements rbase.filter.Filter { - type; - - source_node: AudioNode; - audio_node: NodeType; - - context: AudioContext; - enabled: boolean = false; - - active: boolean = false; /* if true the filter filters! */ - callback_active_change: (new_state: boolean) => any; - - paused: boolean = true; - - abstract initialize(context: AudioContext, source_node: AudioNode); - abstract finalize(); - - /* whatever the input has been paused and we don't expect any input */ - abstract set_pause(flag: boolean); - - is_enabled(): boolean { - return this.enabled; - } - } - - export class JThresholdFilter extends JAbstractFilter implements rbase.filter.ThresholdFilter { - public static update_task_interval = 20; /* 20ms */ - - type = rbase.filter.Type.THRESHOLD; - callback_level?: (value: number) => any; - - private _threshold = 50; - - private _update_task: any; - private _analyser: AnalyserNode; - private _analyse_buffer: Uint8Array; - - private _silence_count = 0; - private _margin_frames = 5; - - private _current_level = 0; - private _smooth_release = 0; - private _smooth_attack = 0; - - finalize() { - this.set_pause(true); - - if(this.source_node) { - try { this.source_node.disconnect(this._analyser) } catch (error) {} - try { this.source_node.disconnect(this.audio_node) } catch (error) {} - } - - this._analyser = undefined; - this.source_node = undefined; - this.audio_node = undefined; - this.context = undefined; - } - - initialize(context: AudioContext, source_node: AudioNode) { - this.context = context; - this.source_node = source_node; - - this.audio_node = context.createGain(); - this._analyser = context.createAnalyser(); - - const optimal_ftt_size = Math.ceil((source_node.context || context).sampleRate * (JThresholdFilter.update_task_interval / 1000)); - const base2_ftt = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); - this._analyser.fftSize = base2_ftt; - - if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser.fftSize) - this._analyse_buffer = new Uint8Array(this._analyser.fftSize); - - this.active = false; - this.audio_node.gain.value = 1; - - this.source_node.connect(this.audio_node); - this.source_node.connect(this._analyser); - - /* force update paused state */ - this.set_pause(!(this.paused = !this.paused)); - } - - get_margin_frames(): number { return this._margin_frames; } - set_margin_frames(value: number) { - this._margin_frames = value; - } - - get_attack_smooth(): number { - return this._smooth_attack; - } - - get_release_smooth(): number { - return this._smooth_release; - } - - set_attack_smooth(value: number) { - this._smooth_attack = value; - } - - set_release_smooth(value: number) { - this._smooth_release = value; - } - - get_threshold(): number { - return this._threshold; - } - - set_threshold(value: number): Promise { - this._threshold = value; - return Promise.resolve(); - } - - public static process(buffer: Uint8Array, ftt_size: number, previous: number, smooth: number) { - let level; - { - let total = 0, float, rms; - - for(let index = 0; index < ftt_size; index++) { - float = ( buffer[index++] / 0x7f ) - 1; - total += (float * float); - } - rms = Math.sqrt(total / ftt_size); - let db = 20 * ( Math.log(rms) / Math.log(10) ); - // sanity check - - db = Math.max(-192, Math.min(db, 0)); - level = 100 + ( db * 1.92 ); - } - - return previous * smooth + level * (1 - smooth); - } - - private _analyse() { - this._analyser.getByteTimeDomainData(this._analyse_buffer); - - let smooth; - if(this._silence_count == 0) - smooth = this._smooth_release; - else - smooth = this._smooth_attack; - - this._current_level = JThresholdFilter.process(this._analyse_buffer, this._analyser.fftSize, this._current_level, smooth); - - this._update_gain_node(); - if(this.callback_level) - this.callback_level(this._current_level); - } - - private _update_gain_node() { - let state; - if(this._current_level > this._threshold) { - this._silence_count = 0; - state = true; - } else { - state = this._silence_count++ < this._margin_frames; - } - if(state) { - this.audio_node.gain.value = 1; - if(this.active) { - this.active = false; - this.callback_active_change(false); - } - } else { - this.audio_node.gain.value = 0; - if(!this.active) { - this.active = true; - this.callback_active_change(true); - } - } - } - - set_pause(flag: boolean) { - if(flag === this.paused) return; - this.paused = flag; - - if(this.paused) { - clearInterval(this._update_task); - this._update_task = undefined; - - if(this.active) { - this.active = false; - this.callback_active_change(false); - } - } else { - if(!this._update_task && this._analyser) - this._update_task = setInterval(() => this._analyse(), JThresholdFilter.update_task_interval); - } - } - } - - export class JStateFilter extends JAbstractFilter implements rbase.filter.StateFilter { - type = rbase.filter.Type.STATE; - - finalize() { - if(this.source_node) { - try { this.source_node.disconnect(this.audio_node) } catch (error) {} - } - - this.source_node = undefined; - this.audio_node = undefined; - this.context = undefined; - } - - initialize(context: AudioContext, source_node: AudioNode) { - this.context = context; - this.source_node = source_node; - - this.audio_node = context.createGain(); - this.audio_node.gain.value = this.active ? 0 : 1; - - this.source_node.connect(this.audio_node); - } - - is_active(): boolean { - return this.active; - } - - set_state(state: boolean): Promise { - if(this.active === state) - return Promise.resolve(); - - this.active = state; - if(this.audio_node) - this.audio_node.gain.value = state ? 0 : 1; - this.callback_active_change(state); - return Promise.resolve(); - } - - set_pause(flag: boolean) { - this.paused = flag; - } - } -} - -class JavascriptInput implements AbstractInput { - private _state: InputState = InputState.PAUSED; - private _current_device: JavascriptInputDevice | undefined; - private _current_consumer: InputConsumer; - - private _current_stream: MediaStream; - private _current_audio_stream: MediaStreamAudioSourceNode; - - private _audio_context: AudioContext; - private _source_node: AudioNode; /* last node which could be connected to the target; target might be the _consumer_node */ - private _consumer_callback_node: ScriptProcessorNode; - private readonly _consumer_audio_callback; - private _volume_node: GainNode; - private _mute_node: GainNode; - - private _filters: rbase.filter.Filter[] = []; - private _filter_active: boolean = false; - - private _volume: number = 1; - - callback_begin: () => any = undefined; - callback_end: () => any = undefined; - - constructor() { - aplayer.on_ready(() => this._audio_initialized()); - this._consumer_audio_callback = this._audio_callback.bind(this); - } - - private _audio_initialized() { - this._audio_context = aplayer.context(); - if(!this._audio_context) - return; - - this._mute_node = this._audio_context.createGain(); - this._mute_node.gain.value = 0; - this._mute_node.connect(this._audio_context.destination); - - this._consumer_callback_node = this._audio_context.createScriptProcessor(1024 * 4); - this._consumer_callback_node.connect(this._mute_node); - - this._volume_node = this._audio_context.createGain(); - this._volume_node.gain.value = this._volume; - - this._initialize_filters(); - if(this._state === InputState.INITIALIZING) - this.start(); - } - - private _initialize_filters() { - const filters = this._filters as any as filter.JAbstractFilter[]; - for(const filter of filters) { - if(filter.is_enabled()) - filter.finalize(); - } - - if(this._audio_context && this._volume_node) { - const active_filter = filters.filter(e => e.is_enabled()); - let stream: AudioNode = this._volume_node; - for(const f of active_filter) { - f.initialize(this._audio_context, stream); - stream = f.audio_node; - } - this._switch_source_node(stream); - } - } - - private _audio_callback(event: AudioProcessingEvent) { - if(!this._current_consumer || this._current_consumer.type !== InputConsumerType.CALLBACK) - return; - - const callback = this._current_consumer as CallbackInputConsumer; - if(callback.callback_audio) - callback.callback_audio(event.inputBuffer); - - if(callback.callback_buffer) { - log.warn(LogCategory.AUDIO, tr("AudioInput has callback buffer, but this isn't supported yet!")); - } - } - - current_state() : InputState { return this._state; }; - - private _start_promise: Promise; - async start() : Promise { - if(this._start_promise) { - try { - await this._start_promise; - if(this._state != InputState.PAUSED) - return; - } catch(error) { - log.debug(LogCategory.AUDIO, tr("JavascriptInput:start() Start promise await resulted in an error: %o"), error); - } - } - - return await (this._start_promise = this._start()); - } - - /* request permission for devices only one per time! */ - private static _running_request: Promise; - static async request_media_stream(device_id: string, group_id: string) : Promise { - while(this._running_request) { - try { - await this._running_request; - } catch(error) { } - } - const promise = (this._running_request = this.request_media_stream0(device_id, group_id)); - try { - return await this._running_request; - } finally { - if(this._running_request === promise) - this._running_request = undefined; - } - } - - static async request_media_stream0(device_id: string, group_id: string) : Promise { - const media_function = getUserMediaFunctionPromise(); - if(!media_function) return InputStartResult.ENOTSUPPORTED; - - try { - log.info(LogCategory.AUDIO, tr("Requesting a microphone stream for device %s in group %s"), device_id, group_id); - - const audio_constrains: MediaTrackConstraints = {}; - audio_constrains.deviceId = device_id; - audio_constrains.groupId = group_id; - - audio_constrains.echoCancellation = true; - audio_constrains.autoGainControl = true; - audio_constrains.noiseSuppression = true; - /* disabled because most the time we get a OverconstrainedError */ //audio_constrains.sampleSize = {min: 420, max: 960 * 10, ideal: 960}; - - const stream = await media_function({ - audio: audio_constrains, - video: undefined - }); - if(!_queried_permissioned) query_devices(); /* we now got permissions, requery devices */ - return stream; - } catch(error) { - if('name' in error) { - if(error.name === "NotAllowedError") { - //createErrorModal(tr("Failed to create microphone"), tr("Microphone recording failed. Please allow TeaWeb access to your microphone")).open(); - //FIXME: Move this to somewhere else! - - log.warn(LogCategory.AUDIO, tr("Microphone request failed (No permissions). Browser message: %o"), error.message); - return InputStartResult.ENOTALLOWED; - } else { - log.warn(LogCategory.AUDIO, tr("Microphone request failed. Request resulted in error: %o: %o"), error.name, error); - } - } else { - log.warn(LogCategory.AUDIO, tr("Failed to initialize recording stream (%o)"), error); - } - return InputStartResult.EUNKNOWN; - } - } - - private async _start() : Promise { - try { - if(this._state != InputState.PAUSED) - throw tr("recorder already started"); - - this._state = InputState.INITIALIZING; - if(!this._current_device) - throw tr("invalid device"); - - if(!this._audio_context) { - debugger; - throw tr("missing audio context"); - } - - const _result = await JavascriptInput.request_media_stream(this._current_device.device_id, this._current_device.group_id); - if(!(_result instanceof MediaStream)) { - this._state = InputState.PAUSED; - return _result; - } - this._current_stream = _result; - - for(const f of this._filters) - if(f.is_enabled() && f instanceof filter.JAbstractFilter) - f.set_pause(false); - this._consumer_callback_node.addEventListener('audioprocess', this._consumer_audio_callback); - - this._current_audio_stream = this._audio_context.createMediaStreamSource(this._current_stream); - this._current_audio_stream.connect(this._volume_node); - this._state = InputState.RECORDING; - return InputStartResult.EOK; - } catch(error) { - if(this._state == InputState.INITIALIZING) { - this._state = InputState.PAUSED; - } - throw error; - } finally { - this._start_promise = undefined; - } - } - - async stop() { - /* await all starts */ - try { - if(this._start_promise) - await this._start_promise; - } catch(error) {} - - this._state = InputState.PAUSED; - if(this._current_audio_stream) - this._current_audio_stream.disconnect(); - - if(this._current_stream) { - if(this._current_stream.stop) - this._current_stream.stop(); - else - this._current_stream.getTracks().forEach(value => { - value.stop(); - }); - } - - this._current_stream = undefined; - this._current_audio_stream = undefined; - for(const f of this._filters) - if(f.is_enabled() && f instanceof filter.JAbstractFilter) - f.set_pause(true); - if(this._consumer_callback_node) - this._consumer_callback_node.removeEventListener('audioprocess', this._consumer_audio_callback); - return undefined; - } - - - current_device(): InputDevice | undefined { - return this._current_device; - } - - async set_device(device: InputDevice | undefined) { - if(this._current_device === device) - return; - - - const saved_state = this._state; - try { - await this.stop(); - } catch(error) { - log.warn(LogCategory.AUDIO, tr("Failed to stop previous record session (%o)"), error); - } - - this._current_device = device as any; /* TODO: Test for device_id and device_group */ - if(!device) { - this._state = saved_state === InputState.PAUSED ? InputState.PAUSED : InputState.DRY; - return; - } - - if(saved_state !== InputState.PAUSED) { - try { - await this.start() - } catch(error) { - log.warn(LogCategory.AUDIO, tr("Failed to start new recording stream (%o)"), error); - throw "failed to start record"; - } - } - return; - } - - - get_filter(type: rbase.filter.Type): rbase.filter.Filter | undefined { - for(const filter of this._filters) - if(filter.type == type) - return filter; - - let new_filter: filter.JAbstractFilter; - switch (type) { - case rbase.filter.Type.STATE: - new_filter = new filter.JStateFilter(); - break; - case rbase.filter.Type.VOICE_LEVEL: - throw "voice filter isn't supported!"; - case rbase.filter.Type.THRESHOLD: - new_filter = new filter.JThresholdFilter(); - break; - default: - throw "invalid filter type, or type isn't implemented! (" + type + ")"; - } - - new_filter.callback_active_change = () => this._recalculate_filter_status(); - this._filters.push(new_filter as any); - this.enable_filter(type); - return new_filter as any; - } - - supports_filter(type: rbase.filter.Type) : boolean { - switch (type) { - case rbase.filter.Type.THRESHOLD: - case rbase.filter.Type.STATE: - return true; - default: - return false; - } - } - - private find_filter(type: rbase.filter.Type) : filter.JAbstractFilter | undefined { - for(const filter of this._filters) - if(filter.type == type) - return filter as any; - return undefined; - } - - clear_filter() { - for(const _filter of this._filters) { - if(!_filter.is_enabled()) - continue; - - const c_filter = _filter as any as filter.JAbstractFilter; - c_filter.finalize(); - c_filter.enabled = false; - } - - this._initialize_filters(); - this._recalculate_filter_status(); - } - - disable_filter(type: rbase.filter.Type) { - const filter = this.find_filter(type); - if(!filter) return; - - /* test if the filter is active */ - if(!filter.is_enabled()) - return; - - filter.enabled = false; - filter.set_pause(true); - filter.finalize(); - this._initialize_filters(); - this._recalculate_filter_status(); - } - - enable_filter(type: rbase.filter.Type) { - const filter = this.get_filter(type) as any as filter.JAbstractFilter; - if(filter.is_enabled()) - return; - - filter.enabled = true; - filter.set_pause(typeof this._current_audio_stream !== "object"); - this._initialize_filters(); - this._recalculate_filter_status(); - } - - private _recalculate_filter_status() { - let filtered = this._filters.filter(e => e.is_enabled()).filter(e => (e as any as filter.JAbstractFilter).active).length > 0; - if(filtered === this._filter_active) - return; - - this._filter_active = filtered; - if(filtered) { - if(this.callback_end) - this.callback_end(); - } else { - if(this.callback_begin) - this.callback_begin(); - } - } - - current_consumer(): InputConsumer | undefined { - return this._current_consumer; - } - - async set_consumer(consumer: InputConsumer) { - if(this._current_consumer) { - if(this._current_consumer.type == InputConsumerType.NODE) { - if(this._source_node) - (this._current_consumer as NodeInputConsumer).callback_disconnect(this._source_node) - } else if(this._current_consumer.type === InputConsumerType.CALLBACK) { - if(this._source_node) - this._source_node.disconnect(this._consumer_callback_node); - } - } - - if(consumer) { - if(consumer.type == InputConsumerType.CALLBACK) { - if(this._source_node) - this._source_node.connect(this._consumer_callback_node); - } else if(consumer.type == InputConsumerType.NODE) { - if(this._source_node) - (consumer as NodeInputConsumer).callback_node(this._source_node); - } else { - throw "native callback consumers are not supported!"; - } - } - this._current_consumer = consumer; - } - - private _switch_source_node(new_node: AudioNode) { - if(this._current_consumer) { - if(this._current_consumer.type == InputConsumerType.NODE) { - const node_consumer = this._current_consumer as NodeInputConsumer; - if(this._source_node) - node_consumer.callback_disconnect(this._source_node); - if(new_node) - node_consumer.callback_node(new_node); - } else if(this._current_consumer.type == InputConsumerType.CALLBACK) { - this._source_node.disconnect(this._consumer_callback_node); - if(new_node) - new_node.connect(this._consumer_callback_node); - } - } - this._source_node = new_node; - } - - get_volume(): number { - return this._volume; - } - - set_volume(volume: number) { - if(volume === this._volume) - return; - this._volume = volume; - this._volume_node.gain.value = volume; - } -} - -class JavascriptLevelmeter implements LevelMeter { - private static _instances: JavascriptLevelmeter[] = []; - private static _update_task: number; - - readonly _device: JavascriptInputDevice; - - private _callback: (num: number) => any; - - private _context: AudioContext; - private _gain_node: GainNode; - private _source_node: MediaStreamAudioSourceNode; - private _analyser_node: AnalyserNode; - - private _media_stream: MediaStream; - - private _analyse_buffer: Uint8Array; - - private _current_level = 0; - - constructor(device: JavascriptInputDevice) { - this._device = device; - } - - async initialize() { - try { - await new Promise((resolve, reject) => { - const timeout = setTimeout(reject, 5000); - aplayer.on_ready(() => { - clearTimeout(timeout); - resolve(); - }); - }); - } catch(error) { - throw tr("audio context timeout"); - } - this._context = aplayer.context(); - if(!this._context) throw tr("invalid context"); - - this._gain_node = this._context.createGain(); - this._gain_node.gain.setValueAtTime(0, 0); - - /* analyser node */ - this._analyser_node = this._context.createAnalyser(); - - const optimal_ftt_size = Math.ceil(this._context.sampleRate * (filter.JThresholdFilter.update_task_interval / 1000)); - this._analyser_node.fftSize = Math.pow(2, Math.ceil(Math.log2(optimal_ftt_size))); - - if(!this._analyse_buffer || this._analyse_buffer.length < this._analyser_node.fftSize) - this._analyse_buffer = new Uint8Array(this._analyser_node.fftSize); - - /* starting stream */ - const _result = await JavascriptInput.request_media_stream(this._device.device_id, this._device.group_id); - if(!(_result instanceof MediaStream)){ - if(_result === InputStartResult.ENOTALLOWED) - throw tr("No permissions"); - if(_result === InputStartResult.ENOTSUPPORTED) - throw tr("Not supported"); - if(_result === InputStartResult.EBUSY) - throw tr("Device busy"); - if(_result === InputStartResult.EUNKNOWN) - throw tr("an error occurred"); - throw _result; - } - this._media_stream = _result; - - this._source_node = this._context.createMediaStreamSource(this._media_stream); - this._source_node.connect(this._analyser_node); - this._analyser_node.connect(this._gain_node); - this._gain_node.connect(this._context.destination); - - JavascriptLevelmeter._instances.push(this); - if(JavascriptLevelmeter._instances.length == 1) { - clearInterval(JavascriptLevelmeter._update_task); - JavascriptLevelmeter._update_task = setInterval(() => JavascriptLevelmeter._analyse_all(), filter.JThresholdFilter.update_task_interval) as any; - } - } - - destory() { - JavascriptLevelmeter._instances.remove(this); - if(JavascriptLevelmeter._instances.length == 0) { - clearInterval(JavascriptLevelmeter._update_task); - JavascriptLevelmeter._update_task = 0; - } - - if(this._source_node) { - this._source_node.disconnect(); - this._source_node = undefined; - } - if(this._media_stream) { - if(this._media_stream.stop) - this._media_stream.stop(); - else - this._media_stream.getTracks().forEach(value => { - value.stop(); - }); - this._media_stream = undefined; - } - if(this._gain_node) { - this._gain_node.disconnect(); - this._gain_node = undefined; - } - if(this._analyser_node) { - this._analyser_node.disconnect(); - this._analyser_node = undefined; - } - } - - device(): InputDevice { - return this._device; - } - - set_observer(callback: (value: number) => any) { - this._callback = callback; - } - - private static _analyse_all() { - for(const instance of [...this._instances]) - instance._analyse(); - } - - private _analyse() { - this._analyser_node.getByteTimeDomainData(this._analyse_buffer); - - this._current_level = filter.JThresholdFilter.process(this._analyse_buffer, this._analyser_node.fftSize, this._current_level, .75); - if(this._callback) - this._callback(this._current_level); - } -} \ No newline at end of file diff --git a/web/app/connection/ServerConnection.ts b/web/app/connection/ServerConnection.ts index 8cc4912c..1993b83c 100644 --- a/web/app/connection/ServerConnection.ts +++ b/web/app/connection/ServerConnection.ts @@ -19,7 +19,6 @@ import {EventType} from "tc-shared/ui/frames/log/Definitions"; import {WrappedWebSocket} from "tc-backend/web/connection/WrappedWebSocket"; import {AbstractVoiceConnection} from "tc-shared/connection/VoiceConnection"; import {DummyVoiceConnection} from "tc-shared/connection/DummyVoiceConnection"; -import {ServerConnectionFactory, setServerConnectionFactory} from "tc-shared/connection/ConnectionFactory"; class ReturnListener { resolve: (value?: T | PromiseLike) => void; diff --git a/web/app/hooks/AudioRecorder.ts b/web/app/hooks/AudioRecorder.ts new file mode 100644 index 00000000..592d56c1 --- /dev/null +++ b/web/app/hooks/AudioRecorder.ts @@ -0,0 +1,4 @@ +import {setRecorderBackend} from "tc-shared/audio/recorder"; +import {WebAudioRecorder} from "../audio/Recorder"; + +setRecorderBackend(new WebAudioRecorder()); \ No newline at end of file diff --git a/web/app/factories/ExternalModal.ts b/web/app/hooks/ExternalModal.ts similarity index 100% rename from web/app/factories/ExternalModal.ts rename to web/app/hooks/ExternalModal.ts diff --git a/web/app/factories/ServerConnection.ts b/web/app/hooks/ServerConnection.ts similarity index 100% rename from web/app/factories/ServerConnection.ts rename to web/app/hooks/ServerConnection.ts diff --git a/web/app/index.ts b/web/app/index.ts index 6d5bf472..4fbad1bb 100644 --- a/web/app/index.ts +++ b/web/app/index.ts @@ -2,7 +2,8 @@ import "webrtc-adapter"; import "./index.scss"; import "./FileTransfer"; -import "./factories/ServerConnection"; -import "./factories/ExternalModal"; +import "./hooks/ServerConnection"; +import "./hooks/ExternalModal"; +import "./hooks/AudioRecorder"; export = require("tc-shared/main"); \ No newline at end of file