TeaSpeak/TeaWeb
TeaSpeak/TeaWeb
1
0
Fork 0
Code Issues 1 Pull requests Activity

Updated the video codec, added screen sharing and added a lot of video configure options.

Browse source
This commit is contained in:
WolverinDEV 2020-11-22 13:48:15 +01:00
parent 9f98481057
commit 0bde320faa
23 changed files with 1252 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
shared/js/connection/VideoConnection.ts
View file

@ -5,6 +5,23 @@ import {ConnectionStatistics} from "tc-shared/connection/ConnectionBase";
export type VideoBroadcastType = "camera" | "screen";
export type VideoBroadcastStatistics = {
dimensions: { width: number, height: number },
frameRate: number,
codec: { name: string, payloadType: number }
maxBandwidth: number,
bandwidth: number,
qualityLimitation: "cpu" | "bandwidth",
source: {
frameRate: number,
dimensions: { width: number, height: number },
}
};
export interface VideoConnectionEvent {
notify_status_changed: { oldState: VideoConnectionStatus, newState: VideoConnectionStatus },
notify_local_broadcast_state_changed: { broadcastType: VideoBroadcastType, oldState: VideoBroadcastState, newState: VideoBroadcastState },
@ -53,6 +70,7 @@ export interface VideoConnection {
isBroadcasting(type: VideoBroadcastType);
getBroadcastingSource(type: VideoBroadcastType) : VideoSource | undefined;
getBroadcastingState(type: VideoBroadcastType) : VideoBroadcastState;
getBroadcastStatistics(type: VideoBroadcastType) : Promise<VideoBroadcastStatistics | undefined>;
/**
* @param type

40
shared/js/events/ClientGlobalControlHandler.ts
View file

@ -18,6 +18,7 @@ import {server_connections} from "tc-shared/ConnectionManager";
import {spawnAbout} from "tc-shared/ui/modal/ModalAbout";
import {spawnVideoSourceSelectModal} from "tc-shared/ui/modal/video-source/Controller";
import {LogCategory, logError} from "tc-shared/log";
import {getVideoDriver} from "tc-shared/video/VideoSource";
/*
function initialize_sounds(event_registry: Registry<ClientGlobalControlEvents>) {
@ -178,26 +179,29 @@ export function initialize(event_registry: Registry<ClientGlobalControlEvents>)
event_registry.on("action_toggle_video_broadcasting", event => {
if(event.enabled) {
if(event.broadcastType === "camera") {
spawnVideoSourceSelectModal().then(source => {
if(!source) { return; }
spawnVideoSourceSelectModal(event.broadcastType, true).then(async source => {
if(!source) { return; }
try {
event.connection.getServerConnection().getVideoConnection().startBroadcasting("camera", source)
.catch(error => {
logError(LogCategory.VIDEO, tr("Failed to start video broadcasting: %o"), error);
if(typeof error !== "string") {
error = tr("lookup the console for detail");
}
const videoTrack = source.getStream().getVideoTracks()[0];
try {
event.connection.getServerConnection().getVideoConnection().startBroadcasting("camera", source)
.catch(error => {
logError(LogCategory.VIDEO, tr("Failed to start %s broadcasting: %o"), event.broadcastType, error);
if(typeof error !== "string") {
error = tr("lookup the console for detail");
}
if(event.broadcastType === "camera") {
createErrorModal(tr("Failed to start video broadcasting"), tra("Failed to start video broadcasting:\n{}", error)).open();
});
} finally {
}
});
} else {
/* TODO! */
}
} else {
createErrorModal(tr("Failed to start screen sharing"), tra("Failed to start screen sharing:\n{}", error)).open();
}
});
} finally {
source.deref();
}
});
} else {
event.connection.getServerConnection().getVideoConnection().stopBroadcasting(event.broadcastType);
}

10
shared/js/main.tsx
View file

@ -247,16 +247,6 @@ function main() {
/* schedule it a bit later then the main because the main function is still within the loader */
setTimeout(() => {
(window as any).spawnVideo = async () => {
const source = await spawnVideoSourceSelectModal();
if(!source) { return; }
await server_connections.active_connection().getServerConnection().getVideoConnection().startBroadcasting("camera", source);
};
(window as any).videoDriver = getVideoDriver();
const connection = server_connections.active_connection();
//(window as any).spawnVideo();
/*
Modals.createChannelModal(connection, undefined, undefined, connection.permissions, (cb, perms) => {

1
shared/js/text/markdown.ts
View file

@ -20,6 +20,7 @@ export class MD2BBCodeRenderer {
"hardbreak": () => "\n",
"paragraph_open": (renderer: MD2BBCodeRenderer, token: ParagraphOpenToken) => {
debugger;
const last_line = !renderer.last_paragraph || !renderer.last_paragraph.lines ? 0 : renderer.last_paragraph.lines[1];
const lines = token.lines[0] - last_line;
return [...new Array(lines)].map(() => "[br]").join("");

39
shared/js/ui/frames/control-bar/Controller.ts
View file

@ -4,7 +4,7 @@ import {
ControlBarEvents,
ControlBarMode,
HostButtonInfo,
VideoCamaraState
VideoState
} from "tc-shared/ui/frames/control-bar/Definitions";
import {server_connections} from "tc-shared/ConnectionManager";
import {ConnectionHandler, ConnectionState} from "tc-shared/ConnectionHandler";
@ -22,7 +22,7 @@ import {
} from "tc-shared/bookmarks";
import {LogCategory, logWarn} from "tc-shared/log";
import {createErrorModal, createInputModal} from "tc-shared/ui/elements/Modal";
import {VideoBroadcastState, VideoConnectionStatus} from "tc-shared/connection/VideoConnection";
import {VideoBroadcastState, VideoBroadcastType, VideoConnectionStatus} from "tc-shared/connection/VideoConnection";
class InfoController {
private readonly mode: ControlBarMode;
@ -49,13 +49,15 @@ class InfoController {
this.registerGlobalHandlerEvents(event.handler);
this.sendConnectionState();
this.sendAwayState();
this.sendCamaraState();
this.sendVideoState("screen");
this.sendVideoState("camera");
}));
events.push(server_connections.events().on("notify_handler_deleted", event => {
this.unregisterGlobalHandlerEvents(event.handler);
this.sendConnectionState();
this.sendAwayState();
this.sendCamaraState();
this.sendVideoState("screen");
this.sendVideoState("camera");
}));
events.push(bookmarkEvents.on("notify_bookmarks_updated", () => this.sendBookmarks()));
@ -97,7 +99,8 @@ class InfoController {
events.push(handler.events().on("notify_connection_state_changed", event => {
if(event.old_state === ConnectionState.CONNECTED || event.new_state === ConnectionState.CONNECTED) {
this.sendHostButton();
this.sendCamaraState();
this.sendVideoState("screen");
this.sendVideoState("camera");
}
}));
@ -123,7 +126,8 @@ class InfoController {
const videoConnection = handler.getServerConnection().getVideoConnection();
events.push(videoConnection.getEvents().on(["notify_local_broadcast_state_changed", "notify_status_changed"], () => {
this.sendCamaraState();
this.sendVideoState("screen");
this.sendVideoState("camera");
}));
}
@ -149,7 +153,8 @@ class InfoController {
this.sendSubscribeState();
this.sendQueryState();
this.sendHostButton();
this.sendCamaraState();
this.sendVideoState("screen");
this.sendVideoState("camera");
}
public sendConnectionState() {
@ -241,26 +246,26 @@ class InfoController {
});
}
public sendCamaraState() {
let status: VideoCamaraState;
public sendVideoState(type: VideoBroadcastType) {
let state: VideoState;
if(this.currentHandler?.connected) {
const videoConnection = this.currentHandler.getServerConnection().getVideoConnection();
if(videoConnection.getStatus() === VideoConnectionStatus.Connected) {
if(videoConnection.getBroadcastingState("camera") === VideoBroadcastState.Running) {
status = "enabled";
if(videoConnection.getBroadcastingState(type) === VideoBroadcastState.Running) {
state = "enabled";
} else {
status = "disabled";
state = "disabled";
}
} else if(videoConnection.getStatus() === VideoConnectionStatus.Unsupported) {
status = "unsupported";
state = "unsupported";
} else {
status = "unavailable";
state = "unavailable";
}
} else {
status = "disconnected";
state = "disconnected";
}
this.events.fire_react("notify_camara_state", { state: status });
this.events.fire_react("notify_video_state", { state: state, broadcastType: type });
}
}
@ -287,7 +292,7 @@ export function initializeControlBarController(events: Registry<ControlBarEvents
events.on("query_speaker_state", () => infoHandler.sendSpeakerState());
events.on("query_subscribe_state", () => infoHandler.sendSubscribeState());
events.on("query_host_button", () => infoHandler.sendHostButton());
events.on("query_camara_state", () => infoHandler.sendCamaraState());
events.on("query_video_state", event => infoHandler.sendVideoState(event.broadcastType));
events.on("action_connection_connect", event => global_client_actions.fire("action_open_window_connect", { newTab: event.newTab }));
events.on("action_connection_disconnect", event => {

6
shared/js/ui/frames/control-bar/Definitions.ts
View file

@ -6,7 +6,7 @@ export type ConnectionState = { currentlyConnected: boolean, generallyConnected:
export type Bookmark = { uniqueId: string, label: string, icon: RemoteIconInfo | undefined, children?: Bookmark[] };
export type AwayState = { locallyAway: boolean, globallyAway: "partial" | "full" | "none" };
export type MicrophoneState = "enabled" | "disabled" | "muted";
export type VideoCamaraState = "enabled" | "disabled" | "unavailable" | "unsupported" | "disconnected";
export type VideoState = "enabled" | "disabled" | "unavailable" | "unsupported" | "disconnected";
export type HostButtonInfo = { title?: string, target?: string, url: string };
export interface ControlBarEvents {
@ -32,7 +32,7 @@ export interface ControlBarEvents {
query_subscribe_state: {},
query_query_state: {},
query_host_button: {},
query_camara_state: {},
query_video_state: { broadcastType: VideoBroadcastType },
notify_mode: { mode: ControlBarMode }
notify_connection_state: { state: ConnectionState },
@ -43,7 +43,7 @@ export interface ControlBarEvents {
notify_subscribe_state: { subscribe: boolean },
notify_query_state: { shown: boolean },
notify_host_button: { button: HostButtonInfo | undefined },
notify_camara_state: { state: VideoCamaraState },
notify_video_state: { broadcastType: VideoBroadcastType, state: VideoState },
notify_destroy: {}
}

57
shared/js/ui/frames/control-bar/Renderer.tsx
View file

@ -7,7 +7,7 @@ import {
ControlBarMode,
HostButtonInfo,
MicrophoneState,
VideoCamaraState
VideoState
} from "tc-shared/ui/frames/control-bar/Definitions";
import * as React from "react";
import {useContext, useRef, useState} from "react";
@ -17,6 +17,7 @@ import {Button} from "tc-shared/ui/frames/control-bar/Button";
import {spawnContextMenu} from "tc-shared/ui/ContextMenu";
import {ClientIcon} from "svg-sprites/client-icons";
import {createErrorModal} from "tc-shared/ui/elements/Modal";
import {VideoBroadcastType} from "tc-shared/connection/VideoConnection";
const cssStyle = require("./Renderer.scss");
const cssButtonStyle = require("./Button.scss");
@ -204,36 +205,51 @@ const AwayButton = () => {
);
};
const VideoButton = () => {
const VideoButton = (props: { type: VideoBroadcastType }) => {
const events = useContext(Events);
const [ state, setState ] = useState<VideoCamaraState>(() => {
events.fire("query_camara_state");
const [ state, setState ] = useState<VideoState>(() => {
events.fire("query_video_state", { broadcastType: props.type });
return "unsupported";
});
events.on("notify_camara_state", event => setState(event.state));
events.on("notify_video_state", event => event.broadcastType === props.type && setState(event.state));
let icon: ClientIcon = props.type === "camera" ? ClientIcon.VideoMuted : ClientIcon.ShareScreen;
switch (state) {
case "unsupported":
return <Button switched={true} colorTheme={"red"} autoSwitch={false} iconNormal={ClientIcon.VideoMuted} tooltip={tr("Video broadcasting not supported")} key={"unsupported"}
onToggle={() => createErrorModal(tr("Video broadcasting unsupported"), tr("Video broadcasting isn't supported by the target server.")).open()}
case "unsupported": {
let tooltip = props.type === "camera" ? tr("Video broadcasting not supported") : tr("Screen sharing not supported");
let modalTitle = props.type === "camera" ? tr("Video broadcasting unsupported") : tr("Screen sharing unsupported");
let modalBody = props.type === "camera" ? tr("Video broadcasting isn't supported by the target server.") : tr("Screen sharing isn't supported by the target server.");
return <Button switched={true} colorTheme={"red"} autoSwitch={false} iconNormal={icon} tooltip={tooltip}
key={"unsupported"}
onToggle={() => createErrorModal(modalTitle, modalBody).open()}
/>;
}
case "unavailable":
return <Button switched={true} colorTheme={"red"} autoSwitch={false} iconNormal={ClientIcon.VideoMuted} tooltip={tr("Video broadcasting not available")} key={"unavailable"}
onToggle={() => createErrorModal(tr("Video broadcasting unavailable"), tr("Video broadcasting isn't available right now.")).open()} />;
case "unavailable": {
let tooltip = props.type === "camera" ? tr("Video broadcasting not available") : tr("Screen sharing not available");
let modalTitle = props.type === "camera" ? tr("Video broadcasting unavailable") : tr("Screen sharing unavailable");
let modalBody = props.type === "camera" ? tr("Video broadcasting isn't available right now.") : tr("Screen sharing isn't available right now.");
return <Button switched={true} colorTheme={"red"} autoSwitch={false} iconNormal={icon} tooltip={tooltip}
key={"unavailable"}
onToggle={() => createErrorModal(modalTitle, modalBody).open()}/>;
}
case "disconnected":
case "disabled":
return <Button switched={true} colorTheme={"red"} autoSwitch={false} iconNormal={ClientIcon.VideoMuted}
onToggle={() => events.fire("action_toggle_video", { enable: true, broadcastType: "camera" })}
tooltip={tr("Start video broadcasting")} key={"enable"}/>;
case "disabled": {
let tooltip = props.type === "camera" ? tr("Start video broadcasting") : tr("Start screen sharing");
return <Button switched={true} colorTheme={"red"} autoSwitch={false} iconNormal={icon}
onToggle={() => events.fire("action_toggle_video", {enable: true, broadcastType: props.type})}
tooltip={tooltip} key={"enable"}/>;
}
case "enabled":
return <Button switched={false} colorTheme={"red"} autoSwitch={false} iconNormal={ClientIcon.VideoMuted}
onToggle={() => events.fire("action_toggle_video", { enable: false, broadcastType: "camera" })}
tooltip={tr("Stop video broadcasting")} key={"disable"}/>;
case "enabled": {
let tooltip = props.type === "camera" ? tr("Stop video broadcasting") : tr("Stop screen sharing");
return <Button switched={false} colorTheme={"red"} autoSwitch={false} iconNormal={icon}
onToggle={() => events.fire("action_toggle_video", {enable: false, broadcastType: props.type})}
tooltip={tooltip} key={"disable"}/>;
}
}
}
@ -388,7 +404,8 @@ export const ControlBar2 = (props: { events: Registry<ControlBarEvents>, classNa
items.push(<div className={cssStyle.divider + " " + cssStyle.hideSmallPopout} key={"divider-1"} />);
}
items.push(<AwayButton key={"away"} />);
items.push(<VideoButton key={"video"} />);
items.push(<VideoButton key={"video"} type={"camera"} />);
items.push(<VideoButton key={"screen"} type={"screen"} />);
items.push(<MicrophoneButton key={"microphone"} />);
items.push(<SpeakerButton key={"speaker"} />);
items.push(<div className={cssStyle.divider + " " + cssStyle.hideSmallPopout} key={"divider-2"} />);

4
shared/js/ui/frames/video/Controller.ts
View file

@ -174,6 +174,10 @@ class LocalVideoController extends RemoteClientVideoController {
return videoConnection.isBroadcasting("camera") || videoConnection.isBroadcasting("screen");
}
async getStatistics(target: VideoBroadcastType) {
}
protected isVideoActive(): boolean {
return true;
}

35
shared/js/ui/frames/video/Definitions.ts
View file

@ -1,4 +1,5 @@
import {ClientIcon} from "svg-sprites/client-icons";
import {VideoBroadcastType} from "tc-shared/connection/VideoConnection";
export const kLocalVideoId = "__local__video__";
@ -17,6 +18,34 @@ export type ChannelVideo ={
status: "no-video"
};
export type VideoStatistics = {
type: "sender",
mode: "camara" | "screen",
dimensions: { width: number, height: number },
frameRate: number,
codec: { name: string, payloadType: number }
maxBandwidth: number,
bandwidth: number,
qualityLimitation: "cpu" | "bandwidth",
source: {
frameRate: number,
dimensions: { width: number, height: number },
}
} | {
type: "receiver",
mode: "camara" | "screen",
dimensions: { width: number, height: number },
frameRate: number,
codec: { name: string, payloadType: number }
};
export interface ChannelVideoEvents {
action_toggle_expended: { expended: boolean },
action_video_scroll: { direction: "left" | "right" },
@ -26,6 +55,7 @@ export interface ChannelVideoEvents {
query_videos: {},
query_video: { videoId: string },
query_video_info: { videoId: string },
query_video_statistics: { videoId: string, broadcastType: VideoBroadcastType },
query_spotlight: {},
notify_expended: { expended: boolean },
@ -50,5 +80,10 @@ export interface ChannelVideoEvents {
},
notify_spotlight: {
videoId: string | undefined
},
notify_video_statistics: {
videoId: string | undefined,
broadcastType: VideoBroadcastType,
statistics: VideoStatistics
}
}

47
shared/js/ui/frames/video/Renderer.scss
View file

@ -194,6 +194,10 @@ $small_height: 10em;
flex-shrink: 1;
flex-grow: 1;
.videoContainer .requestFullscreen {
opacity: .5;
}
}
.videoContainer {
@ -284,4 +288,47 @@ $small_height: 10em;
}
}
}
.requestFullscreen {
position: absolute;
bottom: 0;
right: 0;
display: flex;
flex-direction: row;
border-top-left-radius: .2em;
background-color: #353535;
padding: .25em;
opacity: 0;
@include transition(all $button_hover_animation_time ease-in-out);
&.hidden {
display: none;
}
.iconContainer {
align-self: center;
display: flex;
padding: .2em;
cursor: pointer;
border-radius: .1em;
@include transition(all $button_hover_animation_time ease-in-out);
&:hover {
background-color: #ffffff1e;
}
}
}
&:hover {
.requestFullscreen {
opacity: 1;
}
}
}

54
shared/js/ui/frames/video/Renderer.tsx
View file

@ -8,6 +8,7 @@ import {Translatable} from "tc-shared/ui/react-elements/i18n";
import {LoadingDots} from "tc-shared/ui/react-elements/LoadingDots";
import {ClientTag} from "tc-shared/ui/tree/EntryTags";
import ResizeObserver from "resize-observer-polyfill";
import {LogCategory, logWarn} from "tc-shared/log";
const EventContext = React.createContext<Registry<ChannelVideoEvents>>(undefined);
const HandlerIdContext = React.createContext<string>(undefined);
@ -154,18 +155,62 @@ const VideoPlayer = React.memo((props: { videoId: string }) => {
return null;
});
const VideoContainer = React.memo((props: { videoId: string }) => {
const VideoContainer = React.memo((props: { videoId: string, isSpotlight: boolean }) => {
const events = useContext(EventContext);
const refContainer = useRef<HTMLDivElement>();
const [ isFullscreen, setFullscreen ] = useState(false);
const fullscreenCapable = "requestFullscreen" in HTMLElement.prototype;
useEffect(() => {
if(!isFullscreen) { return; }
if(document.fullscreenElement !== refContainer.current) {
setFullscreen(false);
return;
}
const listener = () => {
if(document.fullscreenElement !== refContainer.current) {
setFullscreen(false);
}
};
document.addEventListener("fullscreenchange", listener);
return () => document.removeEventListener("fullscreenchange", listener);
}, [ isFullscreen ]);
return (
<div
className={cssStyle.videoContainer}
onDoubleClick={() => events.fire("action_set_spotlight", { videoId: props.videoId, expend: true })}
onDoubleClick={() => {
if(props.isSpotlight) { return; }
events.fire("action_set_spotlight", { videoId: props.videoId, expend: true });
}}
onContextMenu={event => {
event.preventDefault()
}}
ref={refContainer}
>
<VideoPlayer videoId={props.videoId} />
<VideoInfo videoId={props.videoId} />
<div className={cssStyle.requestFullscreen + " " + (isFullscreen || !fullscreenCapable ? cssStyle.hidden : "")}>
<div className={cssStyle.iconContainer} onClick={() => {
if(props.isSpotlight) {
if(!refContainer.current) { return; }
refContainer.current.requestFullscreen().then(() => {
setFullscreen(true);
}).catch(error => {
logWarn(LogCategory.GENERAL, tr("Failed to request fullscreen: %o"), error);
});
} else {
events.fire("action_set_spotlight", { videoId: props.videoId, expend: true });
}
}}>
<ClientIconRenderer className={cssStyle.icon} icon={ClientIcon.Fullscreen} />
</div>
</div>
</div>
);
});
@ -253,7 +298,7 @@ const VideoBar = () => {
<div className={cssStyle.videoBar}>
<div className={cssStyle.videos} ref={refVideos}>
{videos === "loading" ? undefined :
videos.map(videoId => <VideoContainer videoId={videoId} key={videoId} />)
videos.map(videoId => <VideoContainer videoId={videoId} key={videoId} isSpotlight={false} />)
}
</div>
<VideoBarArrow direction={"left"} containerRef={refArrowLeft} />
@ -272,7 +317,7 @@ const Spotlight = () => {
let body;
if(videoId) {
body = <VideoContainer videoId={videoId} key={"video-" + videoId} />;
body = <VideoContainer videoId={videoId} key={"video-" + videoId} isSpotlight={true} />;
} else {
body = (
<div className={cssStyle.videoContainer} key={"no-video"}>
@ -280,6 +325,7 @@ const Spotlight = () => {
</div>
);
}
return (
<div className={cssStyle.spotlight}>
{body}

217
shared/js/ui/modal/video-source/Controller.tsx
View file

@ -4,18 +4,19 @@ import {ModalVideoSourceEvents} from "tc-shared/ui/modal/video-source/Definition
import {ModalVideoSource} from "tc-shared/ui/modal/video-source/Renderer";
import {getVideoDriver, VideoPermissionStatus, VideoSource} from "tc-shared/video/VideoSource";
import {LogCategory, logError} from "tc-shared/log";
import {VideoBroadcastType} from "tc-shared/connection/VideoConnection";
type VideoSourceRef = { source: VideoSource };
export async function spawnVideoSourceSelectModal() : Promise<VideoSource> {
export async function spawnVideoSourceSelectModal(type: VideoBroadcastType, selectDefault: boolean) : Promise<VideoSource> {
const refSource: VideoSourceRef = {
source: undefined
};
const events = new Registry<ModalVideoSourceEvents>();
events.enableDebug("video-source-select");
initializeController(events, refSource);
initializeController(events, refSource, type);
const modal = spawnReactModal(ModalVideoSource, events);
const modal = spawnReactModal(ModalVideoSource, events, type);
modal.events.on("destroy", () => {
events.fire("notify_destroy");
events.destroy();
@ -23,7 +24,11 @@ export async function spawnVideoSourceSelectModal() : Promise<VideoSource> {
events.on(["action_start", "action_cancel"], () => {
modal.destroy();
});
modal.show().then(undefined);
modal.show().then(() => {
if(selectDefault) {
events.fire("action_select_source", { id: undefined });
}
});
await new Promise(resolve => {
modal.events.one(["destroy", "close"], resolve);
@ -32,9 +37,15 @@ export async function spawnVideoSourceSelectModal() : Promise<VideoSource> {
return refSource.source;
}
function initializeController(events: Registry<ModalVideoSourceEvents>, currentSourceRef: VideoSourceRef) {
type SourceConstraints = { width?: number, height?: number, frameRate?: number };
function initializeController(events: Registry<ModalVideoSourceEvents>, currentSourceRef: VideoSourceRef, type: VideoBroadcastType) {
let currentSource: VideoSource | string;
let currentConstraints: SourceConstraints;
/* preselected current source id */
let currentSourceId: string;
/* fallback current source name if "currentSource" is empty */
let fallbackCurrentSourceName: string;
const notifyStartButton = () => {
@ -63,7 +74,7 @@ function initializeController(events: Registry<ModalVideoSourceEvents>, currentS
});
}
const notifyCurrentSource = () => {
const notifyVideoPreview = () => {
const driver = getVideoDriver();
switch (driver.getPermissionStatus()) {
case VideoPermissionStatus.SystemDenied:
@ -91,23 +102,96 @@ function initializeController(events: Registry<ModalVideoSourceEvents>, currentS
}
};
const notifyCurrentSource = () => {
if(typeof currentSource === "object") {
events.fire_react("notify_source", {
state: {
type: "selected",
deviceId: currentSource.getId(),
name: currentSource?.getName() || fallbackCurrentSourceName
}
});
} else if(typeof currentSource === "string") {
events.fire_react("notify_source", {
state: {
type: "errored",
error: currentSource
}
});
} else {
events.fire_react("notify_source", {
state: {
type: "none"
}
});
}
}
const notifySettingDimension = () => {
if(typeof currentSource === "object") {
const videoTrack = currentSource.getStream().getVideoTracks()[0];
const settings = videoTrack.getSettings();
const capabilities = videoTrack.getCapabilities();
events.fire_react("notify_setting_dimension", {
setting: {
minWidth: capabilities.width ? capabilities.width.min : settings.width,
maxWidth: capabilities.width ? capabilities.width.max : settings.width,
minHeight: capabilities.height ? capabilities.height.min : settings.height,
maxHeight: capabilities.height ? capabilities.height.max : settings.height,
originalWidth: settings.width,
originalHeight: settings.height,
currentWidth: settings.width,
currentHeight: settings.height
}
});
} else {
events.fire_react("notify_setting_dimension", { setting: undefined });
}
};
const notifySettingFramerate = () => {
if(typeof currentSource === "object") {
const videoTrack = currentSource.getStream().getVideoTracks()[0];
const settings = videoTrack.getSettings();
const capabilities = videoTrack.getCapabilities();
const round = (value: number) => Math.round(value * 100) / 100;
events.fire_react("notify_settings_framerate", {
frameRate: {
min: round(capabilities.frameRate ? capabilities.frameRate.min : settings.frameRate),
max: round(capabilities.frameRate ? capabilities.frameRate.max : settings.frameRate),
original: round(settings.frameRate)
}
});
} else {
events.fire_react("notify_settings_framerate", { frameRate: undefined });
}
};
const setCurrentSource = (source: VideoSource | string | undefined) => {
if(typeof currentSource === "object") {
currentSource.deref();
}
if(typeof source === "object") {
currentSourceRef.source = source;
}
currentConstraints = {};
currentSource = source;
notifyCurrentSource();
notifyVideoPreview();
notifyStartButton();
notifyCurrentSource();
notifySettingDimension();
notifySettingFramerate();
}
events.on("query_source", () => notifyCurrentSource());
events.on("query_device_list", () => notifyDeviceList());
events.on("query_video_preview", () => notifyCurrentSource());
events.on("query_video_preview", () => notifyVideoPreview());
events.on("query_start_button", () => notifyStartButton());
events.on("query_setting_dimension", () => notifySettingDimension());
events.on("query_setting_framerate", () => notifySettingFramerate());
events.on("action_request_permissions", () => {
getVideoDriver().requestPermissions().then(result => {
@ -126,49 +210,92 @@ function initializeController(events: Registry<ModalVideoSourceEvents>, currentS
events.on("action_select_source", event => {
const driver = getVideoDriver();
currentSourceId = event.id;
fallbackCurrentSourceName = tr("loading...");
notifyDeviceList();
if(type === "camera") {
currentSourceId = event.id;
fallbackCurrentSourceName = tr("loading...");
notifyDeviceList();
driver.createVideoSource(event.id).then(stream => {
setCurrentSource(stream);
fallbackCurrentSourceName = stream.getName();
}).catch(error => {
fallbackCurrentSourceName = "invalid device";
if(typeof error === "string") {
setCurrentSource(error);
} else {
logError(LogCategory.GENERAL, tr("Failed to open video device %s: %o"), event.id, error);
setCurrentSource(tr("Failed to open video device (Lookup the console)"));
}
});
driver.createVideoSource(currentSourceId).then(stream => {
fallbackCurrentSourceName = stream.getName();
setCurrentSource(stream);
}).catch(error => {
fallbackCurrentSourceName = "invalid device";
if(typeof error === "string") {
setCurrentSource(error);
} else {
logError(LogCategory.GENERAL, tr("Failed to open video device %s: %o"), event.id, error);
setCurrentSource(tr("Failed to open video device (Lookup the console)"));
}
});
} else {
currentSourceId = undefined;
fallbackCurrentSourceName = tr("loading...");
driver.createScreenSource().then(stream => {
setCurrentSource(stream);
fallbackCurrentSourceName = stream.getName();
}).catch(error => {
fallbackCurrentSourceName = "screen capture failed";
if(typeof error === "string") {
setCurrentSource(error);
} else {
logError(LogCategory.GENERAL, tr("Failed to open screen capture device %s: %o"), event.id, error);
setCurrentSource(tr("Failed to open screen capture device (Lookup the console)"));
}
});
}
});
events.on("action_cancel", () => {
if(typeof currentSource === "object") {
currentSourceRef.source = undefined;
currentSource.deref();
}
currentSourceRef.source = undefined;
});
events.on("notify_destroy", getVideoDriver().getEvents().on("notify_permissions_changed", () => {
if(getVideoDriver().getPermissionStatus() !== VideoPermissionStatus.Granted) {
currentSourceId = undefined;
fallbackCurrentSourceName = undefined;
notifyDeviceList();
if(type === "camera") {
/* only the camara requires a device list */
events.on("notify_destroy", getVideoDriver().getEvents().on("notify_permissions_changed", () => {
if(getVideoDriver().getPermissionStatus() !== VideoPermissionStatus.Granted) {
currentSourceId = undefined;
fallbackCurrentSourceName = undefined;
notifyDeviceList();
/* implicitly updates the start button */
setCurrentSource(undefined);
} else {
notifyDeviceList();
notifyCurrentSource();
notifyStartButton();
}
}));
/* implicitly updates the start button */
setCurrentSource(undefined);
} else {
notifyDeviceList();
notifyVideoPreview();
notifyStartButton();
}
}));
}
events.on("notify_destroy", () => {
if(typeof currentSource === "object" && currentSourceRef.source !== currentSource) {
if(typeof currentSource === "object") {
currentSource.deref();
}
});
events.on("action_start", () => {
if(typeof currentSource === "object") {
currentSourceRef.source = currentSource.ref();
}
})
const applyConstraints = async () => {
if(typeof currentSource === "object") {
const videoTrack = currentSource.getStream().getVideoTracks()[0];
if(!videoTrack) { return; }
await videoTrack.applyConstraints(currentConstraints);
}
};
events.on("action_setting_dimension", event => {
currentConstraints.height = event.height;
currentConstraints.width = event.width;
applyConstraints().then(undefined);
});
events.on("action_setting_framerate", event => {
currentConstraints.frameRate = event.frameRate;
applyConstraints().then(undefined);
});
}

43
shared/js/ui/modal/video-source/Definitions.ts
View file

@ -17,21 +17,60 @@ export type VideoPreviewStatus = {
message?: string
} | {
status: "none";
}
};
export type VideoSourceState = {
type: "none"
} | {
type: "selected",
deviceId: string,
name: string,
} | {
type: "errored",
error: string
};
export type SettingFrameRate = {
min: number,
max: number,
original: number,
};
export interface ModalVideoSourceEvents {
action_cancel: {},
action_start: {},
action_request_permissions: {},
action_select_source: { id: string },
action_select_source: { id: string | undefined },
action_setting_dimension: { width: number, height: number },
action_setting_framerate: { frameRate: number },
query_source: {},
query_device_list: {},
query_video_preview: {},
query_start_button: {},
query_setting_dimension: {},
query_setting_framerate: {},
notify_source: { state: VideoSourceState }
notify_device_list: { status: DeviceListResult },
notify_video_preview: { status: VideoPreviewStatus },
notify_start_button: { enabled: boolean },
notify_setting_dimension: {
setting: {
minWidth: number,
currentWidth: number,
originalWidth: number,
maxWidth: number,
minHeight: number,
currentHeight: number,
originalHeight: number,
maxHeight: number
} | undefined
},
notify_settings_framerate: {
frameRate: SettingFrameRate | undefined
}
notify_destroy: {}
}

134
shared/js/ui/modal/video-source/Renderer.scss
View file

@ -11,6 +11,9 @@
@include user-select(none);
.content {
display: flex;
flex-direction: row;
position: relative;
.overlay {
@ -29,20 +32,67 @@
}
}
.columnSource {
display: flex;
flex-direction: column;
justify-content: flex-start;
}
.columnSettings {
margin-left: 1em;
width: 20em;
.body .title {
display: flex;
flex-direction: row;
justify-content: space-between;
font-weight: bold;
color: #e0e0e0;
}
}
.section {
margin-bottom: 1em;
.head {
display: flex;
flex-direction: row;
justify-content: flex-start;
flex-grow: 1;
flex-shrink: 1;
align-self: flex-end;
font-weight: bold;
color: #e0e0e0;
@include text-dotdotdot();
width: 100%;
align-self: flex-end;
&.title, .title {
font-size: 1.2em;
color: #557edc;
text-transform: uppercase;
align-self: center;
@include text-dotdotdot();
}
.advanced {
margin-left: auto;
align-self: center;
label {
display: flex;
flex-direction: revert;
}
}
}
.body {
display: flex;
flex-direction: column;
justify-content: flex-start;
.selectError {
color: #a10000;
}
@ -64,6 +114,10 @@
video {
max-height: 100%;
max-width: 100%;
min-height: 100%;
min-width: 100%;
align-self: center;
}
@ -111,9 +165,29 @@
padding-bottom: 1em;
font-weight: 600;
color: #4d4d4d;
&.selected {}
&.none {}
&.error {
color: #a10000;
}
}
}
}
.sourcePrompt {
display: flex;
flex-direction: row;
justify-content: flex-start;
button {
margin-right: .5em;
}
> * {
align-self: center;
}
}
}
}
}
@ -124,3 +198,55 @@
justify-content: space-between;
}
}
.columnSettings {
display: flex;
flex-direction: column;
justify-content: flex-start;
.setting:not(:last-of-type) {
margin-bottom: .5em;
}
.head {
flex-grow: 0!important;
}
.body {
flex-grow: 1;
}
.dimensions {
.aspectRatio {
margin-top: .5em;
}
.sliderTitle {
display: flex;
flex-direction: row;
justify-content: space-between;
}
.slider:not(:first-of-type) {
margin-top: .75em;
}
.advanced {
display: none;
flex-direction: column;
justify-content: flex-start;
&.shown {
display: flex;
}
}
}
.bpsInfo {
margin-top: auto;
.body {
font-size: .8em;
}
}
}

444
shared/js/ui/modal/video-source/Renderer.tsx
View file

@ -2,20 +2,24 @@ import {Registry} from "tc-shared/events";
import * as React from "react";
import {
DeviceListResult,
ModalVideoSourceEvents,
VideoPreviewStatus
ModalVideoSourceEvents, SettingFrameRate,
VideoPreviewStatus, VideoSourceState
} from "tc-shared/ui/modal/video-source/Definitions";
import {InternalModal} from "tc-shared/ui/react-elements/internal-modal/Controller";
import {Translatable} from "tc-shared/ui/react-elements/i18n";
import {Translatable, VariadicTranslatable} from "tc-shared/ui/react-elements/i18n";
import {Select} from "tc-shared/ui/react-elements/InputField";
import {Button} from "tc-shared/ui/react-elements/Button";
import {useContext, useEffect, useRef, useState} from "react";
import {VideoBroadcastType} from "tc-shared/connection/VideoConnection";
import {Slider} from "tc-shared/ui/react-elements/Slider";
import {Checkbox} from "tc-shared/ui/react-elements/Checkbox";
const cssStyle = require("./Renderer.scss");
const ModalEvents = React.createContext<Registry<ModalVideoSourceEvents>>(undefined);
const AdvancedSettings = React.createContext<boolean>(false);
const kNoDeviceId = "__no__device";
const VideoSourceBody = () => {
const VideoSourceSelector = () => {
const events = useContext(ModalEvents);
const [ deviceList, setDeviceList ] = useState<DeviceListResult | "loading">(() => {
events.fire("query_device_list");
@ -76,6 +80,54 @@ const VideoSourceBody = () => {
}
};
const VideoSourceRequester = () => {
const events = useContext(ModalEvents);
const [ source, setSource ] = useState<VideoSourceState>(() => {
events.fire("query_source");
return { type: "none" };
});
events.reactUse("notify_source", event => setSource(event.state), undefined, []);
let info;
switch (source.type) {
case "selected":
let name = source.name === "Screen" ? source.deviceId : source.name;
info = (
<div className={cssStyle.info + " " + cssStyle.selected} key={"selected"}>
<VariadicTranslatable text={"Selected source: {}"}>{name || source.deviceId}</VariadicTranslatable>
</div>
);
break;
case "errored":
info = (
<div className={cssStyle.info + " " + cssStyle.error} key={"errored"}>
{source.error}
</div>
);
break;
case "none":
info = (
<div className={cssStyle.info + " " + cssStyle.none} key={"none"}>
<Translatable>No source has been selected</Translatable>
</div>
);
break;
}
return (
<div className={cssStyle.body} key={"normal"}>
<div className={cssStyle.sourcePrompt}>
<Button type={"small"} onClick={() => events.fire("action_select_source", { id: undefined })}>
<Translatable>Select source</Translatable>
</Button>
{info}
</div>
</div>
);
};
const VideoPreviewMessage = (props: { message: any, kind: "info" | "error" }) => {
return (
<div className={cssStyle.overlay + " " + (props.message ? cssStyle.shown : "")}>
@ -198,12 +250,368 @@ const ButtonStart = () => {
);
}
type DimensionPresetId = "480p" | "720p" | "1080p" | "2160p";
const DimensionPresets: {[T in DimensionPresetId]: {
name: string,
width: number,
height: number
}} = {
"480p": {
name: "480p",
width: 640,
height: 480
},
"720p": {
name: "720p",
width: 1280,
height: 720
},
"1080p": {
name: "Full HD",
width: 1920,
height: 1080
},
"2160p": {
name: "4k UHD",
width: 3840,
height: 2160
}
};
type SettingDimensionValues = {
minWidth: number,
maxWidth: number,
minHeight: number,
maxHeight: number,
originalHeight: number,
originalWidth: number
}
const SettingDimension = () => {
const events = useContext(ModalEvents);
const advanced = useContext(AdvancedSettings);
const [ settings, setSettings ] = useState<SettingDimensionValues | undefined>(() => {
events.fire("query_setting_dimension");
return undefined;
});
events.reactUse("notify_setting_dimension", event => {
if(event.setting) {
setSettings({
minWidth: event.setting.minWidth,
minHeight: event.setting.minHeight,
maxWidth: event.setting.maxWidth,
maxHeight: event.setting.maxHeight,
originalHeight: event.setting.originalHeight,
originalWidth: event.setting.originalWidth
});
setWidth(event.setting.currentWidth);
setHeight(event.setting.currentHeight);
refSliderWidth.current?.setState({ value: event.setting.currentWidth });
refSliderHeight.current?.setState({ value: event.setting.currentHeight });
setSelectValue("original");
} else {
setSettings(undefined);
setSelectValue("no-source");
}
}, undefined, []);
const [ width, setWidth ] = useState(1);
const [ height, setHeight ] = useState(1);
const [ selectValue, setSelectValue ] = useState("no-source");
const [ aspectRatio, setAspectRatio ] = useState(true);
const refSliderWidth = useRef<Slider>();
const refSliderHeight = useRef<Slider>();
const bounds = (id: DimensionPresetId) => {
const dimension = DimensionPresets[id];
let scale = Math.min(dimension.height / settings.maxHeight, dimension.width / settings.maxWidth);
return [Math.ceil(settings.maxWidth * scale), Math.ceil(settings.maxHeight * scale)];
}
const boundsString = (id: DimensionPresetId) => {
const value = bounds(id);
return value[0] + "x" + value[1];
}
const updateHeight = (newHeight: number, triggerUpdate: boolean) => {
let newWidth = width;
setHeight(newHeight);
if(aspectRatio) {
newWidth = Math.ceil(settings.originalWidth * (newHeight / settings.originalHeight));
setWidth(newWidth);
refSliderWidth.current?.setState({ value: newWidth });
}
if(triggerUpdate) {
events.fire("action_setting_dimension", { height: newHeight, width: newWidth });
}
}
const updateWidth = (newWidth: number, triggerUpdate: boolean) => {
let newHeight = height;
setWidth(newWidth);
if(aspectRatio) {
newHeight = Math.ceil(settings.originalHeight * (newWidth / settings.originalWidth));
setHeight(newHeight);
refSliderHeight.current?.setState({ value: newHeight });
}
if(triggerUpdate) {
events.fire("action_setting_dimension", { height: newHeight, width: newWidth });
}
}
return (
<div className={cssStyle.setting + " " + cssStyle.dimensions}>
<div className={cssStyle.title}>
<div><Translatable>Dimension</Translatable></div>
<div>{settings ? width + "x" + height : ""}</div>
</div>
<div className={cssStyle.body}>
<Select
type={"boxed"}
onChange={event => {
const value = event.target.value;
setSelectValue(value);
let newWidth, newHeight;
if(value === "original") {
newWidth = settings.originalWidth;
newHeight = settings.originalHeight;
} else if(value === "custom") {
/* nothing to do; no need to trigger an update as well */
return;
} else if(DimensionPresets[value]) {
const val = bounds(value as any);
newWidth = val[0];
newHeight = val[1];
}
setWidth(newWidth);
setHeight(newHeight);
refSliderWidth.current?.setState({ value: newWidth });
refSliderHeight.current?.setState({ value: newHeight });
events.fire("action_setting_dimension", { height: newHeight, width: newWidth });
}}
value={selectValue}
disabled={!settings}
>
{Object.keys(DimensionPresets).filter(e => {
if(!settings) { return false; }
if(DimensionPresets[e].height < settings.minHeight || DimensionPresets[e].height > settings.maxHeight) { return false; }
return !(DimensionPresets[e].width < settings.minWidth || DimensionPresets[e].width > settings.maxWidth);
}).map(dimensionId =>
<option value={dimensionId} key={dimensionId}>{DimensionPresets[dimensionId].name + " (" + boundsString(dimensionId as any) + ")"}</option>
)}
<option value={"original"} key={"original"}>{tr("Default")} ({(settings ? settings.originalWidth + "x" + settings.originalHeight : "0x0")})</option>
<option value={"custom"} key={"custom"} style={{ display: advanced ? undefined : "none" }}>{tr("Custom")}</option>
<option value={"no-source"} key={"no-source"} style={{ display: "none" }}>{tr("No source selected")}</option>
</Select>
<div className={cssStyle.advanced + " " + (advanced ? cssStyle.shown : "")}>
<div className={cssStyle.aspectRatio}>
<Checkbox
label={<Translatable>Aspect ratio</Translatable>}
disabled={selectValue !== "custom" || !settings}
value={aspectRatio}
onChange={value => setAspectRatio(value)}
/>
</div>
<div className={cssStyle.sliderTitle}>
<div><Translatable>Width</Translatable></div>
<div>{settings ? width + "px" : ""}</div>
</div>
<Slider tooltip={null} minValue={settings ? settings.minWidth : 0} maxValue={settings ? settings.maxWidth : 1}
stepSize={1} value={width} className={cssStyle.slider}
onInput={value => updateWidth(value, false)}
disabled={selectValue !== "custom" || !settings}
onChange={value => updateWidth(value, true)}
ref={refSliderWidth}
/>
<div className={cssStyle.sliderTitle}>
<div><Translatable>Height</Translatable></div>
<div>{settings ? height + "px" : ""}</div>
</div>
<Slider tooltip={null} minValue={settings ? settings.minHeight : 0} maxValue={settings ? settings.maxHeight : 1}
stepSize={1} value={height} className={cssStyle.slider}
onInput={value => updateHeight(value, false)}
onChange={value => updateHeight(value, true)}
disabled={selectValue !== "custom" || !settings}
ref={refSliderHeight}
/>
</div>
</div>
</div>
);
}
const FrameRatePresents = {
"10": 10,
"20": 20,
"24 NTSC": 24,
"25 PAL": 25,
"29.97": 29.97,
"30": 30,
"48": 48,
"50 PAL": 50,
"59.94": 59.94,
"60": 60
}
const SettingFramerate = () => {
const events = useContext(ModalEvents);
const [ frameRate, setFrameRate ] = useState<SettingFrameRate | undefined>(() => {
events.fire("query_setting_framerate");
return undefined;
});
const [ currentRate, setCurrentRate ] = useState(0);
const [ selectedValue, setSelectedValue ] = useState("original");
events.reactUse("notify_settings_framerate", event => {
setFrameRate(event.frameRate);
setCurrentRate(event.frameRate ? event.frameRate.original : 1);
if(event.frameRate) {
setSelectedValue(event.frameRate.original.toString());
} else {
setSelectedValue("no-source");
}
});
const FrameRates = Object.assign({}, FrameRatePresents);
if(frameRate) {
if(Object.keys(FrameRates).findIndex(key => FrameRates[key] === frameRate.original) === -1) {
FrameRates[frameRate.original.toString()] = frameRate.original;
}
}
return (
<div className={cssStyle.setting + " " + cssStyle.dimensions}>
<div className={cssStyle.title}>
<div><Translatable>Framerate</Translatable></div>
<div>{frameRate ? currentRate : ""}</div>
</div>
<div className={cssStyle.body}>
<Select
type={"boxed"}
disabled={!frameRate}
value={selectedValue}
onChange={event => {
const value = parseFloat(event.target.value);
if(!isNaN(value)) {
setSelectedValue(event.target.value);
setCurrentRate(value);
events.fire("action_setting_framerate", { frameRate: value });
}
}}
>
{Object.keys(FrameRates).sort((a, b) => FrameRates[a] - FrameRates[b]).filter(key => {
if(!frameRate) { return false; }
const value = FrameRates[key];
if(frameRate.min > value) { return false; }
return frameRate.max >= value;
}).map(key => (
<option value={FrameRates[key]} key={key}>{key}</option>
))}
<option value={"1"} key={"no-source"} style={{ display: "none" }}>{tr("No source selected")}</option>
</Select>
</div>
</div>
);
}
const calculateBps = (width: number, height: number, frameRate: number) => {
/* Based on the tables showed here: http://www.lighterra.com/papers/videoencodingh264/ */
const estimatedBitsPerPixed = 3.9;
return estimatedBitsPerPixed * width * height * (frameRate / 30);
}
const BpsInfo = () => {
const events = useContext(ModalEvents);
const [ dimensions, setDimensions ] = useState<{ width: number, height: number } | undefined>(undefined);
const [ frameRate, setFrameRate ] = useState<number | undefined>(undefined);
events.reactUse("notify_setting_dimension", event => setDimensions(event.setting ? {
height: event.setting.currentHeight,
width: event.setting.currentWidth
} : undefined));
events.reactUse("notify_settings_framerate", event => setFrameRate(event.frameRate ? event.frameRate.original : undefined))
events.reactUse("action_setting_dimension", event => setDimensions({ width: event.width, height: event.height }));
events.reactUse("action_setting_framerate", event => setFrameRate(event.frameRate));
let bpsText;
if(dimensions && frameRate) {
const bps = calculateBps(dimensions.width, dimensions.height, frameRate);
if(bps > 1000 * 1000) {
bpsText = (bps / 1000 / 1000).toFixed(2) + " MBits/Second";
} else {
bpsText = (bps / 1000).toFixed(2) + " KBits/Second";
}
}
return (
<div className={cssStyle.setting + " " + cssStyle.bpsInfo}>
<div className={cssStyle.title}>
<div><Translatable>Estimated Bitrate</Translatable></div>
<div>{bpsText}</div>
</div>
<div className={cssStyle.body}>
<Translatable>
This is a rough estimate of the bitrate used to transfer video of that quality.
The real bitrate might have higher peaks but averages below the estimate.
Influential factors are the video size as well as the frame rate.
</Translatable>
</div>
</div>
);
}
const Settings = () => {
const [ advanced, setAdvanced ] = useState(false);
return (
<AdvancedSettings.Provider value={advanced}>
<div className={cssStyle.section + " " + cssStyle.columnSettings}>
<div className={cssStyle.head}>
<div className={cssStyle.title}><Translatable>Settings</Translatable></div>
<div className={cssStyle.advanced}>
<Checkbox label={<Translatable>Advanced</Translatable>} onChange={value => setAdvanced(value)} />
</div>
</div>
<div className={cssStyle.body}>
<SettingDimension />
<SettingFramerate />
<BpsInfo />
</div>
</div>
</AdvancedSettings.Provider>
);
}
export class ModalVideoSource extends InternalModal {
protected readonly events: Registry<ModalVideoSourceEvents>;
private readonly sourceType: VideoBroadcastType;
constructor(events: Registry<ModalVideoSourceEvents>) {
constructor(events: Registry<ModalVideoSourceEvents>, type: VideoBroadcastType) {
super();
this.sourceType = type;
this.events = events;
}
@ -212,21 +620,23 @@ export class ModalVideoSource extends InternalModal {
<ModalEvents.Provider value={this.events}>
<div className={cssStyle.container}>
<div className={cssStyle.content}>
<div className={cssStyle.section}>
<div className={cssStyle.head}>
<Translatable>Select your source</Translatable>
<div className={cssStyle.columnSource}>
<div className={cssStyle.section}>
<div className={cssStyle.head + " " + cssStyle.title}>
<Translatable>Select your source</Translatable>
</div>
{this.sourceType === "camera" ? <VideoSourceSelector key={"source-selector"} /> : <VideoSourceRequester key={"source-requester"} />}
</div>
<VideoSourceBody />
</div>
<div className={cssStyle.section}>
<div className={cssStyle.head}>
<Translatable>Video preview</Translatable>
</div>
<div className={cssStyle.body}>
<VideoPreview />
<div className={cssStyle.section}>
<div className={cssStyle.head + " " + cssStyle.title}>
<Translatable>Video preview</Translatable>
</div>
<div className={cssStyle.body}>
<VideoPreview />
</div>
</div>
</div>
{ /* TODO: All the overlays */ }
<Settings />
</div>
<div className={cssStyle.buttons}>
<Button type={"small"} color={"red"} onClick={() => this.events.fire("action_cancel")}>

12
shared/js/ui/react-elements/Slider.tsx
View file

@ -17,7 +17,7 @@ export interface SliderProperties {
inverseFiller?: boolean;
unit?: string;
tooltip?: (value: number) => ReactElement | string;
tooltip?: (value: number) => ReactElement | string | null;
onInput?: (value: number) => void;
onChange?: (value: number) => void;
@ -67,7 +67,7 @@ export class Slider extends React.Component<SliderProperties, SliderState> {
else if(offset > range)
offset = range;
this.refTooltip.current.setState({
this.refTooltip.current?.setState({
pageX: bounds.left + offset * bounds.width / range,
});
@ -133,9 +133,11 @@ export class Slider extends React.Component<SliderProperties, SliderState> {
right: this.props.inverseFiller ? 0 : (100 - offset) + "%",
left: this.props.inverseFiller ? offset + "%" : 0
}} />
<Tooltip ref={this.refTooltip} tooltip={() => this.props.tooltip ? this.props.tooltip(this.state.value) : this.renderTooltip()}>
<div className={cssStyle.thumb} style={{left: offset + "%"}} />
</Tooltip>
{this.props.tooltip === null ? <div className={cssStyle.thumb} style={{left: offset + "%"}} key={"thumb"} /> :
<Tooltip ref={this.refTooltip} tooltip={() => this.props.tooltip ? this.props.tooltip(this.state.value) : this.renderTooltip()} key={"tooltip"}>
<div className={cssStyle.thumb} style={{left: offset + "%"}} />
</Tooltip>
}
</div>
);
}

5
shared/js/video/VideoSource.ts
View file

@ -43,8 +43,11 @@ export interface VideoDriver {
* @returns A VideoSource on success with an initial ref count of one
* Will throw a string on error
*/
createVideoSource(id: string) : Promise<VideoSource>;
createVideoSource(id: string | undefined) : Promise<VideoSource>;
/**
* Create a source from the screen
*/
createScreenSource() : Promise<VideoSource>;
}

6
shared/svg-sprites/client-icons.d.ts vendored
View file

File diff suppressed because one or more lines are too long

6
web/app/media/Stream.ts
View file

@ -52,7 +52,7 @@ async function requestMediaStream0(constraints: MediaTrackConstraints, type: Med
/* request permission for devices only one per time! */
let currentMediaStreamRequest: Promise<MediaStream | MediaStreamRequestResult>;
export async function requestMediaStream(deviceId: string, groupId: string | undefined, type: MediaStreamType) : Promise<MediaStream | MediaStreamRequestResult> {
export async function requestMediaStream(deviceId: string | undefined, groupId: string | undefined, type: MediaStreamType) : Promise<MediaStream | MediaStreamRequestResult> {
/* wait for the current media stream requests to finish */
while(currentMediaStreamRequest) {
try {
@ -66,9 +66,11 @@ export async function requestMediaStream(deviceId: string, groupId: string | und
* Firefox only allows to open one mic/video as well deciding whats the input device it.
* It does not respect the deviceId nor the groupId
*/
} else {
} else if(deviceId !== undefined) {
constrains.deviceId = deviceId;
constrains.groupId = groupId;
} else {
/* Nothing to select. Let the browser select the right device */
}
constrains.echoCancellation = true;

33
web/app/media/Video.ts
View file

@ -8,7 +8,13 @@ import {
import {Registry} from "tc-shared/events";
import {queryMediaPermissions, requestMediaStream, stopMediaStream} from "tc-backend/web/media/Stream";
import {MediaStreamRequestResult} from "tc-shared/voice/RecorderBase";
import {LogCategory, logError, logWarn} from "tc-shared/log";
import {LogCategory, logDebug, logError, logWarn} from "tc-shared/log";
declare global {
interface MediaDevices {
getDisplayMedia(options?: any) : Promise<MediaStream>;
}
}
function getStreamVideoDeviceId(stream: MediaStream) : string | undefined {
const track = stream.getVideoTracks()[0];
@ -161,8 +167,8 @@ export class WebVideoDriver implements VideoDriver {
return this.currentPermissionStatus;
}
async createVideoSource(id: string): Promise<VideoSource> {
const result = await requestMediaStream(id, undefined, "video");
async createVideoSource(id: string | undefined): Promise<VideoSource> {
const result = await requestMediaStream(id ? id : "default", undefined, "video");
/*
* If we've got denied of requesting a stream reset the state to not allowed.
@ -184,8 +190,12 @@ export class WebVideoDriver implements VideoDriver {
if(result instanceof MediaStream) {
const deviceId = getStreamVideoDeviceId(result);
if(deviceId === undefined) {
if(id === undefined && deviceId === undefined) {
logWarn(LogCategory.GENERAL, tr("Requested default video source, but returned source is nothing."));
} else if(deviceId === undefined) {
/* Do nothing. We've to trust that the given track origins from the requested id. */
} else if(id === undefined) {
/* We requested the default id and received something */
} else if(deviceId !== id) {
logWarn(LogCategory.GENERAL, tr("Requested video source %s but received %s"), id, deviceId);
} else {
@ -201,8 +211,19 @@ export class WebVideoDriver implements VideoDriver {
}
}
createScreenSource(): Promise<VideoSource> {
return Promise.resolve(undefined);
async createScreenSource(): Promise<VideoSource> {
try {
const source = await navigator.mediaDevices.getDisplayMedia({ audio: false, video: true });
const videoTrack = source.getVideoTracks()[0];
if(!videoTrack) { throw tr("missing video track"); }
logDebug(LogCategory.VIDEO, tr("Display media received with settings: %o"), videoTrack.getSettings());
return new WebVideoSource(videoTrack.getSettings().deviceId, tr("Screen"), source);
} catch (error) {
logWarn(LogCategory.VIDEO, tr("Failed to create a screen source: %o"), error);
}
return undefined;
}
}

168
web/app/rtc/Connection.ts
View file

@ -30,6 +30,27 @@ declare global {
}
}
export type RtcVideoBroadcastStatistics = {
dimensions: { width: number, height: number },
frameRate: number,
codec?: { name: string, payloadType: number }
bandwidth?: {
/* bits per second */
currentBps: number,
/* bits per second */
maxBps: number
},
qualityLimitation: "cpu" | "bandwidth" | "none",
source: {
frameRate: number,
dimensions: { width: number, height: number },
},
};
class RetryTimeCalculator {
private readonly minTime: number;
private readonly maxTime: number;
@ -64,6 +85,49 @@ class RetryTimeCalculator {
}
}
class RTCStatsWrapper {
private readonly supplier: () => Promise<RTCStatsReport>;
private readonly statistics;
constructor(supplier: () => Promise<RTCStatsReport>) {
this.supplier = supplier;
this.statistics = {};
}
async initialize() {
for(const [key, value] of await this.supplier()) {
if(typeof this.statistics[key] !== "undefined") {
logWarn(LogCategory.WEBRTC, tr("Duplicated statistics entry for key %s. Dropping duplicate."), key);
continue;
}
this.statistics[key] = value;
}
}
getValues() : (RTCStats & {[T: string]: string | number})[] {
return Object.values(this.statistics);
}
getStatistic(key: string) : RTCStats & {[T: string]: string | number} {
return this.statistics[key];
}
getStatisticsByType(type: string) : (RTCStats & {[T: string]: string | number})[] {
return Object.values(this.statistics).filter((e: any) => e.type?.replace(/-/g, "") === type) as any;
}
getStatisticByType(type: string): RTCStats & {[T: string]: string | number} {
const entries = this.getStatisticsByType(type);
if(entries.length === 0) {
throw tra("missing statistic entry {}", type);
} else if(entries.length === 1) {
return entries[0];
} else {
throw tra("duplicated statistics entry of type {}", type);
}
}
}
let dummyVideoTrack: MediaStreamTrack | undefined;
let dummyAudioTrack: MediaStreamTrack | undefined;
@ -135,7 +199,7 @@ class CommandHandler extends AbstractCommandHandler {
return;
}
if(RTCConnection.kEnableSdpTrace) {
logTrace(LogCategory.WEBRTC, tr("Patched remote %s:\n%s"), data.mode, data.sdp);
logTrace(LogCategory.WEBRTC, tr("Patched remote %s:\n%s"), data.mode, sdp);
}
if(data.mode === "answer") {
this.handle["peer"].setRemoteDescription({
@ -647,12 +711,14 @@ export class RTCConnection {
this.currentTransceiver["video-screen"] = this.peer.addTransceiver("video");
/* add some other transceivers for later use */
/*
for(let i = 0; i < 8; i++) {
this.peer.addTransceiver("audio");
}
for(let i = 0; i < 4; i++) {
this.peer.addTransceiver("video");
}
*/
this.peer.onicecandidate = event => this.handleIceCandidate(event.candidate);
this.peer.onicecandidateerror = event => this.handleIceCandidateError(event);
@ -969,4 +1035,104 @@ export class RTCConnection {
};
}
}
async getVideoBroadcastStatistics(type: RTCBroadcastableTrackType) : Promise<RtcVideoBroadcastStatistics | undefined> {
if(!this.currentTransceiver[type]?.sender) { return undefined; }
const senderStatistics = new RTCStatsWrapper(() => this.currentTransceiver[type].sender.getStats());
await senderStatistics.initialize();
if(senderStatistics.getValues().length === 0) { return undefined; }
const trackSettings = this.currentTransceiver[type].sender.track?.getSettings() || {};
const result = {} as RtcVideoBroadcastStatistics;
const outboundStream = senderStatistics.getStatisticByType("outboundrtp");
/* only available in chrome */
if("codecId" in outboundStream) {
if(typeof outboundStream.codecId !== "string") { throw tr("invalid codec id type"); }
if(senderStatistics[outboundStream.codecId]?.type !== "codec") { throw tra("invalid/missing codec statistic for codec {}", outboundStream.codecId); }
const codecInfo = senderStatistics[outboundStream.codecId];
if(typeof codecInfo.mimeType !== "string") { throw tr("codec statistic missing mine type"); }
if(typeof codecInfo.payloadType !== "number") { throw tr("codec statistic has invalid payloadType type"); }
result.codec = {
name: codecInfo.mimeType.startsWith("video/") ? codecInfo.mimeType.substr(6) : codecInfo.mimeType || tr("unknown"),
payloadType: codecInfo.payloadType
};
} else {
/* TODO: Get the only one video type from the sdp */
}
if("frameWidth" in outboundStream && "frameHeight" in outboundStream) {
if(typeof outboundStream.frameWidth !== "number") { throw tr("invalid frameWidth attribute of outboundrtp statistic"); }
if(typeof outboundStream.frameHeight !== "number") { throw tr("invalid frameHeight attribute of outboundrtp statistic"); }
result.dimensions = {
width: outboundStream.frameWidth,
height: outboundStream.frameHeight
};
} else if("height" in trackSettings && "width" in trackSettings) {
result.dimensions = {
height: trackSettings.height,
width: trackSettings.width
};
} else {
result.dimensions = {
width: 0,
height: 0
};
}
if("framesPerSecond" in outboundStream) {
if(typeof outboundStream.framesPerSecond !== "number") { throw tr("invalid framesPerSecond attribute of outboundrtp statistic"); }
result.frameRate = outboundStream.framesPerSecond;
} else if("frameRate" in trackSettings) {
result.frameRate = trackSettings.frameRate;
} else {
result.frameRate = 0;
}
if("qualityLimitationReason" in outboundStream) {
/* TODO: verify the value? */
if(typeof outboundStream.qualityLimitationReason !== "string") { throw tr("invalid qualityLimitationReason attribute of outboundrtp statistic"); }
result.qualityLimitation = outboundStream.qualityLimitationReason as any;
} else {
result.qualityLimitation = "none";
}
if("mediaSourceId" in outboundStream) {
if(typeof outboundStream.mediaSourceId !== "string") { throw tr("invalid media source type"); }
if(senderStatistics[outboundStream.mediaSourceId]?.type !== "media-source") { throw tra("invalid/missing media source statistic for source {}", outboundStream.mediaSourceId); }
const source = senderStatistics[outboundStream.mediaSourceId];
if(typeof source.width !== "number") { throw tr("invalid width attribute of media-source statistic"); }
if(typeof source.height !== "number") { throw tr("invalid height attribute of media-source statistic"); }
if(typeof source.framesPerSecond !== "number") { throw tr("invalid framesPerSecond attribute of media-source statistic"); }
result.source = {
dimensions: { height: source.height, width: source.width },
frameRate: source.framesPerSecond
};
} else {
result.source = {
dimensions: { width: 0, height: 0 },
frameRate: 0
};
if("height" in trackSettings && "width" in trackSettings) {
result.source.dimensions = {
height: trackSettings.height,
width: trackSettings.width
};
}
if("frameRate" in trackSettings) {
result.source.frameRate = trackSettings.frameRate;
}
}
return result;
}
}

30
web/app/rtc/SdpUtils.ts
View file

@ -13,7 +13,7 @@ interface SdpCodec {
/* These MUST be the payloads used by the remote as well */
const OPUS_VOICE_PAYLOAD_TYPE = 111;
const OPUS_MUSIC_PAYLOAD_TYPE = 112;
const VP8_PAYLOAD_TYPE = 120;
const H264_PAYLOAD_TYPE = 126;
type SdpMedia = {
type: string;
@ -45,11 +45,18 @@ export class SdpProcessor {
];
private static readonly kVideoCodecs: SdpCodec[] = [
/* TODO: Set AS as well! */
{
payload: VP8_PAYLOAD_TYPE,
codec: "VP8",
payload: H264_PAYLOAD_TYPE,
codec: "H264",
rate: 90000,
rtcpFb: [ "nack", "nack pli", "ccm fir", "transport-cc" ]
rtcpFb: [ "nack", "nack pli", "ccm fir", "transport-cc" ],
//42001f | Original: 42e01f
fmtp: {
"level-asymmetry-allowed": 1, "packetization-mode": 1, "profile-level-id": "42e01f", "max-br": 25000, "max-fr": 60,
"x-google-max-bitrate": 22 * 1000,
"x-google-start-bitrate": 22 * 1000, /* Fun fact: This actually controls the max bitrate for google chrome */
}
}
];
@ -74,8 +81,15 @@ export class SdpProcessor {
}
processIncomingSdp(sdpString: string, _mode: "offer" | "answer") : string {
/* The server somehow does not encode the level id in hex */
sdpString = sdpString.replace(/profile-level-id=4325407/g, "profile-level-id=42e01f");
const sdp = sdpTransform.parse(sdpString);
this.rtpRemoteChannelMapping = SdpProcessor.generateRtpSSrcMapping(sdp);
/* FIXME! */
SdpProcessor.patchLocalCodecs(sdp);
return sdpTransform.write(sdp);
}
@ -154,11 +168,17 @@ export class SdpProcessor {
payload: codec.payload,
config: Object.keys(codec.fmtp).map(e => e + "=" + codec.fmtp[e]).join(";")
});
media.maxptime = media.fmtp["maxptime"];
if(media.type === "audio") {
media.maxptime = media.fmtp["maxptime"];
}
}
}
media.payloads = media.rtp.map(e => e.payload).join(" ");
media.bandwidth = [{
type: "AS",
limit: 12000
}]
}
}
}

7
web/app/rtc/video/Connection.ts
View file

@ -1,5 +1,5 @@
import {
VideoBroadcastState,
VideoBroadcastState, VideoBroadcastStatistics,
VideoBroadcastType,
VideoClient,
VideoConnection,
@ -15,7 +15,6 @@ import {RtpVideoClient} from "tc-backend/web/rtc/video/VideoClient";
import {tr} from "tc-shared/i18n/localize";
import {ConnectionState} from "tc-shared/ConnectionHandler";
import {ConnectionStatistics} from "tc-shared/connection/ConnectionBase";
import {VoiceConnectionStatus} from "tc-shared/connection/VoiceConnection";
type VideoBroadcast = {
readonly source: VideoSource;
@ -128,6 +127,10 @@ export class RtpVideoConnection implements VideoConnection {
return this.broadcasts[type] ? this.broadcasts[type].state : VideoBroadcastState.Stopped;
}
getBroadcastStatistics(type: VideoBroadcastType): Promise<VideoBroadcastStatistics | undefined> {
return this.rtcConnection.getVideoBroadcastStatistics(type);
}
getBroadcastingSource(type: VideoBroadcastType): VideoSource | undefined {
return this.broadcasts[type]?.source;
}