Adding support for PIP

shared/js/ui/frames/video/Controller.ts

@@ -6,7 +6,7 @@ import {Registry} from "tc-shared/events";
 import {
     ChannelVideoEvents,
     ChannelVideoStreamState,
-    kLocalVideoId,
+    kLocalVideoId, makeVideoAutoplay,
     VideoStreamState
 } from "tc-shared/ui/frames/video/Definitions";
 import {
@@ -22,6 +22,7 @@ import {tr} from "tc-shared/i18n/localize";
 import {Settings, settings} from "tc-shared/settings";
 import * as _ from "lodash";
 import PermissionType from "tc-shared/permission/PermissionType";
+import {createErrorModal} from "tc-shared/ui/elements/Modal";
 
 const cssStyle = require("./Renderer.scss");
 
@@ -32,6 +33,7 @@ interface ClientVideoController {
     subscribeVideo(type: VideoBroadcastType);
     muteVideo(type: VideoBroadcastType);
     dismissVideo(type: VideoBroadcastType);
+    showPip(type: VideoBroadcastType) : Promise<void>;
 
     notifyVideoInfo();
     notifyVideo();
@@ -60,6 +62,9 @@ class RemoteClientVideoController implements ClientVideoController {
         camera: "none"
     };
 
+    private pipElement: HTMLVideoElement | undefined;
+    private pipBroadcastType: VideoBroadcastType | undefined;
+
     constructor(client: ClientEntry, eventRegistry: Registry<ChannelVideoEvents>, videoId?: string) {
         this.client = client;
         this.events = eventRegistry;
@@ -114,6 +119,9 @@ class RemoteClientVideoController implements ClientVideoController {
 
         this.eventListener?.forEach(callback => callback());
         this.eventListener = undefined;
+
+        this.pipElement?.remove();
+        this.pipElement = undefined;
     }
 
     isBroadcasting() {
@@ -201,6 +209,10 @@ class RemoteClientVideoController implements ClientVideoController {
                 this.callbackSubscriptionStateChanged();
             }
         }
+
+        if(this.pipBroadcastType && this.currentStreamStates[this.pipBroadcastType] !== "streaming") {
+            this.stopPip();
+        }
     }
 
     notifyVideo() {
@@ -230,6 +242,14 @@ class RemoteClientVideoController implements ClientVideoController {
             }
         }
 
+        if(this.pipBroadcastType === type && this.pipElement) {
+            if(state.state === "connected") {
+                this.pipElement.srcObject = state.stream;
+            } else {
+                this.stopPip();
+            }
+        }
+
         this.events.fire_react("notify_video_stream", {
             videoId: this.videoId,
             broadcastType: type,
@@ -250,6 +270,81 @@ class RemoteClientVideoController implements ClientVideoController {
         const videoClient = this.client.getVideoClient();
         return videoClient ? videoClient.getVideoStream(target) : undefined;
     }
+
+    private stopPip() {
+        if((document as any).pictureInPictureElement === this.pipElement && "exitPictureInPicture" in document) {
+            (document as any).exitPictureInPicture();
+        }
+
+        this.pipElement?.remove();
+        this.pipElement = undefined;
+        this.pipBroadcastType = undefined;
+    }
+
+    async showPip(type: VideoBroadcastType) {
+        if(this.pipBroadcastType === type) {
+            return;
+        }
+        this.pipBroadcastType = type;
+
+        if(!("requestPictureInPicture" in HTMLVideoElement.prototype)) {
+            throw tr("Picture in picture isn't supported");
+        }
+
+        const stream = this.getBroadcastStream(type);
+        if(!stream) {
+            throw tr("Missing video stream");
+        }
+
+        const element = document.createElement("video");
+        element.srcObject = stream;
+        element.muted = true;
+        element.style.position = "absolute";
+        element.style.top = "-1000000px";
+
+        this.pipElement?.remove();
+        this.pipElement = element;
+        this.pipBroadcastType = type;
+
+        try {
+            document.body.appendChild(element);
+
+            try {
+                await new Promise((resolve, reject) => {
+                    element.onloadedmetadata = resolve;
+                    element.onerror = reject;
+                });
+            } catch (error) {
+                throw tr("Failed to load video meta data");
+            } finally {
+                element.onloadedmetadata = undefined;
+                element.onerror = undefined;
+            }
+
+            try {
+                await (element as any).requestPictureInPicture();
+            } catch(error) {
+                throw error;
+            }
+
+            const cancelAutoplay = makeVideoAutoplay(element);
+            element.addEventListener('leavepictureinpicture', () => {
+                cancelAutoplay();
+                element.remove();
+                if(this.pipElement === element) {
+                    this.pipElement = undefined;
+                    this.pipBroadcastType = undefined;
+                }
+            });
+        } catch(error) {
+            element.remove();
+            if(this.pipElement === element) {
+                this.pipElement = undefined;
+                this.pipBroadcastType = undefined;
+            }
+            throw error;
+        }
+    }
 }
 
 const kLocalBroadcastChannels: VideoBroadcastType[] = ["screen", "camera"];
@@ -467,6 +562,20 @@ class ChannelVideoController {
             controller.notifyVideoStream(event.broadcastType);
         });
 
+        this.events.on("action_set_pip", async event => {
+            const controller = this.findVideoById(event.videoId);
+            if (!controller) {
+                logWarn(LogCategory.VIDEO, tr("Tried to enable video pip for a non existing video id (%s)."), event.videoId);
+                return;
+            }
+
+            try {
+                await controller.showPip(event.broadcastType);
+            } catch (error) {
+                createErrorModal(tr("Failed to start PIP"), tra("Failed to popout video:\n{}", error)).open();
+            }
+        });
+
         const channelTree = this.connection.channelTree;
         events.push(channelTree.events.on("notify_tree_reset", () => {
             this.resetClientVideos();

shared/js/ui/frames/video/Definitions.ts

@@ -1,5 +1,6 @@
 import {ClientIcon} from "svg-sprites/client-icons";
 import {VideoBroadcastType} from "tc-shared/connection/VideoConnection";
+import {LogCategory, logWarn} from "tc-shared/log";
 
 export const kLocalVideoId = "__local__video__";
 export const kLocalBroadcastChannels: VideoBroadcastType[] = ["screen", "camera"];
@@ -71,6 +72,7 @@ export interface ChannelVideoEvents {
     action_set_spotlight: { videoId: string | undefined, expend: boolean },
     action_focus_spotlight: {},
     action_set_fullscreen: { videoId: string | undefined },
+    action_set_pip: { videoId: string | undefined, broadcastType: VideoBroadcastType },
     action_toggle_mute: { videoId: string, broadcastType: VideoBroadcastType | undefined, muted: boolean },
     action_dismiss: { videoId: string, broadcastType: VideoBroadcastType },
 
@@ -121,4 +123,46 @@ export interface ChannelVideoEvents {
     notify_subscribe_info: {
         info: VideoSubscribeInfo
     }
+}
+
+export function makeVideoAutoplay(video: HTMLVideoElement) : () => void {
+    let replayTimeout;
+
+    video.autoplay = true;
+
+    const executePlay = () => {
+        if(replayTimeout) {
+            return;
+        }
+
+        video.play().then(undefined).catch(() => {
+            logWarn(LogCategory.VIDEO, tr("Failed to start video replay. Retrying in 500ms intervals."));
+            replayTimeout = setInterval(() => {
+                video.play().then(() => {
+                    clearInterval(replayTimeout);
+                    replayTimeout = undefined;
+                }).catch(() => {});
+            });
+        });
+    };
+
+    const listenerPause = () => {
+        logWarn(LogCategory.VIDEO, tr("Video replay paused. Executing play again."));
+        executePlay();
+    };
+
+    const listenerEnded = () => {
+        logWarn(LogCategory.VIDEO, tr("Video replay ended. Executing play again."));
+        executePlay();
+    };
+
+    video.addEventListener("pause", listenerPause);
+    video.addEventListener("ended", listenerEnded);
+    executePlay();
+
+    return () => {
+        clearTimeout(replayTimeout);
+        video.removeEventListener("pause", listenerPause);
+        video.removeEventListener("ended", listenerEnded);
+    };
 }

shared/js/ui/frames/video/Renderer.tsx

@@ -7,7 +7,7 @@ import {
     ChannelVideoEvents,
     ChannelVideoInfo,
     ChannelVideoStreamState,
-    kLocalVideoId, VideoStreamState, VideoSubscribeInfo
+    kLocalVideoId, makeVideoAutoplay, VideoStreamState, VideoSubscribeInfo
 } from "tc-shared/ui/frames/video/Definitions";
 import {Translatable} from "tc-shared/ui/react-elements/i18n";
 import {LoadingDots} from "tc-shared/ui/react-elements/LoadingDots";
@@ -17,6 +17,7 @@ import {LogCategory, logWarn} from "tc-shared/log";
 import {spawnContextMenu} from "tc-shared/ui/ContextMenu";
 import {VideoBroadcastType} from "tc-shared/connection/VideoConnection";
 import {ErrorBoundary} from "tc-shared/ui/react-elements/ErrorBoundary";
+import {useTr} from "tc-shared/ui/react-elements/Helper";
 
 const SubscribeContext = React.createContext<VideoSubscribeInfo>(undefined);
 const EventContext = React.createContext<Registry<ChannelVideoEvents>>(undefined);
@@ -83,44 +84,17 @@ const VideoInfo = React.memo((props: { videoId: string }) => {
     );
 });
 
-const VideoStreamReplay = React.memo((props: { stream: MediaStream | undefined, className: string, title: string }) => {
+const VideoStreamReplay = React.memo((props: { stream: MediaStream | undefined, className: string, streamType: VideoBroadcastType }) => {
     const refVideo = useRef<HTMLVideoElement>();
-    const refReplayTimeout = useRef<number>();
 
     useEffect(() => {
+        let cancelAutoplay;
         const video = refVideo.current;
         if(props.stream) {
             video.style.opacity = "1";
             video.srcObject = props.stream;
-            video.autoplay = true;
             video.muted = true;
-
+            cancelAutoplay = makeVideoAutoplay(video);
-            const executePlay = () => {
-                if(refReplayTimeout.current) {
-                    return;
-                }
-
-                video.play().then(undefined).catch(() => {
-                    logWarn(LogCategory.VIDEO, tr("Failed to start video replay. Retrying in 500ms intervals."));
-                    refReplayTimeout.current = setInterval(() => {
-                        video.play().then(() => {
-                            clearInterval(refReplayTimeout.current);
-                            refReplayTimeout.current = undefined;
-                        }).catch(() => {});
-                    });
-                });
-            };
-
-            video.onpause = () => {
-                logWarn(LogCategory.VIDEO, tr("Video replay paused. Executing play again."));
-                executePlay();
-            }
-
-            video.onended = () => {
-                logWarn(LogCategory.VIDEO, tr("Video replay ended. Executing play again."));
-                executePlay();
-            }
-            executePlay();
         } else {
             video.style.opacity = "0";
         }
@@ -132,13 +106,21 @@ const VideoStreamReplay = React.memo((props: { stream: MediaStream | undefined,
                 video.onended = undefined;
             }
 
-            clearInterval(refReplayTimeout.current);
+            if(cancelAutoplay) {
-            refReplayTimeout.current = undefined;
+                cancelAutoplay();
+            }
         }
     }, [ props.stream ]);
 
+    let title;
+    if(props.streamType === "camera") {
+        title = useTr("Camera");
+    } else {
+        title = useTr("Screen");
+    }
+
     return (
-        <video ref={refVideo} className={cssStyle.video + " " + props.className} title={props.title} />
+        <video ref={refVideo} className={cssStyle.video + " " + props.className} title={title} x-stream-type={props.streamType} />
     )
 });
 
@@ -265,7 +247,7 @@ const VideoStreamRenderer = (props: { videoId: string, streamType: VideoBroadcas
             );
 
         case "connected":
-            return <VideoStreamReplay stream={state.stream} className={props.className} title={props.streamType === "camera" ? tr("Camera") : tr("Screen")} key={"connected"} />;
+            return <VideoStreamReplay stream={state.stream} className={props.className} streamType={props.streamType} key={"connected"} />;
 
         case "failed":
             return (
@@ -465,11 +447,22 @@ const VideoContainer = React.memo((props: { videoId: string, isSpotlight: boolea
                 }
             }}
             onContextMenu={event => {
+                const streamType = (event.target as HTMLElement).getAttribute("x-stream-type");
+
                 event.preventDefault();
                 spawnContextMenu({
                     pageY: event.pageY,
                     pageX: event.pageX
                 }, [
+                    {
+                        type: "normal",
+                        label: tr("Popout Video"),
+                        icon: ClientIcon.Fullscreen,
+                        click: () => {
+                            events.fire("action_set_pip", { videoId: props.videoId, broadcastType: streamType as any });
+                        },
+                        visible: !!streamType && "requestPictureInPicture" in HTMLVideoElement.prototype
+                    },
                     {
                         type: "normal",
                         label: isFullscreen ? tr("Release fullscreen") : tr("Show in fullscreen"),