Video playback (#4878)

* organizing

* rm Tmp

* cleanup

* add example back

* cleaning

* useKeyEvents hook

* after load handling

* fix import

* enable timeline

* default to true

* refreshers

app/packages/core/src/components/Modal/ImaVidLooker.tsx

@@ -17,13 +17,14 @@ import React, {
 import { useErrorHandler } from "react-error-boundary";
 import { useRecoilValue, useSetRecoilState } from "recoil";
 import { v4 as uuid } from "uuid";
-import { useInitializeImaVidSubscriptions, useModalContext } from "./hooks";
+import { useClearSelectedLabels, useShowOverlays } from "./ModalLooker";
 import {
-  shortcutToHelpItems,
-  useClearSelectedLabels,
+  useInitializeImaVidSubscriptions,
   useLookerOptionsUpdate,
-  useShowOverlays,
-} from "./ModalLooker";
+  useModalContext,
+} from "./hooks";
+import useKeyEvents from "./use-key-events";
+import { shortcutToHelpItems } from "./utils";
 
 interface ImaVidLookerReactProps {
   sample: fos.ModalSample;
@@ -132,19 +133,7 @@ export const ImaVidLookerReact = React.memo(
 
     useEventHandler(looker, "clear", useClearSelectedLabels());
 
-    const hoveredSample = useRecoilValue(fos.hoveredSample);
-
-    useEffect(() => {
-      const hoveredSampleId = hoveredSample?._id;
-      looker.updater((state) => ({
-        ...state,
-        // todo: always setting it to true might not be wise
-        shouldHandleKeyEvents: true,
-        options: {
-          ...state.options,
-        },
-      }));
-    }, [hoveredSample, sample, looker]);
+    useKeyEvents(initialRef, sample._id, looker);
 
     const ref = useRef<HTMLDivElement>(null);
     useEffect(() => {

app/packages/core/src/components/Modal/ModalLooker.tsx

@@ -1,35 +1,12 @@
 import { useTheme } from "@fiftyone/components";
-import { AbstractLooker } from "@fiftyone/looker";
-import { BaseState } from "@fiftyone/looker/src/state";
+import type { ImageLooker } from "@fiftyone/looker";
 import * as fos from "@fiftyone/state";
-import { useEventHandler, useOnSelectLabel } from "@fiftyone/state";
-import React, { useEffect, useMemo, useRef, useState } from "react";
-import { useErrorHandler } from "react-error-boundary";
+import React, { useEffect, useMemo } from "react";
 import { useRecoilCallback, useRecoilValue, useSetRecoilState } from "recoil";
-import { v4 as uuid } from "uuid";
-import { useModalContext } from "./hooks";
 import { ImaVidLookerReact } from "./ImaVidLooker";
-
-export const useLookerOptionsUpdate = () => {
-  return useRecoilCallback(
-    ({ snapshot, set }) =>
-      async (update: object, updater?: (updated: {}) => void) => {
-        const currentOptions = await snapshot.getPromise(
-          fos.savedLookerOptions
-        );
-
-        const panels = await snapshot.getPromise(fos.lookerPanels);
-        const updated = {
-          ...currentOptions,
-          ...update,
-          showJSON: panels.json.isOpen,
-          showHelp: panels.help.isOpen,
-        };
-        set(fos.savedLookerOptions, updated);
-        if (updater) updater(updated);
-      }
-  );
-};
+import { VideoLookerReact } from "./VideoLooker";
+import { useModalContext } from "./hooks";
+import useLooker from "./use-looker";
 
 export const useShowOverlays = () => {
   return useRecoilCallback(({ set }) => async (event: CustomEvent) => {
@@ -47,137 +24,40 @@ export const useClearSelectedLabels = () => {
 };
 
 interface LookerProps {
-  sample?: fos.ModalSample;
-  onClick?: React.MouseEventHandler<HTMLDivElement>;
+  sample: fos.ModalSample;
 }
 
-const ModalLookerNoTimeline = React.memo(
-  ({ sample: sampleDataWithExtraParams }: LookerProps) => {
-    const [id] = useState(() => uuid());
-    const colorScheme = useRecoilValue(fos.colorScheme);
-
-    const { sample } = sampleDataWithExtraParams;
-
-    const theme = useTheme();
-    const initialRef = useRef<boolean>(true);
-    const lookerOptions = fos.useLookerOptions(true);
-    const [reset, setReset] = useState(false);
-    const selectedMediaField = useRecoilValue(fos.selectedMediaField(true));
-    const setModalLooker = useSetRecoilState(fos.modalLooker);
-
-    const createLooker = fos.useCreateLooker(true, false, {
-      ...lookerOptions,
-    });
-
-    const { setActiveLookerRef } = useModalContext();
-
-    const looker = React.useMemo(
-      () => createLooker.current(sampleDataWithExtraParams),
-      [reset, createLooker, selectedMediaField]
-    ) as AbstractLooker<BaseState>;
-
-    useEffect(() => {
-      setModalLooker(looker);
-    }, [looker]);
-
-    useEffect(() => {
-      if (looker) {
-        setActiveLookerRef(looker as fos.Lookers);
-      }
-    }, [looker]);
-
-    useEffect(() => {
-      !initialRef.current && looker.updateOptions(lookerOptions);
-    }, [lookerOptions]);
-
-    useEffect(() => {
-      !initialRef.current && looker.updateSample(sample);
-    }, [sample, colorScheme]);
-
-    useEffect(() => {
-      return () => looker?.destroy();
-    }, [looker]);
-
-    const handleError = useErrorHandler();
+const ModalLookerNoTimeline = React.memo((props: LookerProps) => {
+  const { id, looker, ref } = useLooker<ImageLooker>(props);
+  const theme = useTheme();
+  const setModalLooker = useSetRecoilState(fos.modalLooker);
 
-    const updateLookerOptions = useLookerOptionsUpdate();
-    useEventHandler(looker, "options", (e) => updateLookerOptions(e.detail));
-    useEventHandler(looker, "showOverlays", useShowOverlays());
-    useEventHandler(looker, "reset", () => {
-      setReset((c) => !c);
-    });
+  const { setActiveLookerRef } = useModalContext();
 
-    const jsonPanel = fos.useJSONPanel();
-    const helpPanel = fos.useHelpPanel();
+  useEffect(() => {
+    setModalLooker(looker);
+  }, [looker, setModalLooker]);
 
-    useEventHandler(looker, "select", useOnSelectLabel());
-    useEventHandler(looker, "error", (event) => handleError(event.detail));
-    useEventHandler(
-      looker,
-      "panels",
-      async ({ detail: { showJSON, showHelp, SHORTCUTS } }) => {
-        if (showJSON) {
-          jsonPanel[showJSON](sample);
-        }
-        if (showHelp) {
-          if (showHelp == "close") {
-            helpPanel.close();
-          } else {
-            helpPanel[showHelp](shortcutToHelpItems(SHORTCUTS));
-          }
-        }
-
-        updateLookerOptions({}, (updatedOptions) =>
-          looker.updateOptions(updatedOptions)
-        );
-      }
-    );
-
-    useEffect(() => {
-      initialRef.current = false;
-    }, []);
-
-    useEffect(() => {
-      looker.attach(id);
-    }, [looker, id]);
-
-    useEventHandler(looker, "clear", useClearSelectedLabels());
-
-    const hoveredSample = useRecoilValue(fos.hoveredSample);
-
-    useEffect(() => {
-      const hoveredSampleId = hoveredSample?._id;
-      looker.updater((state) => ({
-        ...state,
-        shouldHandleKeyEvents: hoveredSampleId === sample._id,
-        options: {
-          ...state.options,
-        },
-      }));
-    }, [hoveredSample, sample, looker]);
-
-    const ref = useRef<HTMLDivElement>(null);
-    useEffect(() => {
-      ref.current?.dispatchEvent(
-        new CustomEvent(`looker-attached`, { bubbles: true })
-      );
-    }, [ref]);
-
-    return (
-      <div
-        ref={ref}
-        id={id}
-        data-cy="modal-looker-container"
-        style={{
-          width: "100%",
-          height: "100%",
-          background: theme.background.level2,
-          position: "relative",
-        }}
-      />
-    );
-  }
-);
+  useEffect(() => {
+    if (looker) {
+      setActiveLookerRef(looker as fos.Lookers);
+    }
+  }, [looker, setActiveLookerRef]);
+
+  return (
+    <div
+      ref={ref}
+      id={id}
+      data-cy="modal-looker-container"
+      style={{
+        width: "100%",
+        height: "100%",
+        background: theme.background.level2,
+        position: "relative",
+      }}
+    />
+  );
+});
 
 export const ModalLooker = React.memo(
   ({ sample: propsSampleData }: LookerProps) => {
@@ -197,21 +77,16 @@ export const ModalLooker = React.memo(
     const shouldRenderImavid = useRecoilValue(
       fos.shouldRenderImaVidLooker(true)
     );
+    const video = useRecoilValue(fos.isVideoDataset);
 
     if (shouldRenderImavid) {
       return <ImaVidLookerReact sample={sample} />;
     }
 
+    if (video) {
+      return <VideoLookerReact sample={sample} />;
+    }
+
     return <ModalLookerNoTimeline sample={sample} />;
   }
 );
-
-export function shortcutToHelpItems(SHORTCUTS) {
-  return Object.values(
-    Object.values(SHORTCUTS).reduce((acc, v) => {
-      acc[v.shortcut] = v;
-
-      return acc;
-    }, {})
-  );
-}

app/packages/core/src/components/Modal/VideoLooker.tsx

@@ -0,0 +1,80 @@
+import { useTheme } from "@fiftyone/components";
+import type { VideoLooker } from "@fiftyone/looker";
+import { getFrameNumber } from "@fiftyone/looker";
+import {
+  useCreateTimeline,
+  useDefaultTimelineNameImperative,
+  useTimeline,
+} from "@fiftyone/playback";
+import * as fos from "@fiftyone/state";
+import React, { useEffect, useMemo, useState } from "react";
+import useLooker from "./use-looker";
+
+interface VideoLookerReactProps {
+  sample: fos.ModalSample;
+}
+
+export const VideoLookerReact = (props: VideoLookerReactProps) => {
+  const theme = useTheme();
+  const { id, looker, sample } = useLooker<VideoLooker>(props);
+  const [totalFrames, setTotalFrames] = useState<number>();
+  const frameRate = useMemo(() => {
+    return sample.frameRate;
+  }, [sample]);
+
+  useEffect(() => {
+    const load = () => {
+      const duration = looker.getVideo().duration;
+      setTotalFrames(getFrameNumber(duration, duration, frameRate));
+      looker.removeEventListener("load", load);
+    };
+    looker.addEventListener("load", load);
+  }, [frameRate, looker]);
+
+  return (
+    <>
+      <div
+        id={id}
+        data-cy="modal-looker-container"
+        style={{
+          width: "100%",
+          height: "100%",
+          background: theme.background.level2,
+          position: "relative",
+        }}
+      />
+      {totalFrames !== undefined && (
+        <TimelineController looker={looker} totalFrames={totalFrames} />
+      )}
+    </>
+  );
+};
+
+const TimelineController = ({
+  looker,
+  totalFrames,
+}: {
+  looker: VideoLooker;
+  totalFrames: number;
+}) => {
+  const { getName } = useDefaultTimelineNameImperative();
+  const timelineName = React.useMemo(() => getName(), [getName]);
+
+  useCreateTimeline({
+    name: timelineName,
+    config: totalFrames
+      ? {
+          totalFrames,
+          loop: true,
+        }
+      : undefined,
+    optOutOfAnimation: true,
+  });
+
+  const { pause, play } = useTimeline(timelineName);
+
+  fos.useEventHandler(looker, "pause", pause);
+  fos.useEventHandler(looker, "play", play);
+
+  return null;
+};

app/packages/core/src/components/Modal/hooks.ts

@@ -19,6 +19,27 @@ export const useLookerHelpers = () => {
   };
 };
 
+export const useLookerOptionsUpdate = () => {
+  return useRecoilCallback(
+    ({ snapshot, set }) =>
+      async (update: object, updater?: (updated: {}) => void) => {
+        const currentOptions = await snapshot.getPromise(
+          fos.savedLookerOptions
+        );
+
+        const panels = await snapshot.getPromise(fos.lookerPanels);
+        const updated = {
+          ...currentOptions,
+          ...update,
+          showJSON: panels.json.isOpen,
+          showHelp: panels.help.isOpen,
+        };
+        set(fos.savedLookerOptions, updated);
+        if (updater) updater(updated);
+      }
+  );
+};
+
 export const useInitializeImaVidSubscriptions = () => {
   const subscribeToImaVidStateChanges = useRecoilCallback(
     ({ set }) =>