diff --git a/e2e/playwright/snapshot-tests.spec.ts b/e2e/playwright/snapshot-tests.spec.ts index 81f05c480d..bbf86110cd 100644 --- a/e2e/playwright/snapshot-tests.spec.ts +++ b/e2e/playwright/snapshot-tests.spec.ts @@ -766,7 +766,7 @@ test.describe('Grid visibility', { tag: '@snapshot' }, () => { }) }) -test('theme persists', async ({ page, context }) => { +test('theme persists', async ({ page, context, homePage }) => { const u = await getUtils(page) await context.addInitScript(async () => { localStorage.setItem( @@ -784,7 +784,7 @@ test('theme persists', async ({ page, context }) => { await page.setViewportSize({ width: 1200, height: 500 }) - await u.waitForAuthSkipAppStart() + await homePage.goToModelingScene() await page.waitForTimeout(500) // await page.getByRole('link', { name: 'Settings Settings (tooltip)' }).click() @@ -812,7 +812,7 @@ test('theme persists', async ({ page, context }) => { // Disconnect and reconnect to check the theme persists through a reload // Expect the network to be down - await expect(networkToggle).toContainText('Offline') + await expect(networkToggle).toContainText('Problem') // simulate network up await u.emulateNetworkConditions({ diff --git a/e2e/playwright/snapshot-tests.spec.ts-snapshots/theme-persists-1.png b/e2e/playwright/snapshot-tests.spec.ts-snapshots/theme-persists-1.png new file mode 100644 index 0000000000..1907ac24e7 Binary files /dev/null and b/e2e/playwright/snapshot-tests.spec.ts-snapshots/theme-persists-1.png differ diff --git a/e2e/playwright/test-network-and-connection-issues.spec.ts b/e2e/playwright/test-network-and-connection-issues.spec.ts index fab50e8d0a..663c785b42 100644 --- a/e2e/playwright/test-network-and-connection-issues.spec.ts +++ b/e2e/playwright/test-network-and-connection-issues.spec.ts @@ -1,259 +1,363 @@ import type { EngineCommand } from '@src/lang/std/artifactGraph' import { uuidv4 } from '@src/lib/utils' -import { commonPoints, getUtils } from '@e2e/playwright/test-utils' +import { + commonPoints, + getUtils, + TEST_COLORS, + circleMove, +} from '@e2e/playwright/test-utils' import { expect, test } from '@e2e/playwright/zoo-test' -test.describe( - 'Test network and connection issues', - { - tag: ['@macos', '@windows'], - }, - () => { - test( - 'simulate network down and network little widget', - { tag: '@skipLocalEngine' }, - async ({ page, homePage }) => { - const u = await getUtils(page) - await page.setBodyDimensions({ width: 1200, height: 500 }) - - await homePage.goToModelingScene() - - const networkToggle = page.getByTestId('network-toggle') - - // This is how we wait until the stream is online - await expect( - page.getByRole('button', { name: 'Start Sketch' }) - ).not.toBeDisabled({ timeout: 15000 }) - - const networkWidget = page.locator('[data-testid="network-toggle"]') - await expect(networkWidget).toBeVisible() - await networkWidget.hover() - - const networkPopover = page.locator('[data-testid="network-popover"]') - await expect(networkPopover).not.toBeVisible() - - // (First check) Expect the network to be up - await expect(networkToggle).toContainText('Connected') - - // Click the network widget - await networkWidget.click() - - // Check the modal opened. - await expect(networkPopover).toBeVisible() - - // Click off the modal. - await page.mouse.click(100, 100) - await expect(networkPopover).not.toBeVisible() - - // Turn off the network - await u.emulateNetworkConditions({ - offline: true, - // values of 0 remove any active throttling. crbug.com/456324#c9 - latency: 0, - downloadThroughput: -1, - uploadThroughput: -1, - }) - - // Expect the network to be down - await expect(networkToggle).toContainText('Problem') - - // Click the network widget - await networkWidget.click() - - // Check the modal opened. - await expect(networkPopover).toBeVisible() - - // Click off the modal. - await page.mouse.click(0, 0) - await expect(networkPopover).not.toBeVisible() - - // Turn back on the network - await u.emulateNetworkConditions({ - offline: false, - // values of 0 remove any active throttling. crbug.com/456324#c9 - latency: 0, - downloadThroughput: -1, - uploadThroughput: -1, - }) - - await expect( - page.getByRole('button', { name: 'Start Sketch' }) - ).not.toBeDisabled({ timeout: 15000 }) +test.describe('Test network related behaviors', () => { + test( + 'simulate network down and network little widget', + { tag: '@skipLocalEngine' }, + async ({ page, homePage }) => { + const networkToggleConnectedText = page.getByText('Connected') + const networkToggleWeakText = page.getByText('Network health (Weak)') + + const u = await getUtils(page) + await page.setBodyDimensions({ width: 1200, height: 500 }) + + await homePage.goToModelingScene() + + const networkToggle = page.getByTestId('network-toggle') + + // This is how we wait until the stream is online + await expect( + page.getByRole('button', { name: 'Start Sketch' }) + ).not.toBeDisabled({ timeout: 15000 }) + + const networkWidget = page.locator('[data-testid="network-toggle"]') + await expect(networkWidget).toBeVisible() + await networkWidget.hover() + + const networkPopover = page.locator('[data-testid="network-popover"]') + await expect(networkPopover).not.toBeVisible() + + // (First check) Expect the network to be up + await expect( + networkToggleConnectedText.or(networkToggleWeakText) + ).toBeVisible() + + // Click the network widget + await networkWidget.click() + + // Check the modal opened. + await expect(networkPopover).toBeVisible() + + // Click off the modal. + await page.mouse.click(100, 100) + await expect(networkPopover).not.toBeVisible() + + // Turn off the network + await u.emulateNetworkConditions({ + offline: true, + // values of 0 remove any active throttling. crbug.com/456324#c9 + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1, + }) + + // Expect the network to be down + await expect(networkToggle).toContainText('Problem') + + // Click the network widget + await networkWidget.click() + + // Check the modal opened. + await expect(networkPopover).toBeVisible() + + // Click off the modal. + await page.mouse.click(0, 0) + await expect(networkPopover).not.toBeVisible() + + // Turn back on the network + await u.emulateNetworkConditions({ + offline: false, + // values of 0 remove any active throttling. crbug.com/456324#c9 + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1, + }) + + await expect( + page.getByRole('button', { name: 'Start Sketch' }) + ).not.toBeDisabled({ timeout: 15000 }) + + // (Second check) expect the network to be up + await expect( + networkToggleConnectedText.or(networkToggleWeakText) + ).toBeVisible() + } + ) + + test( + 'Engine disconnect & reconnect in sketch mode', + { tag: '@skipLocalEngine' }, + async ({ page, homePage, toolbar, scene, cmdBar }) => { + const networkToggle = page.getByTestId('network-toggle') + const networkToggleConnectedText = page.getByText('Connected') + const networkToggleWeakText = page.getByText('Network health (Weak)') + + const u = await getUtils(page) + await page.setBodyDimensions({ width: 1200, height: 500 }) + const PUR = 400 / 37.5 //pixeltoUnitRatio + + await homePage.goToModelingScene() + await u.waitForPageLoad() + + await u.openDebugPanel() + // click on "Start Sketch" button + await u.clearCommandLogs() + await page.getByRole('button', { name: 'Start Sketch' }).click() + await page.waitForTimeout(100) + + // select a plane + await page.mouse.click(700, 200) + + await expect(page.locator('.cm-content')).toHaveText( + `@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)` + ) + await u.closeDebugPanel() + + await page.waitForTimeout(500) // TODO detect animation ending, or disable animation + + const startXPx = 600 + await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10) + await expect(page.locator('.cm-content')).toHaveText( + `@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})` + ) + await page.waitForTimeout(100) + + await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10) + await page.waitForTimeout(100) + + await expect( + page.locator('.cm-content') + ).toHaveText(`@settings(defaultLengthUnit = in)sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt}) + |> xLine(length = ${commonPoints.num1})`) - // (Second check) expect the network to be up - await expect(networkToggle).toContainText('Connected') + // Expect the network to be up + await networkToggle.hover() + await expect( + networkToggleConnectedText.or(networkToggleWeakText) + ).toBeVisible() + + // simulate network down + await u.emulateNetworkConditions({ + offline: true, + // values of 0 remove any active throttling. crbug.com/456324#c9 + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1, + }) + + // Expect the network to be down + await networkToggle.hover() + await expect(networkToggle).toContainText('Problem') + + // Ensure we are not in sketch mode + await expect( + page.getByRole('button', { name: 'Exit Sketch' }) + ).not.toBeVisible() + await expect( + page.getByRole('button', { name: 'Start Sketch' }) + ).toBeVisible() + + // simulate network up + await u.emulateNetworkConditions({ + offline: false, + // values of 0 remove any active throttling. crbug.com/456324#c9 + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1, + }) + + // Wait for the app to be ready for use + await expect( + page.getByRole('button', { name: 'Start Sketch' }) + ).not.toBeDisabled({ timeout: 15000 }) + + // Expect the network to be up + await networkToggle.hover() + await expect( + networkToggleConnectedText.or(networkToggleWeakText) + ).toBeVisible() + + await scene.settled(cmdBar) + + // Click off the code pane. + await page.mouse.click(100, 100) + + // select a line + await page + .getByText(`startProfile(sketch001, at = ${commonPoints.startAt})`) + .click() + + // enter sketch again + await toolbar.editSketch() + + // Click the line tool + await page.getByRole('button', { name: 'line Line', exact: true }).click() + + await page.waitForTimeout(150) + + const camCommand: EngineCommand = { + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd: { + type: 'default_camera_look_at', + center: { x: 109, y: 0, z: -152 }, + vantage: { x: 115, y: -505, z: -152 }, + up: { x: 0, y: 0, z: 1 }, + }, } - ) - - test( - 'Engine disconnect & reconnect in sketch mode', - { tag: '@skipLocalEngine' }, - async ({ page, homePage, toolbar, scene, cmdBar }) => { - const networkToggle = page.getByTestId('network-toggle') + const updateCamCommand: EngineCommand = { + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd: { + type: 'default_camera_get_settings', + }, + } + await toolbar.openPane('debug') + await u.sendCustomCmd(camCommand) + await page.waitForTimeout(100) + await u.sendCustomCmd(updateCamCommand) + await page.waitForTimeout(100) + + // click to continue profile + await page.mouse.click(1007, 400) + await page.waitForTimeout(100) + // Ensure we can continue sketching + await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20) + await expect + .poll(u.normalisedEditorCode) + .toBe(`@settings(defaultLengthUnit = in) + + +sketch001 = startSketchOn(XZ) +profile001 = startProfile(sketch001, at = [12.34, -12.34]) + |> xLine(length = 12.34) + |> line(end = [-12.34, 12.34]) - const u = await getUtils(page) - await page.setBodyDimensions({ width: 1200, height: 500 }) - const PUR = 400 / 37.5 //pixeltoUnitRatio +`) + await page.waitForTimeout(100) + await page.mouse.click(startXPx, 500 - PUR * 20) - await homePage.goToModelingScene() - await u.waitForPageLoad() + await expect + .poll(u.normalisedEditorCode) + .toBe(`@settings(defaultLengthUnit = in) - await u.openDebugPanel() - // click on "Start Sketch" button - await u.clearCommandLogs() - await page.getByRole('button', { name: 'Start Sketch' }).click() - await page.waitForTimeout(100) - // select a plane - await page.mouse.click(700, 200) +sketch001 = startSketchOn(XZ) +profile001 = startProfile(sketch001, at = [12.34, -12.34]) + |> xLine(length = 12.34) + |> line(end = [-12.34, 12.34]) + |> xLine(length = -12.34) - await expect(page.locator('.cm-content')).toHaveText( - `sketch001 = startSketchOn(XZ)` - ) - await u.closeDebugPanel() +`) - await page.waitForTimeout(500) // TODO detect animation ending, or disable animation + // Unequip line tool + await page.keyboard.press('Escape') + // Make sure we didn't pop out of sketch mode. + await expect( + page.getByRole('button', { name: 'Exit Sketch' }) + ).toBeVisible() + await expect( + page.getByRole('button', { name: 'line Line', exact: true }) + ).not.toHaveAttribute('aria-pressed', 'true') + + // Exit sketch + await page.keyboard.press('Escape') + await expect( + page.getByRole('button', { name: 'Exit Sketch' }) + ).not.toBeVisible() + } + ) + + test( + 'Paused stream freezes view frame, unpause reconnect is seamless to user', + { tag: ['@electron', '@skipLocalEngine'] }, + async ({ page, homePage, scene, cmdBar, toolbar, tronApp }) => { + const networkToggle = page.getByTestId('network-toggle') + const networkToggleConnectedText = page.getByText('Connected') + const networkToggleWeakText = page.getByText('Network health (Weak)') + + if (!tronApp) { + fail() + } - const startXPx = 600 - await page.mouse.click(startXPx + PUR * 10, 500 - PUR * 10) - await expect(page.locator('.cm-content')).toHaveText( - `sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt})` + await tronApp.cleanProjectDir({ + app: { + stream_idle_mode: 5000, + }, + }) + + await page.addInitScript(async () => { + localStorage.setItem( + 'persistCode', + `sketch001 = startSketchOn(XY) +profile001 = startProfile(sketch001, at = [0.0, 0.0]) + |> line(end = [10.0, 0]) + |> line(end = [0, 10.0]) + |> close()` ) - await page.waitForTimeout(100) - - await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 10) - await page.waitForTimeout(100) - - await expect( - page.locator('.cm-content') - ).toHaveText(`sketch001 = startSketchOn(XZ)profile001 = startProfile(sketch001, at = ${commonPoints.startAt}) - |> xLine(length = ${commonPoints.num1})`) + }) - // Expect the network to be up - await expect(networkToggle).toContainText('Connected') + const dim = { width: 1200, height: 500 } + await page.setBodyDimensions(dim) - // simulate network down - await u.emulateNetworkConditions({ - offline: true, - // values of 0 remove any active throttling. crbug.com/456324#c9 - latency: 0, - downloadThroughput: -1, - uploadThroughput: -1, - }) + await test.step('Go to modeling scene', async () => { + await homePage.goToModelingScene() + await scene.settled(cmdBar) + }) - // Expect the network to be down - await expect(networkToggle).toContainText('Problem') + await test.step('Verify pausing behavior', async () => { + // Wait 5s + 1s to pause. + await page.waitForTimeout(6000) - // Ensure we are not in sketch mode + // We should now be paused. To the user, it should appear we're still + // connected. + await networkToggle.hover() await expect( - page.getByRole('button', { name: 'Exit Sketch' }) - ).not.toBeVisible() - await expect( - page.getByRole('button', { name: 'Start Sketch' }) + networkToggleConnectedText.or(networkToggleWeakText) ).toBeVisible() - // simulate network up - await u.emulateNetworkConditions({ - offline: false, - // values of 0 remove any active throttling. crbug.com/456324#c9 - latency: 0, - downloadThroughput: -1, - uploadThroughput: -1, - }) - - // Wait for the app to be ready for use - await expect( - page.getByRole('button', { name: 'Start Sketch' }) - ).not.toBeDisabled({ timeout: 15000 }) - - // Expect the network to be up - await expect(networkToggle).toContainText('Connected') - await scene.settled(cmdBar) - - // Click off the code pane. - await page.mouse.click(100, 100) - - // select a line - await page - .getByText(`startProfile(sketch001, at = ${commonPoints.startAt})`) - .click() - - // enter sketch again - await toolbar.editSketch() - - // Click the line tool - await page - .getByRole('button', { name: 'line Line', exact: true }) - .click() - - await page.waitForTimeout(150) - - const camCommand: EngineCommand = { - type: 'modeling_cmd_req', - cmd_id: uuidv4(), - cmd: { - type: 'default_camera_look_at', - center: { x: 109, y: 0, z: -152 }, - vantage: { x: 115, y: -505, z: -152 }, - up: { x: 0, y: 0, z: 1 }, - }, + const center = { + x: dim.width / 2, + y: dim.height / 2, } - const updateCamCommand: EngineCommand = { - type: 'modeling_cmd_req', - cmd_id: uuidv4(), - cmd: { - type: 'default_camera_get_settings', - }, - } - await toolbar.openPane('debug') - await u.sendCustomCmd(camCommand) - await page.waitForTimeout(100) - await u.sendCustomCmd(updateCamCommand) - await page.waitForTimeout(100) - - // click to continue profile - await page.mouse.click(1007, 400) - await page.waitForTimeout(100) - // Ensure we can continue sketching - await page.mouse.click(startXPx + PUR * 20, 500 - PUR * 20) - await expect - .poll(u.normalisedEditorCode) - .toBe(`sketch001 = startSketchOn(XZ) -profile001 = startProfile(sketch001, at = [12.34, -12.34]) - |> xLine(length = 12.34) - |> line(end = [-12.34, 12.34]) - -`) - await page.waitForTimeout(100) - await page.mouse.click(startXPx, 500 - PUR * 20) - await expect - .poll(u.normalisedEditorCode) - .toBe(`sketch001 = startSketchOn(XZ) -profile001 = startProfile(sketch001, at = [12.34, -12.34]) - |> xLine(length = 12.34) - |> line(end = [-12.34, 12.34]) - |> xLine(length = -12.34) - -`) - - // Unequip line tool - await page.keyboard.press('Escape') - // Make sure we didn't pop out of sketch mode. + let probe = { x: 0, y: 0 } + + // ... and the model's still visibly there + probe.x = center.x + dim.width / 100 + probe.y = center.y + await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15) + probe = { ...center } + + // Now move the mouse around to unpause! + await circleMove(page, probe.x, probe.y, 20, 10) + + // ONCE AGAIN! Check the view area hasn't changed at all. + // Check the pixel a couple times as it reconnects. + // NOTE: Remember, idle behavior is still on at this point - + // if this test takes longer than 5s shit WILL go south! + probe.x = center.x + dim.width / 100 + probe.y = center.y + await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15) + await page.waitForTimeout(1000) + await scene.expectPixelColor(TEST_COLORS.GREY, probe, 15) + probe = { ...center } + + // Ensure we're still connected + await networkToggle.hover() await expect( - page.getByRole('button', { name: 'Exit Sketch' }) + networkToggleConnectedText.or(networkToggleWeakText) ).toBeVisible() - await expect( - page.getByRole('button', { name: 'line Line', exact: true }) - ).not.toHaveAttribute('aria-pressed', 'true') - - // Exit sketch - await page.keyboard.press('Escape') - await expect( - page.getByRole('button', { name: 'Exit Sketch' }) - ).not.toBeVisible() - } - ) - } -) + }) + } + ) +}) diff --git a/e2e/playwright/test-utils.ts b/e2e/playwright/test-utils.ts index 4a11003729..801956f75c 100644 --- a/e2e/playwright/test-utils.ts +++ b/e2e/playwright/test-utils.ts @@ -44,6 +44,8 @@ export const lowerRightMasks = (page: Page) => [ export type TestColor = [number, number, number] export const TEST_COLORS: { [key: string]: TestColor } = { WHITE: [249, 249, 249], + OFFWHITE: [237, 237, 237], + GREY: [142, 142, 142], YELLOW: [255, 255, 0], BLUE: [0, 0, 255], DARK_MODE_BKGD: [27, 27, 27], diff --git a/src/App.tsx b/src/App.tsx index fbffb4400a..9cb85210ea 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -117,7 +117,9 @@ export function App() { // When leaving the modeling scene, cut the engine stream. return () => { - engineStreamActor.send({ type: EngineStreamTransition.Pause }) + // When leaving the modeling scene, cut the engine stream. + // Stop is more serious than Pause + engineStreamActor.send({ type: EngineStreamTransition.Stop }) } }, []) diff --git a/src/clientSideScene/CameraControls.ts b/src/clientSideScene/CameraControls.ts index e62cb84a48..e42c09b40a 100644 --- a/src/clientSideScene/CameraControls.ts +++ b/src/clientSideScene/CameraControls.ts @@ -975,7 +975,6 @@ export class CameraControls { }, }) } - await this.engineCommandManager.sendSceneCommand({ type: 'modeling_cmd_req', cmd_id: uuidv4(), diff --git a/src/components/EngineStream.tsx b/src/components/EngineStream.tsx index 1c1ce7e862..64c4c16b72 100644 --- a/src/components/EngineStream.tsx +++ b/src/components/EngineStream.tsx @@ -1,3 +1,4 @@ +import { isPlaywright } from '@src/lib/isPlaywright' import { useAppState } from '@src/AppState' import { ClientSideScene } from '@src/clientSideScene/ClientSideSceneComp' import { ViewControlContextMenu } from '@src/components/ViewControlMenu' @@ -5,7 +6,10 @@ import { useModelingContext } from '@src/hooks/useModelingContext' import { useNetworkContext } from '@src/hooks/useNetworkContext' import { NetworkHealthState } from '@src/hooks/useNetworkStatus' import { getArtifactOfTypes } from '@src/lang/std/artifactGraph' -import { EngineCommandManagerEvents } from '@src/lang/std/engineConnection' +import { + EngineCommandManagerEvents, + EngineConnectionStateType, +} from '@src/lang/std/engineConnection' import { btnName } from '@src/lib/cameraControls' import { PATHS } from '@src/lib/paths' import { sendSelectEventToEngine } from '@src/lib/selections' @@ -39,22 +43,38 @@ import { DEFAULT_DEFAULT_LENGTH_UNIT } from '@src/lib/constants' import { createThumbnailPNGOnDesktop } from '@src/lib/screenshot' import type { SettingsViaQueryString } from '@src/lib/settings/settingsTypes' +const TIME_1_SECOND = 1000 + export const EngineStream = (props: { pool: string | null authToken: string | undefined }) => { const { setAppState } = useAppState() - const [firstPlay, setFirstPlay] = useState(true) - - const { overallState } = useNetworkContext() const settings = useSettings() - - const engineStreamState = useSelector(engineStreamActor, (state) => state) + const { state: modelingMachineState, send: modelingMachineActorSend } = + useModelingContext() const { file, project } = useRouteLoaderData(PATHS.FILE) as IndexLoaderData const last = useRef(Date.now()) + + const [firstPlay, setFirstPlay] = useState(true) + const [isRestartRequestStarting, setIsRestartRequestStarting] = + useState(false) + const [attemptTimes, setAttemptTimes] = useState<[number, number]>([ + 0, + TIME_1_SECOND, + ]) + + // These will be passed to the engineStreamActor to handle. + const videoRef = useRef(null) + const canvasRef = useRef(null) + + // For attaching right-click menu events const videoWrapperRef = useRef(null) + const { overallState } = useNetworkContext() + const engineStreamState = useSelector(engineStreamActor, (state) => state) + /** * We omit `pool` here because `engineStreamMachine` will override it anyway * within the `EngineStreamTransition.StartOrReconfigureEngine` Promise actor. @@ -68,19 +88,46 @@ export const EngineStream = (props: { cameraOrbit: settings.modeling.cameraOrbit.current, } - const { state: modelingMachineState, send: modelingMachineActorSend } = - useModelingContext() - const streamIdleMode = settings.app.streamIdleMode.current + useEffect(() => { + engineStreamActor.send({ + type: EngineStreamTransition.SetVideoRef, + videoRef: { current: videoRef.current }, + }) + }, [videoRef.current]) + + useEffect(() => { + engineStreamActor.send({ + type: EngineStreamTransition.SetCanvasRef, + canvasRef: { current: canvasRef.current }, + }) + }, [canvasRef.current]) + + useEffect(() => { + engineStreamActor.send({ + type: EngineStreamTransition.SetPool, + pool: props.pool, + }) + }, [props.pool]) + + useEffect(() => { + engineStreamActor.send({ + type: EngineStreamTransition.SetAuthToken, + authToken: props.authToken, + }) + }, [props.authToken]) + + // We have to call this here because of the dependencies: + // modelingMachineActorSend, setAppState, settingsEngine + // It's possible to pass these in earlier but I (lee) don't want to + // restructure this further at the moment. const startOrReconfigureEngine = () => { engineStreamActor.send({ type: EngineStreamTransition.StartOrReconfigureEngine, modelingMachineActorSend, settings: settingsEngine, setAppState, - - // It's possible a reconnect happens as we drag the window :') onMediaStream(mediaStream: MediaStream) { engineStreamActor.send({ type: EngineStreamTransition.SetMediaStream, @@ -90,18 +137,47 @@ export const EngineStream = (props: { }) } - // When the scene is ready play the stream and execute! + useEffect(() => { + if ( + engineStreamState.value !== EngineStreamState.WaitingForDependencies && + engineStreamState.value !== EngineStreamState.Stopped + ) + return + startOrReconfigureEngine() + }, [engineStreamState, setAppState]) + + // I would inline this but it needs to be a function for removeEventListener. const play = () => { engineStreamActor.send({ type: EngineStreamTransition.Play, }) + } + useEffect(() => { + engineCommandManager.addEventListener( + EngineCommandManagerEvents.SceneReady, + play + ) + return () => { + engineCommandManager.removeEventListener( + EngineCommandManagerEvents.SceneReady, + play + ) + } + }, []) + + // When the scene is ready, execute kcl! + const executeKcl = () => { + console.log('scene is ready, execute kcl') const kmp = kclManager.executeCode().catch(trap) if (!firstPlay) return + setFirstPlay(false) - console.log('scene is ready, fire!') + // Reset the restart timeouts + setAttemptTimes([0, TIME_1_SECOND]) + console.log('firstPlay true, zoom to fit') kmp .then(async () => { // Gotcha: Playwright E2E tests will be zoom_to_fit, when you try to recreate the e2e test manually @@ -142,51 +218,65 @@ export const EngineStream = (props: { useEffect(() => { engineCommandManager.addEventListener( EngineCommandManagerEvents.SceneReady, - play + executeKcl ) return () => { engineCommandManager.removeEventListener( EngineCommandManagerEvents.SceneReady, - play + executeKcl ) } }, [firstPlay]) useEffect(() => { - engineCommandManager.addEventListener( - EngineCommandManagerEvents.SceneReady, - play - ) + // We do a back-off restart, using a fibonacci sequence, since it + // has a nice retry time curve (somewhat quick then exponential) + const attemptRestartIfNecessary = () => { + if (isRestartRequestStarting) return + setIsRestartRequestStarting(true) + setTimeout(() => { + engineStreamState.context.videoRef.current?.pause() + engineCommandManager.tearDown() + startOrReconfigureEngine() + setFirstPlay(false) + setIsRestartRequestStarting(false) + }, attemptTimes[0] + attemptTimes[1]) + setAttemptTimes([attemptTimes[1], attemptTimes[0] + attemptTimes[1]]) + } - engineStreamActor.send({ - type: EngineStreamTransition.SetPool, - data: { pool: props.pool }, - }) - engineStreamActor.send({ - type: EngineStreamTransition.SetAuthToken, - data: { authToken: props.authToken }, - }) + // Poll that we're connected. If not, send a reset signal. + // Do not restart if we're in idle mode. + const connectionCheckIntervalId = setInterval(() => { + // SKIP DURING TESTS BECAUSE IT WILL MESS WITH REUSING THE + // ELECTRON INSTANCE. + if (isPlaywright()) { + return + } - return () => { - engineCommandManager.tearDown() - } - }, []) + // Don't try try to restart if we're already connected! + const hasEngineConnectionInst = engineCommandManager.engineConnection + const isDisconnected = + engineCommandManager.engineConnection?.state.type === + EngineConnectionStateType.Disconnected + const inIdleMode = engineStreamState.value === EngineStreamState.Paused + if ((hasEngineConnectionInst && !isDisconnected) || inIdleMode) return - // In the past we'd try to play immediately, but the proper thing is to way - // for the 'canplay' event to tell us data is ready. - useEffect(() => { - const videoRef = engineStreamState.context.videoRef.current - if (!videoRef) { - return - } - const play = () => { - videoRef.play().catch(console.error) - } - videoRef.addEventListener('canplay', play) + attemptRestartIfNecessary() + }, TIME_1_SECOND) + + engineCommandManager.addEventListener( + EngineCommandManagerEvents.EngineRestartRequest, + attemptRestartIfNecessary + ) return () => { - videoRef.removeEventListener('canplay', play) + clearInterval(connectionCheckIntervalId) + + engineCommandManager.removeEventListener( + EngineCommandManagerEvents.EngineRestartRequest, + attemptRestartIfNecessary + ) } - }, [engineStreamState.context.videoRef.current]) + }, [engineStreamState, attemptTimes, isRestartRequestStarting]) useEffect(() => { if (engineStreamState.value === EngineStreamState.Reconfiguring) return @@ -214,25 +304,6 @@ export const EngineStream = (props: { }).observe(document.body) }, [engineStreamState.value]) - // When the video and canvas element references are set, start the engine. - useEffect(() => { - if ( - engineStreamState.context.canvasRef.current && - engineStreamState.context.videoRef.current - ) { - startOrReconfigureEngine() - } - }, [ - engineStreamState.context.canvasRef.current, - engineStreamState.context.videoRef.current, - ]) - - // On settings change, reconfigure the engine. When paused this gets really tricky, - // and also requires onMediaStream to be set! - useEffect(() => { - startOrReconfigureEngine() - }, Object.values(settingsEngine)) - /** * Subscribe to execute code when the file changes * but only if the scene is already ready. @@ -315,18 +386,7 @@ export const EngineStream = (props: { } if (engineStreamState.value === EngineStreamState.Paused) { - engineStreamActor.send({ - type: EngineStreamTransition.StartOrReconfigureEngine, - modelingMachineActorSend, - settings: settingsEngine, - setAppState, - onMediaStream(mediaStream: MediaStream) { - engineStreamActor.send({ - type: EngineStreamTransition.SetMediaStream, - mediaStream, - }) - }, - }) + startOrReconfigureEngine() } timeoutStart.current = Date.now() @@ -344,7 +404,7 @@ export const EngineStream = (props: { window.document.addEventListener('mouseup', onAnyInput) window.document.addEventListener('scroll', onAnyInput) window.document.addEventListener('touchstart', onAnyInput) - window.document.addEventListener('touchstop', onAnyInput) + window.document.addEventListener('touchend', onAnyInput) return () => { timeoutStart.current = null @@ -355,10 +415,34 @@ export const EngineStream = (props: { window.document.removeEventListener('mouseup', onAnyInput) window.document.removeEventListener('scroll', onAnyInput) window.document.removeEventListener('touchstart', onAnyInput) - window.document.removeEventListener('touchstop', onAnyInput) + window.document.removeEventListener('touchend', onAnyInput) } }, [streamIdleMode, engineStreamState.value]) + // On various inputs save the camera state, in case we get disconnected. + useEffect(() => { + const onInput = () => { + // Save the remote camera state to restore on stream restore. + // Fire-and-forget because we don't know when a camera movement is + // completed on the engine side (there are no responses to data channel + // mouse movements.) + sceneInfra.camControls.saveRemoteCameraState().catch(trap) + } + + // These usually signal a user is done some sort of operation. + window.document.addEventListener('keyup', onInput) + window.document.addEventListener('mouseup', onInput) + window.document.addEventListener('scroll', onInput) + window.document.addEventListener('touchend', onInput) + + return () => { + window.document.removeEventListener('keyup', onInput) + window.document.removeEventListener('mouseup', onInput) + window.document.removeEventListener('scroll', onInput) + window.document.removeEventListener('touchend', onInput) + } + }, []) + const isNetworkOkay = overallState === NetworkHealthState.Ok || overallState === NetworkHealthState.Weak @@ -429,7 +513,7 @@ export const EngineStream = (props: { autoPlay muted key={engineStreamActor.id + 'video'} - ref={engineStreamState.context.videoRef} + ref={videoRef} controls={false} className="w-full cursor-pointer h-full" disablePictureInPicture @@ -437,7 +521,7 @@ export const EngineStream = (props: { /> @@ -454,9 +538,11 @@ export const EngineStream = (props: { } menuTargetElement={videoWrapperRef} /> - {![EngineStreamState.Playing, EngineStreamState.Paused].some( - (s) => s === engineStreamState.value - ) && ( + {![ + EngineStreamState.Playing, + EngineStreamState.Paused, + EngineStreamState.Resuming, + ].some((s) => s === engineStreamState.value) && ( Connecting to engine diff --git a/src/components/Loading.tsx b/src/components/Loading.tsx index 57a1bf19f8..de10e61787 100644 --- a/src/components/Loading.tsx +++ b/src/components/Loading.tsx @@ -138,7 +138,9 @@ const Loading = ({ children, className, dataTestId }: LoadingProps) => { CONNECTION_ERROR_TEXT[error.error] + (error.context ? '\n\nThe error details are: ' + - JSON.stringify(error.context) + (error.context instanceof Object + ? JSON.stringify(error.context) + : error.context) : ''), { renderer: new SafeRenderer(markedOptions), diff --git a/src/components/ModelingMachineProvider.tsx b/src/components/ModelingMachineProvider.tsx index 422ffbf74a..ca31281e8e 100644 --- a/src/components/ModelingMachineProvider.tsx +++ b/src/components/ModelingMachineProvider.tsx @@ -204,12 +204,13 @@ export const ModelingMachineProvider = ({ sceneInfra.camControls.syncDirection = 'engineToClient' + // TODO: Re-evaluate if this pause/play logic is needed. store.videoElement?.pause() return kclManager .executeCode() .then(() => { - if (engineCommandManager.engineConnection?.idleMode) return + if (engineCommandManager.idleMode) return store.videoElement?.play().catch((e) => { console.warn('Video playing was prevented', e) diff --git a/src/lang/std/engineConnection.ts b/src/lang/std/engineConnection.ts index 0a56a8cb8d..724f0b314c 100644 --- a/src/lang/std/engineConnection.ts +++ b/src/lang/std/engineConnection.ts @@ -1,3 +1,4 @@ +import { TEST } from '@src/env' import type { Models } from '@kittycad/lib' import { VITE_KC_API_WS_MODELING_URL, VITE_KC_DEV_TOKEN } from '@src/env' import { jsAppSettings } from '@src/lib/settings/settingsUtils' @@ -7,6 +8,7 @@ import type { MachineManager } from '@src/components/MachineManagerProvider' import type { useModelingContext } from '@src/hooks/useModelingContext' import type { KclManager } from '@src/lang/KclSingleton' import type CodeManager from '@src/lang/codeManager' +import type { SceneInfra } from '@src/clientSideScene/sceneInfra' import type { EngineCommand, ResponseMap } from '@src/lang/std/artifactGraph' import type { CommandLog } from '@src/lang/std/commandLog' import { CommandLogType } from '@src/lang/std/commandLog' @@ -109,6 +111,13 @@ export const CONNECTION_ERROR_TEXT: Record = { 'An unexpected error occurred. Please report this to us.', } +export const WEBSOCKET_READYSTATE_TEXT: Record = { + [WebSocket.CONNECTING]: 'WebSocket.CONNECTING', + [WebSocket.OPEN]: 'WebSocket.OPEN', + [WebSocket.CLOSING]: 'WebSocket.CLOSING', + [WebSocket.CLOSED]: 'WebSocket.CLOSED', +} + export interface ErrorType { // The error we've encountered. error: ConnectionError @@ -208,6 +217,9 @@ export enum EngineConnectionEvents { // We can eventually use it for more, but one step at a time. ConnectionStateChanged = 'connection-state-changed', // (state: EngineConnectionState) => void + // There are various failure scenarios where we want to try a restart. + RestartRequest = 'restart-request', + // These are used for the EngineCommandManager and were created // before onConnectionStateChange existed. ConnectionStarted = 'connection-started', // (engineConnection: EngineConnection) => void @@ -238,11 +250,23 @@ class EngineConnection extends EventTarget { pc?: RTCPeerConnection unreliableDataChannel?: RTCDataChannel mediaStream?: MediaStream - idleMode: boolean = false promise?: Promise sdpAnswer?: RTCSessionDescriptionInit triggeredStart = false + onWebSocketOpen = function (event: Event) {} + onWebSocketClose = function (event: Event) {} + onWebSocketError = function (event: Event) {} + onWebSocketMessage = function (event: MessageEvent) {} + onIceGatheringStateChange = function ( + this: RTCPeerConnection, + event: Event + ) {} + onIceConnectionStateChange = function ( + this: RTCPeerConnection, + event: Event + ) {} + onNegotiationNeeded = function (this: RTCPeerConnection, event: Event) {} onIceCandidate = function ( this: RTCPeerConnection, event: RTCPeerConnectionIceEvent @@ -252,6 +276,9 @@ class EngineConnection extends EventTarget { event: RTCPeerConnectionIceErrorEvent ) {} onConnectionStateChange = function (this: RTCPeerConnection, event: Event) {} + onSignalingStateChange = function (this: RTCDataChannel, event: Event) {} + + onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {} onDataChannelOpen = function (this: RTCDataChannel, event: Event) {} onDataChannelClose = function (this: RTCDataChannel, event: Event) {} onDataChannelError = function (this: RTCDataChannel, event: Event) {} @@ -260,11 +287,7 @@ class EngineConnection extends EventTarget { this: RTCPeerConnection, event: RTCDataChannelEvent ) {} - onTrack = function (this: RTCPeerConnection, event: RTCTrackEvent) {} - onWebSocketOpen = function (event: Event) {} - onWebSocketClose = function (event: Event) {} - onWebSocketError = function (event: Event) {} - onWebSocketMessage = function (event: MessageEvent) {} + onNetworkStatusReady = () => {} private _state: EngineConnectionState = { @@ -309,10 +332,10 @@ class EngineConnection extends EventTarget { private engineCommandManager: EngineCommandManager private pingPongSpan: { ping?: number; pong?: number } - private pingIntervalId: ReturnType | null = null + private pingIntervalId: ReturnType | undefined = undefined isUsingConnectionLite: boolean = false - timeoutToForceConnectId: ReturnType | null = null + timeoutToForceConnectId: ReturnType | undefined = undefined constructor({ engineCommandManager, @@ -416,18 +439,18 @@ class EngineConnection extends EventTarget { return this.state.type === EngineConnectionStateType.ConnectionEstablished } - tearDown(opts?: { idleMode: boolean }) { - this.idleMode = opts?.idleMode ?? false - if (this.pingIntervalId) { - clearInterval(this.pingIntervalId) - } - if (this.timeoutToForceConnectId) { - clearTimeout(this.timeoutToForceConnectId) - } + tearDown() { + clearInterval(this.pingIntervalId) + clearTimeout(this.timeoutToForceConnectId) + + // As each network connection (websocket, webrtc, peer connection) is + // closed, they will handle removing their own event listeners. + // If they didn't then it'd be possible we stop listened to close events + // which is what we want to do in the first place :) this.disconnectAll() - if (this.idleMode) { + if (this.engineCommandManager.idleMode) { this.state = { type: EngineConnectionStateType.Disconnecting, value: { @@ -435,6 +458,7 @@ class EngineConnection extends EventTarget { }, } } + // Pass the state along if (this.state.type === EngineConnectionStateType.Disconnecting) return if (this.state.type === EngineConnectionStateType.Disconnected) return @@ -568,30 +592,41 @@ class EngineConnection extends EventTarget { }, 3000) } this.pc?.addEventListener?.('icecandidate', this.onIceCandidate) + + // Watch out human! The names of the next couple events are really similar! + this.onIceGatheringStateChange = (event) => { + console.log('icegatheringstatechange', event) + + that.initiateConnectionExclusive() + } this.pc?.addEventListener?.( 'icegatheringstatechange', - function (_event) { - console.log('icegatheringstatechange', this.iceGatheringState) - - if (this.iceGatheringState !== 'complete') return - that.initiateConnectionExclusive() - } + this.onIceGatheringStateChange ) + this.onIceConnectionStateChange = (event: Event) => { + console.log('iceconnectionstatechange', event) + } this.pc?.addEventListener?.( 'iceconnectionstatechange', - function (_event) { - console.log('iceconnectionstatechange', this.iceConnectionState) - console.log('iceconnectionstatechange', this.iceGatheringState) - } + this.onIceConnectionStateChange + ) + + this.onNegotiationNeeded = (event: Event) => { + console.log('negotiationneeded', event) + } + this.pc?.addEventListener?.( + 'negotiationneeded', + this.onNegotiationNeeded + ) + + this.onSignalingStateChange = (event) => { + console.log('signalingstatechange', event) + } + this.pc?.addEventListener?.( + 'signalingstatechange', + this.onSignalingStateChange ) - this.pc?.addEventListener?.('negotiationneeded', function (_event) { - console.log('negotiationneeded', this.iceConnectionState) - console.log('negotiationneeded', this.iceGatheringState) - }) - this.pc?.addEventListener?.('signalingstatechange', function (event) { - console.log('signalingstatechange', this.signalingState) - }) this.onIceCandidateError = (_event: Event) => { const event = _event as RTCPeerConnectionIceErrorEvent @@ -618,38 +653,12 @@ class EngineConnection extends EventTarget { detail: { conn: this, mediaStream: this.mediaStream! }, }) ) - - setTimeout(() => { - // Everything is now connected. - this.state = { - type: EngineConnectionStateType.ConnectionEstablished, - } - - this.engineCommandManager.inSequence = 1 - - this.dispatchEvent( - new CustomEvent(EngineConnectionEvents.Opened, { - detail: this, - }) - ) - markOnce('code/endInitialEngineConnect') - }, 2000) break + case 'connecting': break - case 'disconnected': - case 'failed': - this.pc?.removeEventListener('icecandidate', this.onIceCandidate) - this.pc?.removeEventListener( - 'icecandidateerror', - this.onIceCandidateError - ) - this.pc?.removeEventListener( - 'connectionstatechange', - this.onConnectionStateChange - ) - this.pc?.removeEventListener('track', this.onTrack) + case 'failed': this.state = { type: EngineConnectionStateType.Disconnecting, value: { @@ -662,6 +671,43 @@ class EngineConnection extends EventTarget { } this.disconnectAll() break + + // The remote end broke up with us! :( + case 'disconnected': + this.dispatchEvent( + new CustomEvent(EngineConnectionEvents.RestartRequest, {}) + ) + break + case 'closed': + this.pc?.removeEventListener('icecandidate', this.onIceCandidate) + this.pc?.removeEventListener( + 'icegatheringstatechange', + this.onIceGatheringStateChange + ) + this.pc?.removeEventListener( + 'iceconnectionstatechange', + this.onIceConnectionStateChange + ) + this.pc?.removeEventListener( + 'negotiationneeded', + this.onNegotiationNeeded + ) + this.pc?.removeEventListener( + 'signalingstatechange', + this.onSignalingStateChange + ) + this.pc?.removeEventListener( + 'icecandidateerror', + this.onIceCandidateError + ) + this.pc?.removeEventListener( + 'connectionstatechange', + this.onConnectionStateChange + ) + this.pc?.removeEventListener('track', this.onTrack) + this.pc?.removeEventListener('datachannel', this.onDataChannel) + break + default: break } @@ -735,9 +781,7 @@ class EngineConnection extends EventTarget { // The app is eager to use the MediaStream; as soon as onNewTrack is // called, the following sequence happens: // EngineConnection.onNewTrack -> StoreState.setMediaStream -> - // Stream.tsx reacts to mediaStream change, setting a video element. - // We wait until connectionstatechange changes to "connected" - // to pass it to the rest of the application. + // EngineStream.tsx reacts to mediaStream change, setting a video element. this.mediaStream = mediaStream } @@ -761,6 +805,25 @@ class EngineConnection extends EventTarget { type: ConnectingType.DataChannelEstablished, }, } + + // Start firing off engine commands at this point. + // They could be fired at an earlier time, onWebSocketOpen, + // but DataChannel can offer some benefits like speed, + // and it's nice to say everything's connected before interacting + // with the server. + + this.state = { + type: EngineConnectionStateType.ConnectionEstablished, + } + + this.engineCommandManager.inSequence = 1 + + this.dispatchEvent( + new CustomEvent(EngineConnectionEvents.Opened, { + detail: this, + }) + ) + markOnce('code/endInitialEngineConnect') } this.unreliableDataChannel?.addEventListener( 'open', @@ -784,7 +847,6 @@ class EngineConnection extends EventTarget { 'message', this.onDataChannelMessage ) - this.pc?.removeEventListener('datachannel', this.onDataChannel) this.disconnectAll() } @@ -898,16 +960,19 @@ class EngineConnection extends EventTarget { } this.websocket.addEventListener('close', this.onWebSocketClose) - this.onWebSocketError = (event) => { - this.state = { - type: EngineConnectionStateType.Disconnecting, - value: { - type: DisconnectingType.Error, + this.onWebSocketError = (event: Event) => { + if (event.target instanceof WebSocket) { + this.state = { + type: EngineConnectionStateType.Disconnecting, value: { - error: ConnectionError.WebSocketError, - context: event, + type: DisconnectingType.Error, + value: { + error: ConnectionError.WebSocketError, + context: + WEBSOCKET_READYSTATE_TEXT[event.target.readyState] ?? event, + }, }, - }, + } } this.disconnectAll() @@ -1213,20 +1278,27 @@ class EngineConnection extends EventTarget { !this.websocket || this.websocket?.readyState === 3 - if (closedPc && closedUDC && closedWS) { - if (!this.idleMode) { - // Do not notify the rest of the program that we have cut off anything. - this.state = { type: EngineConnectionStateType.Disconnected } - } else { - this.state = { - type: EngineConnectionStateType.Disconnecting, - value: { - type: DisconnectingType.Pause, - }, - } + if (!(closedPc && closedUDC && closedWS)) { + return + } + + // Clean up all the event listeners. + + if (!this.engineCommandManager.idleMode) { + // Do not notify the rest of the program that we have cut off anything. + this.state = { type: EngineConnectionStateType.Disconnected } + this.dispatchEvent( + new CustomEvent(EngineConnectionEvents.RestartRequest, {}) + ) + } else { + this.state = { + type: EngineConnectionStateType.Disconnecting, + value: { + type: DisconnectingType.Pause, + }, } - this.triggeredStart = false } + this.triggeredStart = false } } @@ -1254,6 +1326,9 @@ export enum EngineCommandManagerEvents { // engineConnection is available but scene setup may not have run EngineAvailable = 'engine-available', + // request a restart of engineConnection + EngineRestartRequest = 'engine-restart-request', + // the whole scene is ready (settings loaded) SceneReady = 'scene-ready', } @@ -1366,10 +1441,29 @@ export class EngineCommandManager extends EventTarget { kclManager: null | KclManager = null codeManager?: CodeManager rustContext?: RustContext + sceneInfra?: SceneInfra // The current "manufacturing machine" aka 3D printer, CNC, etc. public machineManager: MachineManager | null = null + // Dispatch to the application the engine needs a restart. + private onEngineConnectionRestartRequest = () => { + this.dispatchEvent( + new CustomEvent(EngineCommandManagerEvents.EngineRestartRequest, {}) + ) + } + + private onOffline = () => { + console.log('Browser reported network is offline') + if (TEST) { + console.warn('DURING TESTS ENGINECONNECTION.ONOFFLINE WILL DO NOTHING.') + return + } + this.onEngineConnectionRestartRequest() + } + + idleMode: boolean = false + start({ setMediaStream, setIsStreamReady, @@ -1415,6 +1509,8 @@ export class EngineCommandManager extends EventTarget { return } + window.addEventListener('offline', this.onOffline) + let additionalSettings = this.settings.enableSSAO ? '&post_effect=ssao' : '' additionalSettings += '&show_grid=' + (this.settings.showScaleGrid ? 'true' : 'false') @@ -1436,29 +1532,51 @@ export class EngineCommandManager extends EventTarget { }) ) + this.engineConnection.addEventListener( + EngineConnectionEvents.RestartRequest, + this.onEngineConnectionRestartRequest as EventListener + ) + // eslint-disable-next-line @typescript-eslint/no-misused-promises this.onEngineConnectionOpened = async () => { - await this.rustContext?.clearSceneAndBustCache( - await jsAppSettings(), - this.codeManager?.currentFilePath || undefined - ) + console.log('onEngineConnectionOpened') + + try { + console.log('clearing scene and busting cache') + await this.rustContext?.clearSceneAndBustCache( + await jsAppSettings(), + this.codeManager?.currentFilePath || undefined + ) + } catch (e) { + // If this happens shit's actually gone south aka the websocket closed. + // Let's restart. + console.warn("shit's gone south") + console.warn(e) + this.engineConnection?.dispatchEvent( + new CustomEvent(EngineConnectionEvents.RestartRequest, {}) + ) + return + } // Set the stream's camera projection type // We don't send a command to the engine if in perspective mode because // for now it's the engine's default. if (settings.cameraProjection === 'orthographic') { - this.sendSceneCommand({ + console.log('Setting camera to orthographic') + await this.sendSceneCommand({ type: 'modeling_cmd_req', cmd_id: uuidv4(), cmd: { type: 'default_camera_set_orthographic', }, - }).catch(reportRejection) + }) } // Set the theme - this.setTheme(this.settings.theme).catch(reportRejection) + console.log('Setting theme', this.settings.theme) + await this.setTheme(this.settings.theme) // Set up a listener for the dark theme media query + console.log('Setup theme media query change') darkModeMatcher?.addEventListener( 'change', this.onDarkThemeMediaQueryChange @@ -1466,7 +1584,8 @@ export class EngineCommandManager extends EventTarget { // Set the edge lines visibility // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.sendSceneCommand({ + console.log('setting edge_lines_visible') + await this.sendSceneCommand({ type: 'modeling_cmd_req', cmd_id: uuidv4(), cmd: { @@ -1475,21 +1594,30 @@ export class EngineCommandManager extends EventTarget { }, }) + console.log('camControlsCameraChange') this._camControlsCameraChange() - // eslint-disable-next-line @typescript-eslint/no-floating-promises - this.sendSceneCommand({ - // CameraControls subscribes to default_camera_get_settings response events - // firing this at connection ensure the camera's are synced initially - type: 'modeling_cmd_req', - cmd_id: uuidv4(), - cmd: { - type: 'default_camera_get_settings', - }, - }) + // We should eventually only have 1 restoral call. + if (this.idleMode) { + await this.sceneInfra?.camControls.restoreRemoteCameraStateAndTriggerSync() + } else { + // NOTE: This code is old. It uses the old hack to restore camera. + console.log('call default_camera_get_settings') + // eslint-disable-next-line @typescript-eslint/no-floating-promises + await this.sendSceneCommand({ + // CameraControls subscribes to default_camera_get_settings response events + // firing this at connection ensure the camera's are synced initially + type: 'modeling_cmd_req', + cmd_id: uuidv4(), + cmd: { + type: 'default_camera_get_settings', + }, + }) + } setIsStreamReady(true) + console.log('Dispatching SceneReady') // Other parts of the application should use this to react on scene ready. this.dispatchEvent( new CustomEvent(EngineCommandManagerEvents.SceneReady, { @@ -1555,7 +1683,7 @@ export class EngineCommandManager extends EventTarget { }) as EventListener) this.onVideoTrackMute = () => { - console.error('video track mute: check webrtc internals -> inbound rtp') + console.warn('video track mute - potentially lost stream for a moment') } this.onEngineConnectionNewTrack = ({ @@ -1746,9 +1874,11 @@ export class EngineCommandManager extends EventTarget { this.engineConnection?.send(resizeCmd) } - tearDown(opts?: { - idleMode: boolean - }) { + tearDown(opts?: { idleMode: boolean }) { + this.idleMode = opts?.idleMode ?? false + + window.removeEventListener('offline', this.onOffline) + if (this.engineConnection) { for (const [cmdId, pending] of Object.entries(this.pendingCommands)) { pending.reject([ @@ -1786,14 +1916,14 @@ export class EngineCommandManager extends EventTarget { this.onDarkThemeMediaQueryChange ) - this.engineConnection?.tearDown(opts) + this.engineConnection?.tearDown() // Our window.engineCommandManager.tearDown assignment causes this case to happen which is // only really for tests. // @ts-ignore } else if (this.engineCommandManager?.engineConnection) { // @ts-ignore - this.engineCommandManager?.engineConnection?.tearDown(opts) + this.engineCommandManager?.engineConnection?.tearDown() // @ts-ignore this.engineCommandManager.engineConnection = null } @@ -2104,25 +2234,25 @@ export class EngineCommandManager extends EventTarget { // Set the stream background color // This takes RGBA values from 0-1 // So we convert from the conventional 0-255 found in Figma - this.sendSceneCommand({ + await this.sendSceneCommand({ cmd_id: uuidv4(), type: 'modeling_cmd_req', cmd: { type: 'set_background_color', color: getThemeColorForEngine(theme), }, - }).catch(reportRejection) + }) // Sets the default line colors const opposingTheme = getOppositeTheme(theme) - this.sendSceneCommand({ + await this.sendSceneCommand({ cmd_id: uuidv4(), type: 'modeling_cmd_req', cmd: { type: 'set_default_system_properties', color: getThemeColorForEngine(opposingTheme), }, - }).catch(reportRejection) + }) } } diff --git a/src/lib/singletons.ts b/src/lib/singletons.ts index 4044306a43..8440edf43d 100644 --- a/src/lib/singletons.ts +++ b/src/lib/singletons.ts @@ -74,6 +74,7 @@ editorManager.kclManager = kclManager // TODO: proper dependency injection. engineCommandManager.kclManager = kclManager engineCommandManager.codeManager = codeManager +engineCommandManager.sceneInfra = sceneInfra engineCommandManager.rustContext = rustContext kclManager.sceneInfraBaseUnitMultiplierSetter = (unit: BaseUnit) => { diff --git a/src/machines/engineStreamMachine.ts b/src/machines/engineStreamMachine.ts index 2f95c1407e..d099f6d57c 100644 --- a/src/machines/engineStreamMachine.ts +++ b/src/machines/engineStreamMachine.ts @@ -4,41 +4,53 @@ import { assign, fromPromise, setup } from 'xstate' import type { AppMachineContext } from '@src/lib/types' export enum EngineStreamState { - Off = 'off', - On = 'on', - WaitForMediaStream = 'wait-for-media-stream', + WaitingForDependencies = 'waiting-for-dependencies', + WaitingForMediaStream = 'waiting-for-media-stream', + WaitingToPlay = 'waiting-to-play', Playing = 'playing', Reconfiguring = 'reconfiguring', Paused = 'paused', + Stopped = 'stopped', // The is the state in-between Paused and Playing *specifically that order*. Resuming = 'resuming', } export enum EngineStreamTransition { - SetMediaStream = 'set-media-stream', + // This brings us back to the configuration loop + WaitForDependencies = 'wait-for-dependencies', + + // Our dependencies to set SetPool = 'set-pool', SetAuthToken = 'set-auth-token', + SetVideoRef = 'set-video-ref', + SetCanvasRef = 'set-canvas-ref', + SetMediaStream = 'set-media-stream', + + // Stream operations Play = 'play', Resume = 'resume', Pause = 'pause', + Stop = 'stop', + + // Used to reconfigure the stream during connection StartOrReconfigureEngine = 'start-or-reconfigure-engine', } export interface EngineStreamContext { pool: string | null authToken: string | undefined - mediaStream: MediaStream | null videoRef: MutableRefObject canvasRef: MutableRefObject + mediaStream: MediaStream | null zoomToFit: boolean } export const engineStreamContextCreate = (): EngineStreamContext => ({ pool: null, authToken: undefined, - mediaStream: null, videoRef: { current: null }, canvasRef: { current: null }, + mediaStream: null, zoomToFit: true, }) @@ -77,76 +89,6 @@ export const engineStreamMachine = setup({ input: {} as EngineStreamContext, }, actors: { - [EngineStreamTransition.Play]: fromPromise( - async ({ - input: { context, params, rootContext }, - }: { - input: { - context: EngineStreamContext - params: { zoomToFit: boolean } - rootContext: AppMachineContext - } - }) => { - const canvas = context.canvasRef.current - if (!canvas) return false - - const video = context.videoRef.current - if (!video) return false - - const mediaStream = context.mediaStream - if (!mediaStream) return false - - // If the video is already playing it means we're doing a reconfigure. - // We don't want to re-run the KCL or touch the video element at all. - if (!video.paused) { - return - } - - await rootContext.sceneInfra.camControls.restoreRemoteCameraStateAndTriggerSync() - - video.style.display = 'block' - canvas.style.display = 'none' - - video.srcObject = mediaStream - } - ), - [EngineStreamTransition.Pause]: fromPromise( - async ({ - input: { context, rootContext }, - }: { - input: { context: EngineStreamContext; rootContext: AppMachineContext } - }) => { - const video = context.videoRef.current - if (!video) return - - video.pause() - - const canvas = context.canvasRef.current - if (!canvas) return - - await holdOntoVideoFrameInCanvas(video, canvas) - video.style.display = 'none' - - await rootContext.sceneInfra.camControls.saveRemoteCameraState() - - // Make sure we're on the next frame for no flickering between canvas - // and the video elements. - window.requestAnimationFrame( - () => - void (async () => { - // Destroy the media stream. We will re-establish it. We could - // leave everything at pausing, preventing video decoders from running - // but we can do even better by significantly reducing network - // cards also. - context.mediaStream?.getVideoTracks()[0].stop() - context.mediaStream = null - video.srcObject = null - - rootContext.engineCommandManager.tearDown({ idleMode: true }) - })() - ) - } - ), [EngineStreamTransition.StartOrReconfigureEngine]: fromPromise( async ({ input: { context, event, rootContext }, @@ -157,21 +99,17 @@ export const engineStreamMachine = setup({ rootContext: AppMachineContext } }) => { - if (!context.authToken) return - - const video = context.videoRef.current - if (!video) return - - const canvas = context.canvasRef.current - if (!canvas) return + if (!context.authToken) return Promise.reject() + if (!context.videoRef.current) return Promise.reject() + if (!context.canvasRef.current) return Promise.reject() const { width, height } = getDimensions( window.innerWidth, window.innerHeight ) - video.width = width - video.height = height + context.videoRef.current.width = width + context.videoRef.current.height = height const settingsNext = { // override the pool param (?pool=) to request a specific engine instance @@ -206,51 +144,183 @@ export const engineStreamMachine = setup({ }) } ), + [EngineStreamTransition.Play]: fromPromise( + async ({ + input: { context, params }, + }: { + input: { context: EngineStreamContext; params: { zoomToFit: boolean } } + }) => { + if (!context.canvasRef.current) return + if (!context.videoRef.current) return + if (!context.mediaStream) return + + // If the video is already playing it means we're doing a reconfigure. + // We don't want to re-run the KCL or touch the video element at all. + if (!context.videoRef.current.paused) { + return + } + + // In the past we'd try to play immediately, but the proper thing is to way + // for the 'canplay' event to tell us data is ready. + const onCanPlay = () => { + if (!context.videoRef.current) { + return + } + + context.videoRef.current.play().catch(console.error) + + // Yes, event listeners can remove themselves because of the + // lazy nature of interpreted languages :D + context.videoRef.current.removeEventListener('canplay', onCanPlay) + } + + // We're receiving video frames, so show the video now. + const onPlay = () => { + // We have to give engine time to crunch all the scene setup we + // ask it to do. As far as I can tell it doesn't block until + // they are done, so we must wait. + setTimeout(() => { + if (!context.videoRef.current) { + return + } + if (!context.canvasRef.current) { + return + } + + context.videoRef.current.style.display = 'block' + context.canvasRef.current.style.display = 'none' + + context.videoRef.current.removeEventListener('play', onPlay) + // I've tried < 400ms and sometimes it's possible to see a flash + // and the camera snap. + }, 400) + } + + context.videoRef.current.addEventListener('canplay', onCanPlay) + context.videoRef.current.addEventListener('play', onPlay) + + // THIS ASSIGNMENT IS *EXTREMELY* EFFECTFUL! The amount of logic + // this triggers is quite far and wide. It drives the above events. + context.videoRef.current.srcObject = context.mediaStream + } + ), + + // Pause is also called when leaving the modeling scene. It's possible + // then videoRef and canvasRef are now null due to their DOM elements + // being destroyed. + [EngineStreamTransition.Pause]: fromPromise( + async ({ + input: { context, rootContext }, + }: { + input: { + context: EngineStreamContext + rootContext: AppMachineContext + } + }) => { + if (context.videoRef.current && context.canvasRef.current) { + await context.videoRef.current.pause() + + await holdOntoVideoFrameInCanvas( + context.videoRef.current, + context.canvasRef.current + ) + context.videoRef.current.style.display = 'none' + } + + await rootContext.sceneInfra.camControls.saveRemoteCameraState() + + // Make sure we're on the next frame for no flickering between canvas + // and the video elements. + window.requestAnimationFrame( + () => + void (async () => { + // Destroy the media stream. We will re-establish it. We could + // leave everything at pausing, preventing video decoders from running + // but we can do even better by significantly reducing network + // cards also. + context.mediaStream?.getVideoTracks()[0].stop() + context.mediaStream = null + + if (context.videoRef.current) { + context.videoRef.current.srcObject = null + } + + rootContext.engineCommandManager.tearDown({ idleMode: true }) + })() + ) + } + ), }, }).createMachine({ - initial: EngineStreamState.Off, + initial: EngineStreamState.WaitingForDependencies, context: (initial) => initial.input, states: { - [EngineStreamState.Off]: { - reenter: true, + [EngineStreamState.WaitingForDependencies]: { on: { [EngineStreamTransition.SetPool]: { - target: EngineStreamState.Off, - actions: [assign({ pool: ({ context, event }) => event.data.pool })], + target: EngineStreamState.WaitingForDependencies, + actions: [assign({ pool: ({ context, event }) => event.pool })], }, [EngineStreamTransition.SetAuthToken]: { - target: EngineStreamState.Off, + target: EngineStreamState.WaitingForDependencies, + actions: [ + assign({ authToken: ({ context, event }) => event.authToken }), + ], + }, + [EngineStreamTransition.SetVideoRef]: { + target: EngineStreamState.WaitingForDependencies, actions: [ - assign({ authToken: ({ context, event }) => event.data.authToken }), + assign({ videoRef: ({ context, event }) => event.videoRef }), + ], + }, + [EngineStreamTransition.SetCanvasRef]: { + target: EngineStreamState.WaitingForDependencies, + actions: [ + assign({ canvasRef: ({ context, event }) => event.canvasRef }), ], }, [EngineStreamTransition.StartOrReconfigureEngine]: { - target: EngineStreamState.On, + target: EngineStreamState.WaitingForMediaStream, }, }, }, - [EngineStreamState.On]: { - reenter: true, + [EngineStreamState.WaitingForMediaStream]: { invoke: { src: EngineStreamTransition.StartOrReconfigureEngine, input: (args) => ({ context: args.context, rootContext: args.self.system.get('root').getSnapshot().context, - params: { zoomToFit: args.context.zoomToFit }, event: args.event, }), + onError: [ + { + target: EngineStreamState.WaitingForDependencies, + reenter: true, + }, + ], }, on: { - // Transition requested by engineConnection + [EngineStreamTransition.StartOrReconfigureEngine]: { + target: EngineStreamState.WaitingForMediaStream, + reenter: true, + }, [EngineStreamTransition.SetMediaStream]: { - target: EngineStreamState.On, + target: EngineStreamState.WaitingToPlay, actions: [ assign({ mediaStream: ({ context, event }) => event.mediaStream }), ], }, + }, + }, + [EngineStreamState.WaitingToPlay]: { + on: { [EngineStreamTransition.Play]: { target: EngineStreamState.Playing, - actions: [assign({ zoomToFit: () => true })], + }, + // We actually failed inbetween needing to play and sending commands. + [EngineStreamTransition.StartOrReconfigureEngine]: { + target: EngineStreamState.WaitingForMediaStream, + reenter: true, }, }, }, @@ -270,6 +340,9 @@ export const engineStreamMachine = setup({ [EngineStreamTransition.Pause]: { target: EngineStreamState.Paused, }, + [EngineStreamTransition.Stop]: { + target: EngineStreamState.Stopped, + }, }, }, [EngineStreamState.Reconfiguring]: { @@ -280,9 +353,7 @@ export const engineStreamMachine = setup({ rootContext: args.self.system.get('root').getSnapshot().context, event: args.event, }), - onDone: { - target: EngineStreamState.Playing, - }, + onDone: [{ target: EngineStreamState.Playing }], }, }, [EngineStreamState.Paused]: { @@ -299,8 +370,27 @@ export const engineStreamMachine = setup({ }, }, }, + [EngineStreamState.Stopped]: { + invoke: { + src: EngineStreamTransition.Pause, + input: (args) => ({ + context: args.context, + rootContext: args.self.system.get('root').getSnapshot().context, + }), + onDone: [ + { + target: EngineStreamState.WaitingForDependencies, + actions: [ + assign({ + videoRef: { current: null }, + canvasRef: { current: null }, + }), + ], + }, + ], + }, + }, [EngineStreamState.Resuming]: { - reenter: true, invoke: { src: EngineStreamTransition.StartOrReconfigureEngine, input: (args) => ({ @@ -315,14 +405,11 @@ export const engineStreamMachine = setup({ target: EngineStreamState.Paused, }, [EngineStreamTransition.SetMediaStream]: { + target: EngineStreamState.Playing, actions: [ assign({ mediaStream: ({ context, event }) => event.mediaStream }), ], }, - [EngineStreamTransition.Play]: { - target: EngineStreamState.Playing, - actions: [assign({ zoomToFit: () => false })], - }, }, }, },