@itk-viewer/remote-viewport - npm Package Compare versions

Comparing version 0.2.5 to 0.2.6

CHANGELOG.md

 # @itk-viewer/remote-viewport
 
+## 0.2.6
+
+### Patch Changes
+
+- 6191b9a: Change remote image scale based on fpsWatcher. Includes image memory size check.
+- Updated dependencies [6191b9a]
+  - @itk-viewer/viewer@0.2.3
+  - @itk-viewer/io@0.1.3
+
 ## 0.2.5
@@ -4,0 +13,0 @@

dist/remote-machine.d.ts

@@ -6,3 +6,3 @@ /// <reference types="gl-matrix/index.js" />
 import { viewportMachine } from '@itk-viewer/viewer/viewport-machine.js';
-import MultiscaleSpatialImage from '@itk-viewer/io/MultiscaleSpatialImage.js';
+import { MultiscaleSpatialImage } from '@itk-viewer/io/MultiscaleSpatialImage.js';
 type RendererProps = {
@@ -27,2 +27,3 @@ density: number;
     toRendererCoordinateSystem: ReadonlyMat4;
+    maxImageBytes: number;
 };
@@ -54,3 +55,6 @@ type ConnectEvent = {
 };
-export declare const remoteMachine: import("xstate").StateMachine<Context, ConnectEvent | UpdateRendererEvent | RenderEvent | SetImage | SlowFps | FastFps | CameraPoseUpdated, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, {
+export declare const remoteMachine: import("xstate").StateMachine<Context, ConnectEvent | UpdateRendererEvent | RenderEvent | SetImage | SlowFps | FastFps | CameraPoseUpdated | {
+    type: 'done.invoke.updateImageScale';
+    output: number;
+}, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, {
     viewport: import("xstate").Actor<import("xstate").StateMachine<{
@@ -61,2 +65,3 @@ image: MultiscaleSpatialImage | undefined;
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -90,2 +95,3 @@ type: "setPose";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -121,2 +127,3 @@ type: "setPose";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -152,2 +159,3 @@ type: "setPose";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -176,3 +184,6 @@ type: "setPose";
     }>;
-}, import("xstate").NonReducibleUnknown, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, ConnectEvent | UpdateRendererEvent | RenderEvent | SetImage | SlowFps | FastFps | CameraPoseUpdated, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject>>;
+}, import("xstate").NonReducibleUnknown, import("xstate").ResolveTypegenMeta<import("xstate").TypegenDisabled, ConnectEvent | UpdateRendererEvent | RenderEvent | SetImage | SlowFps | FastFps | CameraPoseUpdated | {
+    type: 'done.invoke.updateImageScale';
+    output: number;
+}, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject>>;
 export {};

dist/remote-machine.js

 import { mat4 } from 'gl-matrix';
-import { assign, createMachine, raise, sendTo } from 'xstate';
+import { assign, createMachine, fromPromise, raise, sendTo, } from 'xstate';
 import { fpsWatcher } from '@itk-viewer/viewer/fps-watcher-machine.js';
 import { viewportMachine } from '@itk-viewer/viewer/viewport-machine.js';
+import { getVoxelCount, getBytes, } from '@itk-viewer/io/MultiscaleSpatialImage.js';
+const MAX_IMAGE_BYTES_DEFAULT = 4000 * 1000 * 1000; // 4000 MB
 const getEntries = (obj) => Object.entries(obj);
+const getTargetScale = ({ event, context }) => {
+    const image = context.viewport.getSnapshot().context.image;
+    if (!image || context.rendererProps.imageScale === undefined)
+        throw new Error('image or imageScale not found');
+    const currentScale = context.rendererProps.imageScale;
+    const { type } = event;
+    const scaleChange = type === 'slowFps' ? 1 : -1;
+    const targetScale = currentScale + scaleChange;
+    return Math.max(0, Math.min(image.coarsestScale, targetScale));
+};
+const checkTargetScaleExists = ({ event, context }) => {
+    const image = context.viewport.getSnapshot().context.image;
+    if (!image || context.rendererProps.imageScale === undefined)
+        throw new Error('image or imageScale not found');
+    const targetScale = getTargetScale({ event, context });
+    const currentScale = context.rendererProps.imageScale;
+    return targetScale !== currentScale;
+};
+// Assumes image.scaleIndexToWorld has been called with target scale
+const checkMemory = async ({ event, context }) => {
+    const image = context.viewport.getSnapshot().context.image;
+    if (!image)
+        throw new Error('image found');
+    const targetScale = getTargetScale({ event, context });
+    const voxelCount = await getVoxelCount(image, targetScale);
+    const imageBytes = getBytes(image, voxelCount);
+    return imageBytes < context.maxImageBytes;
+};
 export const remoteMachine = createMachine({
@@ -17,2 +47,3 @@ types: {},
         toRendererCoordinateSystem: mat4.create(),
+        maxImageBytes: MAX_IMAGE_BYTES_DEFAULT,
         ...input, // captures injected viewport
@@ -23,3 +54,3 @@ }),
         // imageProcessor computes toRendererCoordinateSystem.
-        // Needs to be a service because MultiscaleSpatialImage.scaleIndexToWorld is async due to coords
+        // Is an actor because MultiscaleSpatialImage.scaleIndexToWorld is async due to coords
         imageProcessor: {
@@ -50,3 +81,3 @@ initial: 'idle',
                                             image: image.name,
-                                            imageScale: image.scaleInfos.length - 1,
+                                            imageScale: image.coarsestScale,
                                         },
@@ -62,3 +93,3 @@ };
         },
-        // root state captures initial rendererProps even when disconnected
+        // root state captures initial rendererProps events even when disconnected
         root: {
@@ -120,4 +151,5 @@ entry: [
                         src: 'connect',
-                        input: ({ context }) => ({
+                        input: ({ context, event }) => ({
                             context,
+                            event,
                         }),
@@ -135,10 +167,2 @@ onDone: {
                 online: {
-                    on: {
-                        slowFps: {
-                            actions: ['updateImageScale'],
-                        },
-                        fastFps: {
-                            actions: ['updateImageScale'],
-                        },
-                    },
                     type: 'parallel',
@@ -152,2 +176,49 @@ states: {
                         },
+                        imageScaleUpdater: {
+                            on: {
+                                slowFps: {
+                                    target: '.updatingScale',
+                                },
+                                fastFps: {
+                                    target: '.updatingScale',
+                                },
+                            },
+                            initial: 'idle',
+                            states: {
+                                idle: {},
+                                updatingScale: {
+                                    invoke: {
+                                        id: 'updateImageScale',
+                                        input: ({ context, event }) => ({
+                                            context,
+                                            event,
+                                        }),
+                                        src: fromPromise(async ({ input: { event, context } }) => {
+                                            const image = context.viewport.getSnapshot().context.image;
+                                            if (!image)
+                                                return; // may be rendering without image
+                                            if (!checkTargetScaleExists({ event, context }))
+                                                return;
+                                            if (!(await checkMemory({ event, context })))
+                                                return;
+                                            return getTargetScale({ event, context });
+                                        }),
+                                        onDone: {
+                                            guard: ({ event }) => event.output !== undefined,
+                                            target: 'raiseImageScale',
+                                        },
+                                    },
+                                },
+                                raiseImageScale: {
+                                    entry: raise(({ event }) => {
+                                        if (event.type !== 'done.invoke.updateImageScale')
+                                            throw new Error('Unexpected event type');
+                                        return {
+                                            type: 'updateRenderer',
+                                            props: { imageScale: event.output },
+                                        };
+                                    }),
+                                },
+                            },
+                        },
                         renderLoop: {
@@ -161,5 +232,4 @@ initial: 'render',
                                         input: ({ context }) => ({
-                                            server: context.server,
                                             events: [...context.stagedRendererEvents],
-                                            toRendererCoordinateSystem: context.toRendererCoordinateSystem,
+                                            context,
                                         }),
@@ -181,3 +251,3 @@ onDone: {
                                         onError: {
-                                            actions: (e) => console.error('Error while updating render', e),
+                                            actions: (e) => console.error(`Error while updating render.`, e.event.data),
                                             target: 'idle', // soldier on
@@ -211,23 +281,3 @@ },
     },
-}, {
-    actions: {
-        updateImageScale: ({ event, context, self }) => {
-            const image = context.viewport.getSnapshot().context.image;
-            if (!image || context.rendererProps.imageScale === undefined)
-                return;
-            const scaleCount = image.scaleInfos.length - 1;
-            const scale = context.rendererProps.imageScale;
-            const { type } = event;
-            const scaleChange = type === 'slowFps' ? 1 : -1;
-            const targetScale = scale + scaleChange;
-            const newScale = Math.max(0, Math.min(scaleCount - 1, targetScale));
-            if (newScale !== scale) {
-                self.send({
-                    type: 'updateRenderer',
-                    props: { imageScale: newScale },
-                });
-            }
-        },
-    },
 });
 //# sourceMappingURL=remote-machine.js.map

dist/remote-viewport.d.ts

@@ -13,3 +13,2 @@ /// <reference types="gl-matrix/index.js" />
     updateRenderer: (events: unknown) => unknown;
-    loadImage: (image: string | undefined) => void;
     render: () => Promise<{
@@ -20,6 +19,8 @@ frame: Uint8Array;
 };
+type MachineContext = Omit<Context, 'server'> & {
+    server: Renderer;
+};
 type RendererInput = {
-    server: Renderer;
+    context: MachineContext;
     events: RendererEntries;
-    toRendererCoordinateSystem: ReadonlyMat4;
 };
@@ -50,3 +51,3 @@ type ConnectInput = {
     type: "setImage";
-    image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+    image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
 } | {
@@ -59,8 +60,12 @@ type: "slowFps";
     pose: ReadonlyMat4;
+} | {
+    type: "done.invoke.updateImageScale";
+    output: number;
 }, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, {
     viewport: import("xstate").Actor<import("xstate").StateMachine<{
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default | undefined;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage | undefined;
         camera?: import("xstate").Actor<import("xstate").StateMachine<{
             pose: ReadonlyMat4;
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -88,3 +93,3 @@ type: "setPose";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -95,2 +100,3 @@ type: "setCamera";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -120,3 +126,3 @@ type: "setPose";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -127,2 +133,3 @@ type: "setCamera";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -152,3 +159,3 @@ type: "setPose";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -159,2 +166,3 @@ type: "setCamera";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -198,3 +206,3 @@ type: "setPose";
     type: "setImage";
-    image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+    image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
 } | {
@@ -207,2 +215,5 @@ type: "slowFps";
     pose: ReadonlyMat4;
+} | {
+    type: "done.invoke.updateImageScale";
+    output: number;
 }, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject>>, {
@@ -223,3 +234,3 @@ type: "connect";
     type: "setImage";
-    image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+    image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
 } | {
@@ -232,2 +243,5 @@ type: "slowFps";
     pose: ReadonlyMat4;
+} | {
+    type: "done.invoke.updateImageScale";
+    output: number;
 }>;
@@ -251,3 +265,3 @@ export type RemoteActor = ReturnType<typeof createRemote>;
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -260,8 +274,12 @@ type: "slowFps";
         pose: ReadonlyMat4;
+    } | {
+        type: "done.invoke.updateImageScale";
+        output: number;
     }, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject, {
         viewport: import("xstate").Actor<import("xstate").StateMachine<{
-            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default | undefined;
+            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage | undefined;
             camera?: import("xstate").Actor<import("xstate").StateMachine<{
                 pose: ReadonlyMat4;
                 lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+                verticalFieldOfView: number;
             }, {
@@ -289,3 +307,3 @@ type: "setPose";
             type: "setImage";
-            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
         } | {
@@ -296,2 +314,3 @@ type: "setCamera";
                 lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+                verticalFieldOfView: number;
             }, {
@@ -321,3 +340,3 @@ type: "setPose";
             type: "setImage";
-            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
         } | {
@@ -328,2 +347,3 @@ type: "setCamera";
                 lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+                verticalFieldOfView: number;
             }, {
@@ -353,3 +373,3 @@ type: "setPose";
             type: "setImage";
-            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+            image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
         } | {
@@ -360,2 +380,3 @@ type: "setCamera";
                 lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+                verticalFieldOfView: number;
             }, {
@@ -399,3 +420,3 @@ type: "setPose";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -408,2 +429,5 @@ type: "slowFps";
         pose: ReadonlyMat4;
+    } | {
+        type: "done.invoke.updateImageScale";
+        output: number;
     }, import("xstate").ProvidedActor, import("xstate").ParameterizedObject, import("xstate").ParameterizedObject>>, {
@@ -424,3 +448,3 @@ type: "connect";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -433,6 +457,9 @@ type: "slowFps";
         pose: ReadonlyMat4;
+    } | {
+        type: "done.invoke.updateImageScale";
+        output: number;
     }>;
     viewport: import("xstate").ActorRef<{
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -443,2 +470,3 @@ type: "setCamera";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -467,6 +495,7 @@ type: "setPose";
     }, import("xstate").State<{
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default | undefined;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage | undefined;
         camera?: import("xstate").Actor<import("xstate").StateMachine<{
             pose: ReadonlyMat4;
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -494,3 +523,3 @@ type: "setPose";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -501,2 +530,3 @@ type: "setCamera";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -526,3 +556,3 @@ type: "setPose";
         type: "setImage";
-        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").default;
+        image: import("@itk-viewer/io/MultiscaleSpatialImage.js").MultiscaleSpatialImage;
     } | {
@@ -533,2 +563,3 @@ type: "setCamera";
             lookAt: import("@itk-viewer/viewer/camera-machine.js").LookAtParams;
+            verticalFieldOfView: number;
         }, {
@@ -535,0 +566,0 @@ type: "setPose";

dist/remote-viewport.js

@@ -21,2 +21,15 @@ import { createActor, fromPromise } from 'xstate';
 };
+const mat4ToLookAt = (transform) => {
+    const eye = vec3.create();
+    mat4.getTranslation(eye, transform);
+    const target = vec3.fromValues(transform[8], transform[9], transform[10]);
+    vec3.subtract(target, eye, target);
+    const up = vec3.fromValues(transform[4], transform[5], transform[6]);
+    return { eye, up, target };
+};
+const makeCameraPoseCommand = (toRendererCoordinateSystem, cameraPose) => {
+    const transform = mat4.create();
+    mat4.multiply(transform, toRendererCoordinateSystem, cameraPose);
+    return ['cameraPose', mat4ToLookAt(transform)];
+};
 export const createHyphaMachineConfig = () => {
@@ -27,23 +40,38 @@ let decodeWorker = null;
             connect: fromPromise(async ({ input }) => createHyphaRenderer(input.context)),
-            renderer: fromPromise(async ({ input: { server, events, toRendererCoordinateSystem }, }) => {
-                const translatedEvents = events
+            renderer: fromPromise(async ({ input: { context, events } }) => {
+                const commands = events
                     .map(([key, value]) => {
                     if (key === 'cameraPose') {
-                        const transform = mat4.create();
-                        mat4.multiply(transform, toRendererCoordinateSystem, value);
-                        const eye = vec3.create();
-                        mat4.getTranslation(eye, transform);
-                        const target = vec3.fromValues(transform[8], transform[9], transform[10]);
-                        vec3.subtract(target, eye, target);
-                        const up = vec3.fromValues(transform[4], transform[5], transform[6]);
-                        return ['cameraPose', { eye, up, target }];
+                        return makeCameraPoseCommand(context.toRendererCoordinateSystem, value);
                     }
                     if (key === 'image') {
-                        server.loadImage(value);
-                        return;
+                        const { imageScale: multiresolution_level } = context.rendererProps;
+                        return [
+                            'loadImage',
+                            { image_path: value, multiresolution_level },
+                        ];
                     }
+                    if (key === 'imageScale') {
+                        const { image: image_path } = context.rendererProps;
+                        return [
+                            'loadImage',
+                            { image_path, multiresolution_level: value },
+                        ];
+                    }
                     return [key, value];
                 })
-                    .filter(Boolean);
-                server.updateRenderer(translatedEvents);
+                    .flatMap((event) => {
+                    const [type] = event;
+                    if (type === 'loadImage') {
+                        // Resend camera pose after load image.
+                        // Camera is reset by Agave after load image?
+                        return [
+                            event,
+                            makeCameraPoseCommand(context.toRendererCoordinateSystem, context.rendererProps.cameraPose),
+                        ];
+                    }
+                    return [event];
+                });
+                const { server } = context;
+                server.updateRenderer(commands);
                 const { frame: encodedImage, renderTime } = await server.render();
@@ -54,7 +82,5 @@ const { image: frame, webWorker } = await decode(decodeWorker, encodedImage);
             }),
+            // compute toRendererCoordinateSystem
             imageProcessor: fromPromise(async ({ input: { event: { image }, }, }) => {
-                // compute toRendererCoordinateSystem
-                const imageScale = image.coarsestScale;
-                await image.scaleIndexToWorld(imageScale); // initializes indexToWorld matrix for getWorldBounds
-                const bounds = image.getWorldBounds(imageScale);
+                const bounds = await image.getWorldBounds(image.coarsestScale);
                 // match Agave by normalizing to largest dim
@@ -61,0 +87,0 @@ const wx = bounds[1] - bounds[0];

package.json

 {
   "name": "@itk-viewer/remote-viewport",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "",
@@ -28,4 +28,4 @@ "type": "module",
     "xstate": "5.0.0-beta.24",
-    "@itk-viewer/io": "^0.1.2",
-    "@itk-viewer/viewer": "^0.2.2"
+    "@itk-viewer/io": "^0.1.3",
+    "@itk-viewer/viewer": "^0.2.3"
   },
@@ -32,0 +32,0 @@ "scripts": {

src/remote-machine.ts

 import { ReadonlyMat4, mat4 } from 'gl-matrix';
-import { ActorRefFrom, assign, createMachine, raise, sendTo } from 'xstate';
+import {
+  ActorRefFrom,
+  assign,
+  createMachine,
+  fromPromise,
+  raise,
+  sendTo,
+} from 'xstate';
 import type { Image } from '@itk-wasm/htj2k';
@@ -8,4 +15,10 @@
 import { viewportMachine } from '@itk-viewer/viewer/viewport-machine.js';
-import MultiscaleSpatialImage from '@itk-viewer/io/MultiscaleSpatialImage.js';
+import {
+  MultiscaleSpatialImage,
+  getVoxelCount,
+  getBytes,
+} from '@itk-viewer/io/MultiscaleSpatialImage.js';
 
+const MAX_IMAGE_BYTES_DEFAULT = 4000 * 1000 * 1000; // 4000 MB
+
 type RendererProps = {
@@ -38,2 +51,3 @@ density: number;
   toRendererCoordinateSystem: ReadonlyMat4;
+  maxImageBytes: number;
 };
@@ -73,72 +87,111 @@
 
-export const remoteMachine = createMachine(
-  {
-    types: {} as {
-      context: Context;
-      events:
-        | ConnectEvent
-        | UpdateRendererEvent
-        | RenderEvent
-        | SetImage
-        | SlowFps
-        | FastFps
-        | CameraPoseUpdated;
+type Event =
+  | ConnectEvent
+  | UpdateRendererEvent
+  | RenderEvent
+  | SetImage
+  | SlowFps
+  | FastFps
+  | CameraPoseUpdated
+  | { type: 'done.invoke.updateImageScale'; output: number };
+
+type ActionArgs = { event: Event; context: Context };
+
+const getTargetScale = ({ event, context }: ActionArgs) => {
+  const image = context.viewport.getSnapshot().context.image;
+  if (!image || context.rendererProps.imageScale === undefined)
+    throw new Error('image or imageScale not found');
+
+  const currentScale = context.rendererProps.imageScale;
+  const { type } = event;
+
+  const scaleChange = type === 'slowFps' ? 1 : -1;
+  const targetScale = currentScale + scaleChange;
+  return Math.max(0, Math.min(image.coarsestScale, targetScale));
+};
+
+const checkTargetScaleExists = ({ event, context }: ActionArgs) => {
+  const image = context.viewport.getSnapshot().context.image;
+  if (!image || context.rendererProps.imageScale === undefined)
+    throw new Error('image or imageScale not found');
+
+  const targetScale = getTargetScale({ event, context });
+  const currentScale = context.rendererProps.imageScale;
+  return targetScale !== currentScale;
+};
+
+// Assumes image.scaleIndexToWorld has been called with target scale
+const checkMemory = async ({ event, context }: ActionArgs) => {
+  const image = context.viewport.getSnapshot().context.image;
+  if (!image) throw new Error('image found');
+
+  const targetScale = getTargetScale({ event, context });
+  const voxelCount = await getVoxelCount(image, targetScale);
+  const imageBytes = getBytes(image, voxelCount);
+
+  return imageBytes < context.maxImageBytes;
+};
+
+export const remoteMachine = createMachine({
+  types: {} as {
+    context: Context;
+    events: Event;
+  },
+  id: 'remote',
+  context: ({ input }: { input: { viewport: Viewport } }) => ({
+    rendererProps: {
+      density: 30,
+      cameraPose: mat4.create(),
     },
-    id: 'remote',
-    context: ({ input }: { input: { viewport: Viewport } }) => ({
-      rendererProps: {
-        density: 30,
-        cameraPose: mat4.create(),
-      },
-      queuedRendererEvents: [],
-      stagedRendererEvents: [],
-      toRendererCoordinateSystem: mat4.create(),
-      ...input, // captures injected viewport
-    }),
-    type: 'parallel',
-    states: {
-      // imageProcessor computes toRendererCoordinateSystem.
-      // Needs to be a service because MultiscaleSpatialImage.scaleIndexToWorld is async due to coords
-      imageProcessor: {
-        initial: 'idle',
-        states: {
-          idle: {
-            on: {
-              setImage: 'processing',
-            },
+    queuedRendererEvents: [],
+    stagedRendererEvents: [],
+    toRendererCoordinateSystem: mat4.create(),
+    maxImageBytes: MAX_IMAGE_BYTES_DEFAULT,
+    ...input, // captures injected viewport
+  }),
+  type: 'parallel',
+  states: {
+    // imageProcessor computes toRendererCoordinateSystem.
+    // Is an actor because MultiscaleSpatialImage.scaleIndexToWorld is async due to coords
+    imageProcessor: {
+      initial: 'idle',
+      states: {
+        idle: {
+          on: {
+            setImage: 'processing',
           },
-          processing: {
-            invoke: {
-              id: 'imageProcessor',
-              src: 'imageProcessor',
-              input: ({ event }) => ({
-                event,
-              }),
-              onDone: {
-                actions: [
-                  assign({
-                    toRendererCoordinateSystem: ({
-                      event: {
-                        output: { toRendererCoordinateSystem },
+        },
+        processing: {
+          invoke: {
+            id: 'imageProcessor',
+            src: 'imageProcessor',
+            input: ({ event }) => ({
+              event,
+            }),
+            onDone: {
+              actions: [
+                assign({
+                  toRendererCoordinateSystem: ({
+                    event: {
+                      output: { toRendererCoordinateSystem },
+                    },
+                  }) => toRendererCoordinateSystem,
+                }),
+                raise(
+                  ({
+                    event: {
+                      output: { image },
+                    },
+                  }) => {
+                    return {
+                      type: 'updateRenderer' as const,
+                      props: {
+                        image: image.name,
+                        imageScale: image.coarsestScale,
                       },
-                    }) => toRendererCoordinateSystem,
-                  }),
-                  raise(
-                    ({
-                      event: {
-                        output: { image },
-                      },
-                    }) => {
-                      return {
-                        type: 'updateRenderer' as const,
-                        props: {
-                          image: image.name,
-                          imageScale: image.scaleInfos.length - 1,
-                        },
-                      };
-                    },
-                  ),
-                ],
-                target: 'idle',
-              },
+                    };
+                  },
+                ),
+              ],
+              target: 'idle',
             },
@@ -148,147 +201,188 @@ },
       },
-      // root state captures initial rendererProps even when disconnected
-      root: {
-        entry: [
-          assign({
-            viewport: ({ spawn }) => spawn(viewportMachine, { id: 'viewport' }),
-          }),
-        ],
-        on: {
-          updateRenderer: {
-            actions: [
-              assign({
-                rendererProps: ({ event: { props }, context }) => {
-                  return {
-                    ...context.rendererProps,
-                    ...props,
-                  };
-                },
-                queuedRendererEvents: ({ event: { props }, context }) => [
-                  ...context.queuedRendererEvents,
-                  ...(getEntries(props) as RendererEntries),
-                ],
-              }),
-              // Trigger a render (if in idle state)
-              raise({ type: 'render' }),
-            ],
-          },
-          cameraPoseUpdated: {
-            actions: [
-              raise(({ event }) => {
+    },
+    // root state captures initial rendererProps events even when disconnected
+    root: {
+      entry: [
+        assign({
+          viewport: ({ spawn }) => spawn(viewportMachine, { id: 'viewport' }),
+        }),
+      ],
+      on: {
+        updateRenderer: {
+          actions: [
+            assign({
+              rendererProps: ({ event: { props }, context }) => {
                 return {
-                  type: 'updateRenderer' as const,
-                  props: { cameraPose: event.pose },
+                  ...context.rendererProps,
+                  ...props,
                 };
-              }),
-            ],
-          },
+              },
+              queuedRendererEvents: ({ event: { props }, context }) => [
+                ...context.queuedRendererEvents,
+                ...(getEntries(props) as RendererEntries),
+              ],
+            }),
+            // Trigger a render (if in idle state)
+            raise({ type: 'render' }),
+          ],
         },
-        initial: 'disconnected',
-        states: {
-          disconnected: {
-            on: {
-              connect: {
-                actions: [
-                  assign({
-                    serverConfig: ({
-                      event: { config },
-                    }: {
-                      event: ConnectEvent;
-                    }) => {
-                      return config;
-                    },
-                  }),
-                ],
-                target: 'connecting',
-              },
+        cameraPoseUpdated: {
+          actions: [
+            raise(({ event }) => {
+              return {
+                type: 'updateRenderer' as const,
+                props: { cameraPose: event.pose },
+              };
+            }),
+          ],
+        },
+      },
+      initial: 'disconnected',
+      states: {
+        disconnected: {
+          on: {
+            connect: {
+              actions: [
+                assign({
+                  serverConfig: ({
+                    event: { config },
+                  }: {
+                    event: ConnectEvent;
+                  }) => {
+                    return config;
+                  },
+                }),
+              ],
+              target: 'connecting',
             },
           },
-          connecting: {
-            invoke: {
-              id: 'connect',
-              src: 'connect',
-              input: ({ context }: { context: Context }) => ({
-                context,
+        },
+        connecting: {
+          invoke: {
+            id: 'connect',
+            src: 'connect',
+            input: ({ context, event }) => ({
+              context,
+              event,
+            }),
+            onDone: {
+              actions: assign({
+                server: ({ event }) => event.output,
+                // initially, send all props to renderer
+                queuedRendererEvents: ({ context }) =>
+                  getEntries(context.rendererProps),
               }),
-              onDone: {
-                actions: assign({
-                  server: ({ event }) => event.output,
-                  // initially, send all props to renderer
-                  queuedRendererEvents: ({ context }) =>
-                    getEntries(context.rendererProps),
-                }),
-                target: 'online',
-              },
+              target: 'online',
             },
           },
-          online: {
-            on: {
-              slowFps: {
-                actions: ['updateImageScale'],
+        },
+        online: {
+          type: 'parallel',
+          states: {
+            fpsWatcher: {
+              invoke: {
+                id: 'fpsWatcher',
+                src: fpsWatcher,
               },
-              fastFps: {
-                actions: ['updateImageScale'],
-              },
             },
-            type: 'parallel',
-            states: {
-              fpsWatcher: {
-                invoke: {
-                  id: 'fpsWatcher',
-                  src: fpsWatcher,
+            imageScaleUpdater: {
+              on: {
+                slowFps: {
+                  target: '.updatingScale',
                 },
+                fastFps: {
+                  target: '.updatingScale',
+                },
               },
-              renderLoop: {
-                initial: 'render',
-                states: {
-                  render: {
-                    invoke: {
-                      id: 'render',
-                      src: 'renderer',
-                      input: ({ context }: { context: Context }) => ({
-                        server: context.server,
-                        events: [...context.stagedRendererEvents],
-                        toRendererCoordinateSystem:
-                          context.toRendererCoordinateSystem,
-                      }),
-                      onDone: {
-                        actions: [
-                          assign({
-                            frame: ({ event }) => {
-                              return event.output.frame;
-                            },
-                          }),
-                          sendTo('fpsWatcher', ({ event }) => ({
-                            type: 'newSample',
-                            renderTime: event.output.renderTime,
-                          })),
-                        ],
-                        target: 'idle',
-                      },
-                      onError: {
-                        actions: (e) =>
-                          console.error('Error while updating render', e),
-                        target: 'idle', // soldier on
-                      },
+              initial: 'idle',
+              states: {
+                idle: {},
+                updatingScale: {
+                  invoke: {
+                    id: 'updateImageScale',
+                    input: ({ context, event }) => ({
+                      context,
+                      event,
+                    }),
+                    src: fromPromise(async ({ input: { event, context } }) => {
+                      const image =
+                        context.viewport.getSnapshot().context.image;
+                      if (!image) return; // may be rendering without image
+
+                      if (!checkTargetScaleExists({ event, context })) return;
+                      if (!(await checkMemory({ event, context }))) return;
+
+                      return getTargetScale({ event, context });
+                    }),
+                    onDone: {
+                      guard: ({ event }) => event.output !== undefined,
+                      target: 'raiseImageScale',
                     },
                   },
-                  idle: {
-                    always: {
-                      // Renderer props changed while rendering? Then render.
-                      guard: ({ context }) =>
-                        context.queuedRendererEvents.length > 0,
-                      target: 'render',
+                },
+                raiseImageScale: {
+                  entry: raise(({ event }) => {
+                    if (event.type !== 'done.invoke.updateImageScale')
+                      throw new Error('Unexpected event type');
+                    return {
+                      type: 'updateRenderer' as const,
+                      props: { imageScale: event.output },
+                    };
+                  }),
+                },
+              },
+            },
+            renderLoop: {
+              initial: 'render',
+              states: {
+                render: {
+                  invoke: {
+                    id: 'render',
+                    src: 'renderer',
+                    input: ({ context }: { context: Context }) => ({
+                      events: [...context.stagedRendererEvents],
+                      context,
+                    }),
+                    onDone: {
+                      actions: [
+                        assign({
+                          frame: ({ event }) => {
+                            return event.output.frame;
+                          },
+                        }),
+                        sendTo('fpsWatcher', ({ event }) => ({
+                          type: 'newSample',
+                          renderTime: event.output.renderTime,
+                        })),
+                      ],
+                      target: 'idle',
                     },
-                    on: {
-                      render: { target: 'render' },
+                    onError: {
+                      actions: (e) =>
+                        console.error(
+                          `Error while updating render.`,
+                          e.event.data,
+                        ),
+                      target: 'idle', // soldier on
                     },
-                    exit: assign({
-                      // consumes queue in prep for renderer
-                      stagedRendererEvents: ({ context }) => [
-                        ...context.queuedRendererEvents,
-                      ],
-                      queuedRendererEvents: [],
-                    }),
                   },
                 },
+                idle: {
+                  always: {
+                    // Renderer props changed while rendering? Then render.
+                    guard: ({ context }) =>
+                      context.queuedRendererEvents.length > 0,
+                    target: 'render',
+                  },
+                  on: {
+                    render: { target: 'render' },
+                  },
+                  exit: assign({
+                    // consumes queue in prep for renderer
+                    stagedRendererEvents: ({ context }) => [
+                      ...context.queuedRendererEvents,
+                    ],
+                    queuedRendererEvents: [],
+                  }),
+                },
               },
@@ -301,25 +395,2 @@ },
   },
-  {
-    actions: {
-      updateImageScale: ({ event, context, self }) => {
-        const image = context.viewport.getSnapshot().context.image;
-        if (!image || context.rendererProps.imageScale === undefined) return;
-
-        const scaleCount = image.scaleInfos.length - 1;
-        const scale = context.rendererProps.imageScale;
-        const { type } = event;
-
-        const scaleChange = type === 'slowFps' ? 1 : -1;
-        const targetScale = scale + scaleChange;
-        const newScale = Math.max(0, Math.min(scaleCount - 1, targetScale));
-
-        if (newScale !== scale) {
-          self.send({
-            type: 'updateRenderer',
-            props: { imageScale: newScale },
-          });
-        }
-      },
-    },
-  },
-);
+});

src/remote-viewport.ts

@@ -16,10 +16,10 @@ import { createActor, fromPromise } from 'xstate';
   updateRenderer: (events: unknown) => unknown;
-  loadImage: (image: string | undefined) => void;
   render: () => Promise<{ frame: Uint8Array; renderTime: number }>;
 };
 
+type MachineContext = Omit<Context, 'server'> & { server: Renderer };
+
 type RendererInput = {
-  server: Renderer;
+  context: MachineContext;
   events: RendererEntries;
-  toRendererCoordinateSystem: ReadonlyMat4;
 };
@@ -57,2 +57,23 @@
 
+const mat4ToLookAt = (transform: ReadonlyMat4) => {
+  const eye = vec3.create();
+  mat4.getTranslation(eye, transform);
+
+  const target = vec3.fromValues(transform[8], transform[9], transform[10]);
+  vec3.subtract(target, eye, target);
+
+  const up = vec3.fromValues(transform[4], transform[5], transform[6]);
+
+  return { eye, up, target };
+};
+
+const makeCameraPoseCommand = (
+  toRendererCoordinateSystem: ReadonlyMat4,
+  cameraPose: ReadonlyMat4,
+) => {
+  const transform = mat4.create();
+  mat4.multiply(transform, toRendererCoordinateSystem, cameraPose);
+  return ['cameraPose', mat4ToLookAt(transform)];
+};
+
 export const createHyphaMachineConfig: () => RemoteMachineOptions = () => {
@@ -67,42 +88,49 @@ let decodeWorker: Worker | null = null;
       renderer: fromPromise(
-        async ({
-          input: { server, events, toRendererCoordinateSystem },
-        }: {
-          input: RendererInput;
-        }) => {
-          const translatedEvents = events
+        async ({ input: { context, events } }: { input: RendererInput }) => {
+          const commands = events
             .map(([key, value]) => {
               if (key === 'cameraPose') {
-                const transform = mat4.create();
-                mat4.multiply(transform, toRendererCoordinateSystem, value);
-
-                const eye = vec3.create();
-                mat4.getTranslation(eye, transform);
-
-                const target = vec3.fromValues(
-                  transform[8],
-                  transform[9],
-                  transform[10],
+                return makeCameraPoseCommand(
+                  context.toRendererCoordinateSystem,
+                  value,
                 );
-                vec3.subtract(target, eye, target);
-
-                const up = vec3.fromValues(
-                  transform[4],
-                  transform[5],
-                  transform[6],
-                );
-
-                return ['cameraPose', { eye, up, target }];
               }
 
               if (key === 'image') {
-                server.loadImage(value);
-                return;
+                const { imageScale: multiresolution_level } =
+                  context.rendererProps;
+                return [
+                  'loadImage',
+                  { image_path: value, multiresolution_level },
+                ];
               }
 
+              if (key === 'imageScale') {
+                const { image: image_path } = context.rendererProps;
+                return [
+                  'loadImage',
+                  { image_path, multiresolution_level: value },
+                ];
+              }
+
               return [key, value];
             })
-            .filter(Boolean);
+            .flatMap((event) => {
+              const [type] = event;
+              if (type === 'loadImage') {
+                // Resend camera pose after load image.
+                // Camera is reset by Agave after load image?
+                return [
+                  event,
+                  makeCameraPoseCommand(
+                    context.toRendererCoordinateSystem,
+                    context.rendererProps.cameraPose,
+                  ),
+                ];
+              }
+              return [event];
+            });
 
-          server.updateRenderer(translatedEvents);
+          const { server } = context;
+          server.updateRenderer(commands);
           const { frame: encodedImage, renderTime } = await server.render();
@@ -118,2 +146,3 @@ const { image: frame, webWorker } = await decode(
       ),
+      // compute toRendererCoordinateSystem
       imageProcessor: fromPromise(
@@ -125,6 +154,3 @@ async ({
         }) => {
-          // compute toRendererCoordinateSystem
-          const imageScale = image.coarsestScale;
-          await image.scaleIndexToWorld(imageScale); // initializes indexToWorld matrix for getWorldBounds
-          const bounds = image.getWorldBounds(imageScale);
+          const bounds = await image.getWorldBounds(image.coarsestScale);
 
@@ -131,0 +157,0 @@ // match Agave by normalizing to largest dim

No alert changes

Improved metrics

Total package byte prevSize

163750

increased by7.31%

Lines of code

1728

increased by13.09%

Dependency changes

• Updated@itk-viewer/io@^0.1.3

• Updated@itk-viewer/viewer@^0.2.3