feat(ui): canvas image error handling

invokeai/frontend/web/public/locales/en.json

@@ -1873,6 +1873,10 @@
             "rgAutoNegativeNotSupported": "Auto-Negative not supported for selected base model",
             "rgNoRegion": "no region drawn"
         },
+        "errors": {
+            "unableToFindImage": "Unable to find image",
+            "unableToLoadImage": "Unable to Load Image"
+        },
         "controlMode": {
             "controlMode": "Control Mode",
             "balanced": "Balanced (recommended)",

invokeai/frontend/web/src/features/controlLayers/konva/CanvasObject/CanvasObjectImage.ts

@@ -1,13 +1,14 @@
 import { Mutex } from 'async-mutex';
 import { deepClone } from 'common/util/deepClone';
+import { withResultAsync } from 'common/util/result';
 import type { CanvasEntityBufferObjectRenderer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityBufferObjectRenderer';
 import type { CanvasEntityFilterer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityFilterer';
 import type { CanvasEntityObjectRenderer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityObjectRenderer';
 import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
 import { CanvasModuleBase } from 'features/controlLayers/konva/CanvasModuleBase';
 import type { CanvasSegmentAnythingModule } from 'features/controlLayers/konva/CanvasSegmentAnythingModule';
 import type { CanvasStagingAreaModule } from 'features/controlLayers/konva/CanvasStagingAreaModule';
-import { loadImage } from 'features/controlLayers/konva/util';
+import { getKonvaNodeDebugAttrs, loadImage } from 'features/controlLayers/konva/util';
 import type { CanvasImageState } from 'features/controlLayers/store/types';
 import { t } from 'i18next';
 import Konva from 'konva';
@@ -94,36 +95,41 @@ export class CanvasObjectImage extends CanvasModuleBase {
   }
 
   updateImageSource = async (imageName: string) => {
-    try {
-      this.log.trace({ imageName }, 'Updating image source');
+    this.log.trace({ imageName }, 'Updating image source');
 
-      this.isLoading = true;
-      this.konva.group.visible(true);
+    this.isLoading = true;
+    this.konva.group.visible(true);
 
-      if (!this.konva.image) {
-        this.konva.placeholder.group.visible(false);
-        this.konva.placeholder.text.text(t('common.loadingImage', 'Loading Image'));
-      }
+    if (!this.konva.image) {
+      this.konva.placeholder.group.visible(false);
+      this.konva.placeholder.text.text(t('common.loadingImage', 'Loading Image'));
+    }
 
-      const imageDTO = await getImageDTOSafe(imageName);
-      if (imageDTO === null) {
-        this.onFailedToLoadImage();
-        return;
-      }
+    const imageDTO = await getImageDTOSafe(imageName);
+    if (imageDTO === null) {
+      // ImageDTO not found (or network error)
+      this.onFailedToLoadImage(t('controlLayers.unableToFindImage', 'Unable to find image'));
+      return;
+    }
 
-      this.imageElement = await loadImage(imageDTO.image_url);
-      await this.updateImageElement();
-    } catch {
-      this.onFailedToLoadImage();
+    const imageElementResult = await withResultAsync(() => loadImage(imageDTO.image_url));
+    if (imageElementResult.isErr()) {
+      // Image loading failed (e.g. the URL to the "physical" image is invalid)
+      this.onFailedToLoadImage(t('controlLayers.unableToLoadImage', 'Unable to load image'));
+      return;
     }
+
+    this.imageElement = imageElementResult.value;
+
+    await this.updateImageElement();
   };
 
-  onFailedToLoadImage = () => {
-    this.log({ image: this.state.image }, 'Failed to load image');
+  onFailedToLoadImage = (message: string) => {
+    this.log({ image: this.state.image }, message);
     this.konva.image?.visible(false);
     this.isLoading = false;
     this.isError = true;
-    this.konva.placeholder.text.text(t('common.imageFailedToLoad', 'Image Failed to Load'));
+    this.konva.placeholder.text.text(message);
     this.konva.placeholder.group.visible(true);
   };
 
@@ -140,6 +146,7 @@ export class CanvasObjectImage extends CanvasModuleBase {
             image: this.imageElement,
             width,
             height,
+            visible: true,
           });
         } else {
           this.log.trace('Creating new Konva image');
@@ -202,6 +209,10 @@ export class CanvasObjectImage extends CanvasModuleBase {
       isLoading: this.isLoading,
       isError: this.isError,
       state: deepClone(this.state),
+      konva: {
+        group: getKonvaNodeDebugAttrs(this.konva.group),
+        image: this.konva.image ? getKonvaNodeDebugAttrs(this.konva.image) : null,
+      },
     };
   };
 }

invokeai/frontend/web/src/features/controlLayers/konva/CanvasStagingAreaModule.ts

@@ -75,35 +75,59 @@ export class CanvasStagingAreaModule extends CanvasModuleBase {
     this.log.trace('Rendering staging area');
     const stagingArea = this.manager.stateApi.runSelector(selectCanvasStagingAreaSlice);
 
-    const { x, y, width, height } = this.manager.stateApi.getBbox().rect;
+    const { x, y } = this.manager.stateApi.getBbox().rect;
     const shouldShowStagedImage = this.$shouldShowStagedImage.get();
 
     this.selectedImage = stagingArea.stagedImages[stagingArea.selectedStagedImageIndex] ?? null;
     this.konva.group.position({ x, y });
 
     if (this.selectedImage) {
       const { imageDTO } = this.selectedImage;
+      const image = imageDTOToImageWithDims(imageDTO);
+
+      /**
+       * When the final output image of a generation is received, we should clear that generation's last progress image.
+       *
+       * It's possible that we have already rendered the progress image from the next generation before the output image
+       * from the previous is fully loaded/rendered. This race condition results in a flicker:
+       * - LAST GENERATION: Render the final progress image
+       * - LAST GENERATION: Start loading the final output image...
+       * - NEXT GENERATION: Render the first progress image
+       * - LAST GENERATION: ...Finish loading the final output image & render it, clearing the progress image <-- Flicker!
+       * - NEXT GENERATION: Render the next progress image
+       *
+       * We can detect the race condition by stashing the session ID of the last progress image when we begin loading
+       * that session's output image. After we render it, if the progress image's session ID is the same as the one we
+       * stashed, we know that we have not yet gotten that next generation's first progress image. We can clear the
+       * progress image without causing a flicker.
+       */
+      const lastProgressEventSessionId = this.manager.progressImage.$lastProgressEvent.get()?.session_id;
+      const hideProgressIfSameSession = () => {
+        const currentProgressEventSessionId = this.manager.progressImage.$lastProgressEvent.get()?.session_id;
+        if (lastProgressEventSessionId === currentProgressEventSessionId) {
+          this.manager.progressImage.$lastProgressEvent.set(null);
+        }
+      };
 
       if (!this.image) {
-        const { image_name } = imageDTO;
         this.image = new CanvasObjectImage(
           {
             id: 'staging-area-image',
             type: 'image',
-            image: {
-              image_name: image_name,
-              width,
-              height,
-            },
+            image,
           },
           this
         );
+        await this.image.update(this.image.state, true);
         this.konva.group.add(this.image.konva.group);
-      }
-
-      if (!this.image.isLoading && !this.image.isError) {
-        await this.image.update({ ...this.image.state, image: imageDTOToImageWithDims(imageDTO) }, true);
-        this.manager.progressImage.$lastProgressEvent.set(null);
+        hideProgressIfSameSession();
+      } else if (this.image.isLoading) {
+        // noop - just wait for the image to load
+      } else if (this.image.state.image.image_name !== image.image_name) {
+        await this.image.update({ ...this.image.state, image }, true);
+        hideProgressIfSameSession();
+      } else if (this.image.isError) {
+        hideProgressIfSameSession();
       }
       this.image.konva.group.visible(shouldShowStagedImage);
     } else {
@@ -136,6 +160,7 @@ export class CanvasStagingAreaModule extends CanvasModuleBase {
       selectedImage: this.selectedImage,
       $shouldShowStagedImage: this.$shouldShowStagedImage.get(),
       $isStaging: this.$isStaging.get(),
+      image: this.image?.repr() ?? null,
     };
   };
 }