feat(cornerstone): demonstrate CS3D multi-resolution volumes, with GPU settings presets, lossy downsampling flags, and CS3D_REF in CircleCI and Netlify

extensions/cornerstone/src/commandsModule.ts

@@ -1492,14 +1492,8 @@ function commandsModule({
       cornerstoneViewportService.storePresentation({ viewportId });
     },
     updateVolumeData: ({ volume }) => {
-      // update vtkOpenGLTexture and imageData of computed volume
-      const { imageData, vtkOpenGLTexture } = volume;
-      const numSlices = imageData.getDimensions()[2];
-      const slicesToUpdate = [...Array(numSlices).keys()];
-      slicesToUpdate.forEach(i => {
-        vtkOpenGLTexture.setUpdatedFrame(i);
-      });
-      imageData.modified();
+      // Marks the imageData and every texture set (full and reduced) of the computed volume dirty.
+      volume.modified();
     },
 
     attachProtocolViewportDataListener: ({ protocol, stageIndex }) => {

extensions/cornerstone/src/customizations/gpuSettingsPresetCustomization.ts

@@ -0,0 +1,23 @@
+/**
+ * The GPU settings preset that a deployment selects.
+ *
+ * The value is the `id` of a Cornerstone3D GPU settings preset: `auto`,
+ * `low-tablet`, `low`, `medium`, `high`, or `high-texture-4096`. A preset is a
+ * named set of values, the largest texture edge and the texture memory, and
+ * nothing probes the device to choose it. `auto` computes the values for the
+ * device, when the build of Cornerstone3D has `auto`.
+ *
+ * A preset applies only with final downsampling on. A value here turns final
+ * downsampling on, also in a locked-down config. `null`, the default, selects
+ * no preset: `appConfig.lossyDownsampling.final` then decides, with the
+ * default preset of Cornerstone3D.
+ *
+ * Set this value in the `bootstrap` phase or in the `global` phase of a
+ * `?customization=` module. `?customization=` needs `customizationUrlPrefixes`,
+ * and `config/default.js` has none: there, set the value through
+ * `customizationService` in the config. The shipped examples are
+ * `platform/app/public/customizations/gpu/<id>.jsonc`.
+ */
+export default {
+  'cornerstone.gpuSettingsPreset': null,
+};

extensions/cornerstone/src/getCustomizationModule.tsx

@@ -16,6 +16,7 @@ import miscCustomization from './customizations/miscCustomization';
 import captureViewportModalCustomization from './customizations/captureViewportModalCustomization';
 import viewportDownloadWarningCustomization from './customizations/viewportDownloadWarningCustomization';
 import getViewportScrollbarCustomization from './customizations/viewportScrollbarCustomization';
+import gpuSettingsPresetCustomization from './customizations/gpuSettingsPresetCustomization';
 
 function getCustomizationModule({ commandsManager, servicesManager, extensionManager }) {
   return [
@@ -44,6 +45,7 @@ function getCustomizationModule({ commandsManager, servicesManager, extensionMan
         ...captureViewportModalCustomization,
         ...viewportDownloadWarningCustomization,
         ...getViewportScrollbarCustomization(),
+        ...gpuSettingsPresetCustomization,
       },
     },
   ];

extensions/cornerstone/src/hps/mpr.ts

@@ -60,7 +60,6 @@ export const mpr: Types.HangingProtocol.Protocol = {
   editableBy: {},
   numberOfPriorsReferenced: 0,
   protocolMatchingRules: [],
-  imageLoadStrategy: 'nth',
   callbacks: {},
   displaySetSelectors: {
     activeDisplaySet: {

extensions/cornerstone/src/index.tsx

@@ -154,13 +154,14 @@ const cornerstoneExtension: Types.Extensions.Extension = {
 
     // Configure the interleaved/HTJ2K loader
     imageRetrieveMetadataProvider.clear();
-    // The default volume interleaved options are to interleave the
-    // image retrieve, but don't perform progressive loading per image
-    // This interleaves images and replicates them for low-resolution depth volume
-    // reconstruction, which progressively improves
+    // Volumes load every 64th image first and replicate it to the empty frames
+    // around it, then refine. A Cornerstone3D without the coarse stages keeps
+    // the stock interleaved stages.
+    const { ProgressiveRetrieveImages } = cornerstone;
     imageRetrieveMetadataProvider.add(
       'volume',
-      cornerstone.ProgressiveRetrieveImages.interleavedRetrieveStages
+      ProgressiveRetrieveImages.coarseInterleavedRetrieveStages ??
+        ProgressiveRetrieveImages.interleavedRetrieveStages
     );
 
     /**

extensions/cornerstone/src/init.tsx

@@ -39,6 +39,7 @@ import interleaveTopToBottom from './utils/interleaveTopToBottom';
 import initContextMenu from './initContextMenu';
 import initDoubleClick from './initDoubleClick';
 import initViewTiming from './utils/initViewTiming';
+import initGpuSettingsPreset from './utils/initGpuSettingsPreset';
 import { colormaps } from './utils/colormaps';
 import { SegmentationRepresentations } from '@cornerstonejs/tools/enums';
 import { useLutPresentationStore } from './stores/useLutPresentationStore';
@@ -74,9 +75,14 @@ export default async function init({
   const statsOverlay =
     new URLSearchParams(window.location.search).get('debug') === 'true';
 
+  // appConfig.lossyDownsampling = { final, interactive, interleavedLoading }.
+  // An absent kind keeps the Cornerstone3D default: final off, the others on.
   await cs3DInit({
     peerImport: appConfig.peerImport,
     debug: { statsOverlay },
+    ...(appConfig.lossyDownsampling && {
+      rendering: { lossyDownsampling: appConfig.lossyDownsampling },
+    }),
   });
 
   cornerstone.setUseCPURendering(Boolean(appConfig.useCPURendering));
@@ -159,6 +165,10 @@ export default async function init({
     toolbarService,
   } = servicesManager.services;
 
+  // A volume viewport reads the GPU settings when it adds its actor, so the
+  // preset must apply before a mode opens a viewport.
+  initGpuSettingsPreset({ customizationService, appConfig });
+
   toolbarService.registerEventForToolbarUpdate(colorbarService, [
     colorbarService.EVENTS.STATE_CHANGED,
   ]);

extensions/cornerstone/src/services/ViewportService/adapter/IViewportAdapter.contract.test.ts

@@ -165,6 +165,30 @@ describe('classification', () => {
     ).toBe(false);
   });
 
+  describe('isInAcquisitionPlane on a native volume', () => {
+    const axialCt = {
+      direction: [1, 0, 0, 0, 1, 0, 0, 0, 1],
+      dimensions: [512, 512, 100],
+      spacing: [0.7, 0.7, 2],
+    } as unknown as ReturnType<typeof cache.getVolume>;
+
+    beforeEach(() => jest.spyOn(cache, 'getVolume').mockReturnValue(axialCt));
+    afterEach(() => jest.restoreAllMocks());
+
+    it('tests the geometry, so axial on an axial CT counts', () => {
+      const axialOn = (viewPlaneNormal: number[]) =>
+        getViewportAdapter(
+          makeNextViewport({
+            getViewState: jest.fn().mockReturnValue({ orientation: OrientationAxis.AXIAL }),
+            getViewReference: jest.fn().mockReturnValue({ volumeId: 'ct', viewPlaneNormal }),
+          })
+        ).isInAcquisitionPlane();
+
+      expect(axialOn([0, 0, 1])).toBe(true);
+      expect(axialOn([1, 0, 0])).toBe(false);
+    });
+  });
+
   it('hasContent: legacy via actors, native via content mode', () => {
     expect(getViewportAdapter(makeLegacyStackViewport()).hasContent()).toBe(true);
     expect(

extensions/cornerstone/src/services/ViewportService/adapter/NextViewportAdapter.ts

@@ -1,4 +1,4 @@
-import { Enums, cache, Types as CoreTypes } from '@cornerstonejs/core';
+import { Enums, cache, Types as CoreTypes, utilities as csUtils } from '@cornerstonejs/core';
 import { isVolume3DViewportType } from '../../../utils/getLegacyViewportType';
 import type {
   IViewportAdapter,
@@ -70,6 +70,15 @@ export class NextViewportAdapter implements IViewportAdapter {
   }
 
   isInAcquisitionPlane(): boolean {
+    // A named orientation (axial on an axial CT) can be the acquisition plane, so test the geometry.
+    const reference = this.viewport.getViewReference?.();
+    const volume = reference?.volumeId ? cache.getVolume(reference.volumeId) : undefined;
+    const { isAcquisitionPlaneNormal } = csUtils;
+
+    if (volume && reference.viewPlaneNormal && typeof isAcquisitionPlaneNormal === 'function') {
+      return isAcquisitionPlaneNormal(volume, reference.viewPlaneNormal as CoreTypes.Point3);
+    }
+
     // orientation defaults to ACQUISITION when unset on the native view state.
     const orientation = this.viewport.getViewState?.()?.orientation;
     return orientation === Enums.OrientationAxis.ACQUISITION || orientation == null;

extensions/cornerstone/src/utils/initGpuSettingsPreset.ts

@@ -0,0 +1,76 @@
+import * as cornerstone from '@cornerstonejs/core';
+
+const CUSTOMIZATION_ID = 'cornerstone.gpuSettingsPreset';
+
+/**
+ * Selects the GPU settings preset from the `cornerstone.gpuSettingsPreset`
+ * customization, and turns on final downsampling when the customization
+ * states a preset.
+ *
+ * A preset is a named set of GPU settings in Cornerstone3D: the largest
+ * texture edge and the texture memory. With final downsampling on, a volume
+ * viewport reduces its texture to fit those values. The value `auto` asks
+ * Cornerstone3D to compute the settings for the device, when the build of
+ * Cornerstone3D has `auto`.
+ *
+ * - The customization states a preset: final downsampling is on, with that
+ *   preset, also in a locked-down config such as `config/default.js`. That
+ *   config has no `customizationUrlPrefixes`, so it must set the value through
+ *   `customizationService` in the config, not with `?customization=`.
+ * - The customization states no preset: `appConfig.lossyDownsampling.final`
+ *   decides, with the default preset of Cornerstone3D.
+ *
+ * A viewport reads the preset when it adds its actor, so the value must arrive
+ * before a mode opens a viewport. The `bootstrap` phase of a `?customization=`
+ * module applies before this extension registers, and the `global` phase
+ * applies immediately after it, so both are early enough.
+ *
+ * A build of Cornerstone3D that has no presets keeps its own behaviour, and
+ * this function reports the omission and does not throw.
+ */
+export default function initGpuSettingsPreset({ customizationService, appConfig }): void {
+  const configuredFinal = Boolean(appConfig?.lossyDownsampling?.final);
+  let appliedId = null;
+
+  const apply = () => {
+    const presetId = customizationService.getValue(CUSTOMIZATION_ID) ?? null;
+
+    if (presetId === appliedId) {
+      return;
+    }
+
+    const { setGpuSettingsPreset, resetGpuSettingsPreset, setLossyDownsampling } = cornerstone;
+
+    if (!presetId) {
+      // A preset that changes to null returns to the config and to the
+      // Cornerstone3D default preset.
+      resetGpuSettingsPreset?.();
+      setLossyDownsampling?.({ final: configuredFinal });
+      appliedId = null;
+      return;
+    }
+
+    if (typeof setGpuSettingsPreset !== 'function') {
+      console.warn(
+        `${CUSTOMIZATION_ID} states "${presetId}", and this build of ` +
+          '@cornerstonejs/core has no GPU settings presets. The viewer keeps ' +
+          'the behaviour of that build.'
+      );
+      appliedId = presetId;
+      return;
+    }
+
+    try {
+      setGpuSettingsPreset(presetId);
+      setLossyDownsampling?.({ final: true });
+      appliedId = presetId;
+    } catch (error) {
+      console.warn(`${CUSTOMIZATION_ID} states "${presetId}", which is not a preset.`, error);
+    }
+  };
+
+  apply();
+
+  customizationService.subscribe(customizationService.EVENTS.GLOBAL_CUSTOMIZATION_MODIFIED, apply);
+  customizationService.subscribe(customizationService.EVENTS.MODE_CUSTOMIZATION_MODIFIED, apply);
+}

platform/app/public/config/customization.js

@@ -4,6 +4,9 @@ window.config = {
   extensions: [],
   modes: ['@ohif/mode-test'],
   showStudyList: true,
+  // Final downsampling reduces a volume texture to fit the GPU settings preset.
+  // config/default.js keeps it off.
+  lossyDownsampling: { final: true },
   // below flag is for performance reasons, but it might not work for all servers
   maxNumberOfWebWorkers: 3,
   showWarningMessageForCrossOrigin: false,

platform/app/public/config/default.js

@@ -62,6 +62,14 @@ window.config = {
   showLoadingIndicator: true,
   experimentalStudyBrowserSort: false,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling is off here. To turn it on, uncomment the line below.
+  // Cornerstone3D defaults: final false, interactive true, interleavedLoading true.
+  // `final` reduces a volume texture to fit the GPU settings preset. Select the
+  // preset with the `cornerstone.gpuSettingsPreset` customization: auto, low-tablet,
+  // low, medium, high, high-texture-4096. These options are not fully safe on every
+  // device: a texture that is too large can lose the WebGL context, and then the
+  // user must reload. See configurationFiles.md and specificCustomizations.md.
+  // lossyDownsampling: { final: true, interactive: true, interleavedLoading: true },
   groupEnabledModesFirst: true,
   allowMultiSelectExport: false,
   maxNumRequests: {

platform/app/public/config/dev.js

@@ -36,6 +36,11 @@ window.config = {
   showLoadingIndicator: true,
   experimentalStudyBrowserSort: false,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling for the tests and the deploy previews. `final` reduces
+  // a volume texture to fit the GPU settings preset (Cornerstone3D default
+  // preset, or `cornerstone.gpuSettingsPreset`). config/default.js keeps final
+  // downsampling off.
+  lossyDownsampling: { final: true },
   groupEnabledModesFirst: true,
   allowMultiSelectExport: false,
   maxNumRequests: {

platform/app/public/config/docker-nginx-dcm4chee-keycloak.js

@@ -9,6 +9,14 @@ window.config = {
   showCPUFallbackMessage: true,
   showLoadingIndicator: true,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling is off here. To turn it on, uncomment the line below.
+  // Cornerstone3D defaults: final false, interactive true, interleavedLoading true.
+  // `final` reduces a volume texture to fit the GPU settings preset. Select the
+  // preset with the `cornerstone.gpuSettingsPreset` customization: auto, low-tablet,
+  // low, medium, high, high-texture-4096. These options are not fully safe on every
+  // device: a texture that is too large can lose the WebGL context, and then the
+  // user must reload. See configurationFiles.md and specificCustomizations.md.
+  // lossyDownsampling: { final: true, interactive: true, interleavedLoading: true },
   defaultDataSourceName: 'dicomweb',
   dataSources: [
     {

platform/app/public/config/docker-nginx-dcm4chee.js

@@ -9,6 +9,14 @@ window.config = {
   showCPUFallbackMessage: true,
   showLoadingIndicator: true,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling is off here. To turn it on, uncomment the line below.
+  // Cornerstone3D defaults: final false, interactive true, interleavedLoading true.
+  // `final` reduces a volume texture to fit the GPU settings preset. Select the
+  // preset with the `cornerstone.gpuSettingsPreset` customization: auto, low-tablet,
+  // low, medium, high, high-texture-4096. These options are not fully safe on every
+  // device: a texture that is too large can lose the WebGL context, and then the
+  // user must reload. See configurationFiles.md and specificCustomizations.md.
+  // lossyDownsampling: { final: true, interactive: true, interleavedLoading: true },
   defaultDataSourceName: 'dicomweb',
   dataSources: [
     {

platform/app/public/config/docker-nginx-orthanc-keycloak.js

@@ -10,6 +10,14 @@ window.config = {
   showCPUFallbackMessage: true,
   showLoadingIndicator: true,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling is off here. To turn it on, uncomment the line below.
+  // Cornerstone3D defaults: final false, interactive true, interleavedLoading true.
+  // `final` reduces a volume texture to fit the GPU settings preset. Select the
+  // preset with the `cornerstone.gpuSettingsPreset` customization: auto, low-tablet,
+  // low, medium, high, high-texture-4096. These options are not fully safe on every
+  // device: a texture that is too large can lose the WebGL context, and then the
+  // user must reload. See configurationFiles.md and specificCustomizations.md.
+  // lossyDownsampling: { final: true, interactive: true, interleavedLoading: true },
   groupEnabledModesFirst: true,
   maxNumRequests: {
     interaction: 100,

platform/app/public/config/docker-nginx-orthanc.js

@@ -10,6 +10,14 @@ window.config = {
   showLoadingIndicator: true,
   experimentalStudyBrowserSort: false,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling is off here. To turn it on, uncomment the line below.
+  // Cornerstone3D defaults: final false, interactive true, interleavedLoading true.
+  // `final` reduces a volume texture to fit the GPU settings preset. Select the
+  // preset with the `cornerstone.gpuSettingsPreset` customization: auto, low-tablet,
+  // low, medium, high, high-texture-4096. These options are not fully safe on every
+  // device: a texture that is too large can lose the WebGL context, and then the
+  // user must reload. See configurationFiles.md and specificCustomizations.md.
+  // lossyDownsampling: { final: true, interactive: true, interleavedLoading: true },
   studyPrefetcher: {
     enabled: true,
     displaySetsCount: 2,

platform/app/public/config/e2e.js

@@ -31,6 +31,11 @@ window.config = {
   showWarningMessageForCrossOrigin: false,
   showCPUFallbackMessage: false,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling for the tests and the deploy previews. `final` reduces
+  // a volume texture to fit the GPU settings preset (Cornerstone3D default
+  // preset, or `cornerstone.gpuSettingsPreset`). config/default.js keeps final
+  // downsampling off.
+  lossyDownsampling: { final: true },
   // filterQueryParam: false,
 
   // Add some customizations to the default e2e datasource

platform/app/public/config/netlify.js

@@ -31,6 +31,11 @@ window.config = {
   showLoadingIndicator: true,
   experimentalStudyBrowserSort: false,
   strictZSpacingForVolumeViewport: true,
+  // Lossy downsampling for the tests and the deploy previews. `final` reduces
+  // a volume texture to fit the GPU settings preset (Cornerstone3D default
+  // preset, or `cornerstone.gpuSettingsPreset`). config/default.js keeps final
+  // downsampling off.
+  lossyDownsampling: { final: true },
   groupEnabledModesFirst: true,
   allowMultiSelectExport: false,
   maxNumRequests: {

platform/app/public/customizations/gpu/high-texture-4096.jsonc

@@ -0,0 +1,27 @@
+// GPU settings preset: high-texture-4096, a control that fits no real device
+//
+// A texture edge of 4096 voxels and 32 GB of texture memory. No device is
+// tested to render an edge above 2048. Some devices report a larger
+// MAX_3D_TEXTURE_SIZE, but a reported size is not a reliable limit, so a
+// deployment must not select this preset in production. Cornerstone3D marks
+// the preset `internal`.
+//
+// The preset is the control of the demonstration. A series of up to 4096
+// images renders with no reduction here, and the same series reduces under
+// `gpu/medium`. The pair shows that the reduction comes from the limit and
+// not from a defect. A real device can fail to allocate a texture of this
+// edge, and that failure is the expected result on hardware.
+//
+// A preset applies only with final downsampling on, and a preset in this
+// customization turns final downsampling on, also in a locked-down config.
+// `?customization=` needs `customizationUrlPrefixes`, and config/default.js has
+// none: there, set the preset through `customizationService` in the config.
+// Nothing probes the device: a preset is a set of values, so a real device may
+// hold more or less than the preset states.
+//
+// Load it with `?customization=gpu/high-texture-4096`.
+{
+  "bootstrap": {
+    "cornerstone.gpuSettingsPreset": "high-texture-4096"
+  }
+}