fix(iOS): Implement texture handling for iOS issues in late 16 and 17

common/reviews/api/core.api.md

@@ -264,6 +264,9 @@ function cancelLoadImage(imageId: string): void;
 // @public (undocumented)
 function cancelLoadImages(imageIds: Array<string>): void;
 
+// @public (undocumented)
+export function canRenderFloatTextures(): boolean;
+
 // @public (undocumented)
 function clamp(value: number, min: number, max: number): number;
 
@@ -377,6 +380,7 @@ function convertVolumeToStackViewport({ viewport, options, }: {
 // @public (undocumented)
 type Cornerstone3DConfig = {
     gpuTier?: TierResult;
+    isMobile: boolean;
     detectGPUConfig: GetGPUTier;
     rendering: {
         preferSizeOverAccuracy: boolean;
@@ -1109,6 +1113,9 @@ function getVolumeViewportScrollInfo(viewport: IVolumeViewport, volumeId: string
 // @public (undocumented)
 export function getWebWorkerManager(): any;
 
+// @public (undocumented)
+const hasFloatScalingParameters: (scalingParameters: ScalingParameters) => boolean;
+
 // @public (undocumented)
 function hasNaNValues(input: number[] | number): boolean;
 
@@ -3095,6 +3102,7 @@ export class StackViewport extends Viewport implements IStackViewport, IImagesLo
         };
         useRGBA: boolean;
         transferSyntaxUID: any;
+        useNativeDataType: boolean;
         priority: number;
         requestType: RequestType;
         additionalDetails: {
@@ -3504,7 +3512,8 @@ declare namespace utilities {
         getViewportImageIds,
         getRandomSampleFromArray,
         getVolumeId,
-        color
+        color,
+        hasFloatScalingParameters
     }
 }
 export { utilities }

packages/core/src/RenderingEngine/StackViewport.ts

@@ -2064,6 +2064,7 @@ class StackViewport extends Viewport implements IStackViewport, IImagesLoader {
       },
       useRGBA: false,
       transferSyntaxUID,
+      useNativeDataType: this.useNativeDataType,
       priority: 5,
       requestType: RequestType.Interaction,
       additionalDetails,

packages/core/src/RenderingEngine/helpers/setDefaultVolumeVOI.ts

@@ -166,6 +166,7 @@ async function getVOIFromMinMax(
     },
     priority: PRIORITY,
     requestType: REQUEST_TYPE,
+    useNativeDataType,
     preScale: {
       enabled: true,
       scalingParameters: scalingParametersToUse,

packages/core/src/index.ts

@@ -42,6 +42,7 @@ import {
   getConfiguration,
   setConfiguration,
   getWebWorkerManager,
+  canRenderFloatTextures,
 } from './init';
 
 // Classes
@@ -89,6 +90,7 @@ export {
   getConfiguration,
   setConfiguration,
   getWebWorkerManager,
+  canRenderFloatTextures,
   // enums
   Enums,
   CONSTANTS,

packages/core/src/init.ts

@@ -13,11 +13,12 @@ import CentralizedWebWorkerManager from './webWorkerManager/webWorkerManager';
 const defaultConfig: Cornerstone3DConfig = {
   gpuTier: undefined,
   detectGPUConfig: {},
+  isMobile: false, // is mobile device
   rendering: {
     useCPURendering: false,
     // GPU rendering options
     preferSizeOverAccuracy: false,
-    useNorm16Texture: false, // _hasNorm16TextureSupport(),
+    useNorm16Texture: false,
     strictZSpacingForVolumeViewport: true,
   },
   // cache
@@ -27,11 +28,12 @@ const defaultConfig: Cornerstone3DConfig = {
 let config: Cornerstone3DConfig = {
   gpuTier: undefined,
   detectGPUConfig: {},
+  isMobile: false, // is mobile device
   rendering: {
     useCPURendering: false,
     // GPU rendering options
     preferSizeOverAccuracy: false,
-    useNorm16Texture: false, // _hasNorm16TextureSupport(),
+    useNorm16Texture: false,
     strictZSpacingForVolumeViewport: true,
   },
   // cache
@@ -77,23 +79,32 @@ function hasSharedArrayBuffer() {
   }
 }
 
-// Todo: commenting this out until proper support for int16 textures
-// are added to browsers, current implementation is buggy
-// function _hasNorm16TextureSupport() {
-//   const gl = _getGLContext();
+function _hasNorm16TextureSupport() {
+  const gl = _getGLContext();
+
+  if (gl) {
+    const ext = (gl as WebGL2RenderingContext).getExtension(
+      'EXT_texture_norm16'
+    );
+
+    if (ext) {
+      return true;
+    }
+  }
 
-//   if (gl) {
-//     const ext = (gl as WebGL2RenderingContext).getExtension(
-//       'EXT_texture_norm16'
-//     );
+  return false;
+}
 
-//     if (ext) {
-//       return true;
-//     }
-//   }
+function isMobile() {
+  const ua = navigator.userAgent;
+  return /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
+    ua
+  );
+}
 
-//   return false;
-// }
+function isMobileIOS() {
+  return /iPhone|iPod/.test(navigator.platform);
+}
 
 /**
  * Initialize the cornerstone-core. If the browser has a webgl context and
@@ -112,6 +123,26 @@ async function init(configuration = config): Promise<boolean> {
   // merge configs
   config = deepMerge(defaultConfig, configuration);
 
+  config.isMobile = isMobile();
+
+  if (config.isMobile) {
+    // iOS devices don't have support for OES_texture_float_linear
+    // and thus we should use native data type if we are on iOS
+    if (isMobileIOS()) {
+      config.rendering.useNorm16Texture = _hasNorm16TextureSupport();
+
+      if (!config.rendering.useNorm16Texture) {
+        if (configuration.rendering?.preferSizeOverAccuracy) {
+          config.rendering.preferSizeOverAccuracy = true;
+        } else {
+          console.log(
+            'norm16 texture not supported, you can turn on the preferSizeOverAccuracy flag to use native data type, but be aware of the inaccuracy of the rendering in high bits'
+          );
+        }
+      }
+    }
+  }
+
   // gpuTier
   const hasWebGLContext = _hasActiveWebGLContext();
   if (!hasWebGLContext) {
@@ -165,6 +196,23 @@ function setPreferSizeOverAccuracy(status: boolean): void {
   _updateRenderingPipelinesForAllViewports();
 }
 
+/**
+ * Only IPhone IOS cannot render float textures right now due to the lack of support for OES_texture_float_linear.
+ * So we should not use float textures on IOS devices.
+ */
+function canRenderFloatTextures(): boolean {
+  const isMobile = config.isMobile;
+  if (!isMobile) {
+    return true;
+  }
+
+  if (isMobileIOS()) {
+    return false;
+  }
+
+  return true;
+}
+
 /**
  * Resets the cornerstone-core init state if it has been manually
  * initialized to force use the cpu rendering (e.g., for tests)
@@ -286,4 +334,5 @@ export {
   getConfiguration,
   setConfiguration,
   getWebWorkerManager,
+  canRenderFloatTextures,
 };

packages/core/src/types/Cornerstone3DConfig.ts

@@ -8,6 +8,11 @@ type Cornerstone3DConfig = {
    * https://github.com/pmndrs/detect-gpu/blob/master/src/index.ts#L82
    */
   gpuTier?: TierResult;
+
+  /**
+   * Whether the device is mobile or not.
+   */
+  isMobile: boolean;
   /**
    * When the `gpuTier` is not provided, the `detectGPUConfig` is passed as
    * an argument to the `getGPUTier` method.

packages/core/src/utilities/generateVolumePropsFromImageIds.ts

@@ -1,12 +1,17 @@
 import { vec3 } from 'gl-matrix';
-import { getConfiguration, getShouldUseSharedArrayBuffer } from '../init';
+import {
+  canRenderFloatTextures,
+  getConfiguration,
+  getShouldUseSharedArrayBuffer,
+} from '../init';
 import createFloat32SharedArray from './createFloat32SharedArray';
 import createInt16SharedArray from './createInt16SharedArray';
 import createUint16SharedArray from './createUInt16SharedArray';
 import createUint8SharedArray from './createUint8SharedArray';
 import getScalingParameters from './getScalingParameters';
 import makeVolumeMetadata from './makeVolumeMetadata';
 import sortImageIdsAndGetSpacing from './sortImageIdsAndGetSpacing';
+import { hasFloatScalingParameters } from './hasFloatScalingParameters';
 import { ImageVolumeProps, Mat3, Point3 } from '../types';
 import cache from '../cache';
 import { Events } from '../enums';
@@ -36,9 +41,8 @@ function generateVolumePropsFromImageIds(
 
   // The prescale is ALWAYS used with modality LUT, so we can assume that
   // if the rescale slope is not an integer, we need to use Float32
-  const hasFloatRescale =
-    scalingParameters.rescaleIntercept % 1 !== 0 ||
-    scalingParameters.rescaleSlope % 1 !== 0;
+  const floatAfterScale = hasFloatScalingParameters(scalingParameters);
+  const canRenderFloat = canRenderFloatTextures();
 
   const {
     BitsAllocated,
@@ -110,7 +114,7 @@ function generateVolumePropsFromImageIds(
       // Temporary fix for 16 bit images to use Float32
       // until the new dicom image loader handler the conversion
       // correctly
-      if (!use16BitDataType || hasFloatRescale) {
+      if (!use16BitDataType || (canRenderFloat && floatAfterScale)) {
         sizeInBytes = length * 4;
         scalarData = useSharedArrayBuffer
           ? createFloat32SharedArray(length)

packages/core/src/utilities/hasFloatScalingParameters.ts

@@ -0,0 +1,15 @@
+import { ScalingParameters } from '../types';
+
+/**
+ * Checks if the scaling parameters contain a float rescale value.
+ * @param scalingParameters - The scaling parameters to check.
+ * @returns True if the scaling parameters contain a float rescale value, false otherwise.
+ */
+export const hasFloatScalingParameters = (
+  scalingParameters: ScalingParameters
+): boolean => {
+  const hasFloatRescale = Object.values(scalingParameters).some(
+    (value) => typeof value === 'number' && !Number.isInteger(value)
+  );
+  return hasFloatRescale;
+};

packages/core/src/utilities/index.ts

@@ -71,6 +71,7 @@ import convertToGrayscale from './convertToGrayscale';
 import getViewportImageIds from './getViewportImageIds';
 import { getRandomSampleFromArray } from './getRandomSampleFromArray';
 import { getVolumeId } from './getVolumeId';
+import { hasFloatScalingParameters } from './hasFloatScalingParameters';
 
 // name spaces
 import * as planar from './planar';
@@ -162,4 +163,5 @@ export {
   getRandomSampleFromArray,
   getVolumeId,
   color,
+  hasFloatScalingParameters,
 };

packages/core/src/utilities/loadImageToCanvas.ts

@@ -113,17 +113,20 @@ export default function loadImageToCanvas(
       );
     }
 
-    const { useNorm16Texture } = getConfiguration().rendering;
+    const { useNorm16Texture, preferSizeOverAccuracy } =
+      getConfiguration().rendering;
+    const useNativeDataType = useNorm16Texture || preferSizeOverAccuracy;
 
     // IMPORTANT: Request type should be passed if not the 'interaction'
     // highest priority will be used for the request type in the imageRetrievalPool
     const options = {
       targetBuffer: {
-        type: useNorm16Texture ? undefined : 'Float32Array',
+        type: useNativeDataType ? undefined : 'Float32Array',
       },
       preScale: {
         enabled: true,
       },
+      useNativeDataType,
       useRGBA: !!useCPURendering,
       requestType,
     };

packages/dicomImageLoader/src/imageLoader/createImage.ts

@@ -121,6 +121,8 @@ function createImage(
   const imageFrame = getImageFrame(imageId);
   imageFrame.decodeLevel = options.decodeLevel;
 
+  options.allowFloatRendering = cornerstone.canRenderFloatTextures();
+
   // Get the scaling parameters from the metadata
   if (options.preScale.enabled) {
     const scalingParameters = getScalingParameters(
@@ -144,6 +146,16 @@ function createImage(
     options.targetBuffer.arrayBuffer instanceof SharedArrayBuffer;
 
   const { decodeConfig } = getOptions();
+
+  // check if the options to use the 16 bit data type is set
+  // on the image load options, and prefer that over the global
+  // options of the dicom loader
+  decodeConfig.use16BitDataType =
+    (options && options.targetBuffer?.type === 'Uint16Array') ||
+    options.targetBuffer?.type === 'Int16Array'
+      ? true
+      : options.useNativeDataType || decodeConfig.use16BitDataType;
+
   const decodePromise = decodeImageFrame(
     imageFrame,
     transferSyntax,