Merge pull request #564 from EarthyScience/jp/update-wasm

netcdf4-wasm update. Netcdf INT support.

Author: TheJeran

package.json

@@ -47,7 +47,7 @@
     "lint": "eslint . --max-warnings=100"
   },
   "dependencies": {
-    "@earthyscience/netcdf4-wasm": "^0.1.3",
+    "@earthyscience/netcdf4-wasm": "0.2.3",
     "@ffmpeg/ffmpeg": "^0.12.15",
     "@ffmpeg/util": "^0.12.2",
     "@monaco-editor/react": "^4.7.0",

src/components/plots/DataCube.tsx

@@ -73,13 +73,12 @@ export const DataCube = ({ volTexture }: DataCubeProps ) => {
       blending: THREE.NormalBlending,
       depthWrite: false,
       side: useOrtho ? THREE.FrontSide : THREE.BackSide,
-    }),[useFragOpt, useOrtho]);
+    }),[useFragOpt, useOrtho, volTexture]);
 
     const geometry = useMemo(() => new THREE.BoxGeometry(shape.x, shape.y, shape.z), [shape]);
     useEffect(() => {
       if (shaderMaterial) {
         const uniforms = shaderMaterial.uniforms
-        uniforms.map.value = volTexture;
         uniforms.cmap.value = colormap;
         uniforms.cOffset.value = cOffset;
         uniforms.cScale.value = cScale;
@@ -100,7 +99,7 @@ export const DataCube = ({ volTexture }: DataCubeProps ) => {
         uniforms.maskValue.value = maskValue
         invalidate() // Needed because Won't trigger re-render if camera is stationary. 
       }
-    }, [volTexture, shape, colormap, cOffset, cScale, valueRange, xRange, yRange, zRange, aspectRatio, latBounds, lonBounds, quality, animProg, transparency, nanTransparency, nanColor, maskValue, fillValue, vTransferScale, vTransferRange]);
+    }, [shape, colormap, cOffset, cScale, valueRange, xRange, yRange, zRange, aspectRatio, latBounds, lonBounds, quality, animProg, transparency, nanTransparency, nanColor, maskValue, fillValue, vTransferScale, vTransferRange]);
     useFrame(({camera})=>{ // This calculates InverseModel matrix for the orthographic raymarcher
       if (!useOrtho || !meshRef.current || !shaderMaterial) return;
       meshRef.current.modelViewMatrix.multiplyMatrices(camera.matrixWorldInverse, meshRef.current.matrixWorld);

src/components/zarr/NCGetters.ts

@@ -1,7 +1,7 @@
 import { useZarrStore, useCacheStore, useGlobalStore, useErrorStore } from "@/GlobalStates"
 import { ToFloat16, CompressArray, DecompressArray, copyChunkToArray, RescaleArray } from "./ZarrLoaderLRU";
 import { Convolve } from "../computation/webGPU";
-import {coarsen3DArray, calculateStrides} from '@/utils/HelperFuncs'
+import {coarsen3DArray, calculateStrides, TypedArray} from '@/utils/HelperFuncs'
 
 export async function GetNCDims(variable: string){
     const {ncModule} = useZarrStore.getState()
@@ -67,6 +67,18 @@ export async function GetNCArray(variable: string){
     if ("_FillValue" in atts){
         fillValue = !Number.isNaN(atts["_FillValue"][0]) ? atts["_FillValue"][0] : fillValue
     }
+    let validRange: {min: number, max: number} | undefined;
+    if("valid_min" in atts && "valid_max" in atts){
+        validRange = {
+            min: atts["valid_min"][0],
+            max: atts["valid_max"][0]
+        }
+    }
+    let preScaling: number | undefined;
+    if ("scale_factor" in atts){
+        const thisScale = atts["scale_factor"][0]
+        if (thisScale != 1) preScaling = thisScale
+    }
 
     //---- Dimension Indices to Grab ----//
     const calcDim = (slice: [number, number | null], dimIdx: number) => {
@@ -169,12 +181,37 @@ export async function GetNCArray(variable: string){
                         return {starts, counts}
                     }
                     const { starts, counts } = getStartsAndCounts();
-                    const chunkArray = await ncModule.getSlicedVariableArray(variable, starts, counts)
+                    let chunkArray = await ncModule.getSlicedVariableArray(variable, starts, counts)
+                    const chunkType = chunkArray.constructor.name
+                    const isInt = chunkType.includes("int") 
                     let chunkStride = rank > 3 
                         ? [counts[3] * counts[2], counts[3], 1] 
                         : [counts[2] * counts[1], counts[2], 1]
                     let thisShape = counts
-                    let [chunkF16, newScalingFactor] = ToFloat16(chunkArray.map((v: number) => v === fillValue ? NaN : v), scalingFactor)
+                    const filterValues = (array: TypedArray) =>{
+                        for (let i = 0; i < array.length; i++){
+                            if (array[i] === fillValue && !isInt) array[i] = NaN
+                            if (validRange){
+                                if (isInt){
+                                    if (array[i] < validRange.min) array[i] = validRange.min 
+                                    if (array[i] > validRange.max) array[i] = validRange.max
+                                } else{
+                                    if (array[i] < validRange.min || array[i] > validRange.max) array[i] = NaN
+                                }
+
+                            }
+                        }
+                    }
+                    const preScale = (array: TypedArray, scaler: number) =>{
+                        const tempArray = new Float32Array(array.length);
+                        for (let i = 0; i < array.length; i++){
+                            tempArray[i] = array[i] * scaler;
+                        }
+                        return tempArray;
+                    }
+                    filterValues(chunkArray)
+                    if (preScaling) chunkArray = preScale(chunkArray, preScaling)
+                    let [chunkF16, newScalingFactor] = ToFloat16(chunkArray, scalingFactor)
                     if (coarsen){
                         chunkF16 = await Convolve(chunkF16, {shape:chunkShape, strides:chunkStride}, "Mean3D", {kernelSize, kernelDepth}) as Float16Array
                         thisShape = thisShape.map((dim: number, idx: number) => Math.floor(dim / (idx === 0 ? kernelDepth : kernelSize)))

src/components/zarr/ZarrLoaderLRU.ts

@@ -21,16 +21,17 @@ export function ToFloat16(array : Float32Array, scalingFactor: number | null) :
 			array[i] /= Math.pow(10,scalingFactor);
 		}
 	}
-	const [minVal, maxVal] = ArrayMinMax(array)
-	if (maxVal <= 65504 && minVal >= -65504 && Math.abs(maxVal) > 1e-3 ){ // If values fit in Float16, use that to save memory
-		newArray = new Float16Array(array)
-	}
-	else {
-		newScalingFactor = Math.ceil(Math.log10(maxVal/65504))
+	const maxVal = array.reduce((max, val) => Math.max(max, Math.abs(val)), -Infinity)
+	newScalingFactor = Math.ceil(Math.log10(maxVal/65504))
+	if (newScalingFactor > 1 || newScalingFactor < -8){
+		newScalingFactor = newScalingFactor + (scalingFactor ?? 0)
 		for (let i = 0; i < array.length; i++) {
 			array[i] /= Math.pow(10,newScalingFactor);
 		}
 		newArray = new Float16Array(array)
+	} else{
+		newArray = new Float16Array(array)
+		newScalingFactor = scalingFactor
 	}
 	return [newArray, newScalingFactor]
 }