Skip to content

Commit 0c560bf

Browse files
TheJeranTheJeran
authored
Merge pull request #630 from EarthyScience/revert-629-jp/first-workers
Revert "Jp/first workers"
2 parents 38b58c2 + 9bb62a4 commit 0c560bf

7 files changed

Lines changed: 45 additions & 184 deletions

File tree

‎src/components/plots/AnalysisWG.tsx‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"use client";
2-
import { ArrayMinMax, GetCurrentArray, GetCurrentArrayWorkers } from '@/utils/HelperFuncs';
2+
import { ArrayMinMax, GetCurrentArray } from '@/utils/HelperFuncs';
33
import * as THREE from 'three';
44
import React, { useEffect, useRef } from 'react';
55
import { DataReduction, Convolve, Multivariate2D, Multivariate3D, CUMSUM3D, Convolve2D, CustomShader } from '../computation/webGPU';
@@ -98,7 +98,7 @@ const AnalysisWG = ({ setTexture, }: { setTexture: React.Dispatch<React.SetState
9898
}
9999

100100
// --- 2. Dispatch GPU computation based on the operation ---
101-
const inputArray = analysisMode ? analysisArray : await GetCurrentArrayWorkers(analysisStore)
101+
const inputArray = analysisMode ? analysisArray : await GetCurrentArray(analysisStore)
102102
const shapeInfo = { shape: dataShape, strides};
103103
const kernelParams = { kernelDepth, kernelSize };
104104
// [1538316, 1481, 1]
@@ -204,7 +204,7 @@ const AnalysisWG = ({ setTexture, }: { setTexture: React.Dispatch<React.SetState
204204

205205
const is2D = outputShape.length === 2
206206
async function Analyze(){
207-
const dataArray = analysisMode ? analysisArray : await GetCurrentArrayWorkers(analysisStore)
207+
const dataArray = analysisMode ? analysisArray : GetCurrentArray(analysisStore)
208208
const newArray = await CustomShader(dataArray, shapeInfo, kernelParams, axis, customShader?? "") as Float16Array
209209
const {minVal, maxVal} = valueScales
210210
const textureData = new Uint8Array(newArray.length)

‎src/components/plots/FlatMap.tsx‎

Lines changed: 39 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { useZarrStore } from '@/GlobalStates/ZarrStore';
99
import { vertShader } from '@/components/computation/shaders'
1010
import { useShallow } from 'zustand/shallow'
1111
import { ThreeEvent } from '@react-three/fiber';
12-
import { coarsenFlatArray, GetCurrentArray, GetCurrentArrayWorkers, GetTimeSeries, parseUVCoords, deg2rad } from '@/utils/HelperFuncs';
12+
import { coarsenFlatArray, GetCurrentArray, GetTimeSeries, parseUVCoords, deg2rad } from '@/utils/HelperFuncs';
1313
import { evaluate_cmap } from 'js-colormaps-es';
1414
import { useCoordBounds } from '@/hooks/useCoordBounds';
1515
import { GetFrag } from '../textures';
@@ -90,19 +90,8 @@ const FlatMap = ({textures, infoSetters} : {textures : THREE.DataTexture[] | THR
9090
const infoRef = useRef<boolean>(false)
9191
const lastUV = useRef<THREE.Vector2>(new THREE.Vector2(0,0))
9292
const rotateMap = analysisMode && axis == 2;
93-
// const sampleArray = useMemo(()=> analysisMode ? analysisArray : GetCurrentArray(),[analysisMode, analysisArray, textures])
93+
const sampleArray = useMemo(()=> analysisMode ? analysisArray : GetCurrentArray(),[analysisMode, analysisArray, textures])
9494
const analysisDims = useMemo(()=>dimArrays.length > 2 ? dimSlices.filter((_e,idx)=> idx != axis) : dimSlices,[dimSlices,axis])
95-
const [sampleArray, setSampleArray] = useState<any | undefined>(undefined) // Moved this to a state as async functions cannot be used in useMemo
96-
useEffect(()=>{
97-
if (analysisMode){
98-
setSampleArray(analysisArray)
99-
return
100-
}
101-
else {
102-
GetCurrentArrayWorkers().then(e=> setSampleArray(e))
103-
return
104-
}
105-
},[analysisMode, analysisArray, textures])
10695

10796
const {lonBounds, latBounds} = useCoordBounds()
10897

@@ -133,44 +122,44 @@ const FlatMap = ({textures, infoSetters} : {textures : THREE.DataTexture[] | THR
133122

134123

135124
// ----- TIMESERIES ----- //
136-
async function HandleTimeSeries(event: THREE.Intersection){
137-
const uv = event.uv;
138-
const normal = new THREE.Vector3(0,0,1)
139-
if(uv){
140-
const tsUV = flipY ? new THREE.Vector2(uv.x, 1-uv.y) : uv
141-
const tempTS = GetTimeSeries({data:analysisMode ? analysisArray : await GetCurrentArrayWorkers(), shape:dataShape, stride:strides},{uv:tsUV,normal})
142-
setPlotDim(0) //I think this 2 is only if there are 3-dims. Need to rework the logic
143-
144-
const coordUV = parseUVCoords({normal:normal,uv:uv})
145-
let dimCoords = coordUV.map((val,idx)=>val ? dimSlices[idx][Math.round(val*dimSlices[idx].length)] : null)
146-
const thisDimNames = dimNames.filter((_,idx)=> dimCoords[idx] !== null)
147-
const thisDimUnits = dimUnits.filter((_,idx)=> dimCoords[idx] !== null)
148-
dimCoords = dimCoords.filter(val => val !== null)
149-
const tsID = `${dimCoords[0]}_${dimCoords[1]}`
150-
const tsObj = {
151-
color:evaluate_cmap(getColorIdx()/10,"Paired"),
152-
data:tempTS
153-
}
154-
incrementColorIdx();
155-
updateTimeSeries({ [tsID] : tsObj})
156-
const dimObj = {
157-
first:{
158-
name:thisDimNames[0],
159-
loc:dimCoords[0] ?? 0,
160-
units:thisDimUnits[0]
161-
},
162-
second:{
163-
name:thisDimNames[1],
164-
loc:dimCoords[1] ?? 0,
165-
units:thisDimUnits[1]
166-
},
167-
plot:{
168-
units:dimUnits[0]
125+
function HandleTimeSeries(event: THREE.Intersection){
126+
const uv = event.uv;
127+
const normal = new THREE.Vector3(0,0,1)
128+
if(uv){
129+
const tsUV = flipY ? new THREE.Vector2(uv.x, 1-uv.y) : uv
130+
const tempTS = GetTimeSeries({data:analysisMode ? analysisArray : GetCurrentArray(), shape:dataShape, stride:strides},{uv:tsUV,normal})
131+
setPlotDim(0) //I think this 2 is only if there are 3-dims. Need to rework the logic
132+
133+
const coordUV = parseUVCoords({normal:normal,uv:uv})
134+
let dimCoords = coordUV.map((val,idx)=>val ? dimSlices[idx][Math.round(val*dimSlices[idx].length)] : null)
135+
const thisDimNames = dimNames.filter((_,idx)=> dimCoords[idx] !== null)
136+
const thisDimUnits = dimUnits.filter((_,idx)=> dimCoords[idx] !== null)
137+
dimCoords = dimCoords.filter(val => val !== null)
138+
const tsID = `${dimCoords[0]}_${dimCoords[1]}`
139+
const tsObj = {
140+
color:evaluate_cmap(getColorIdx()/10,"Paired"),
141+
data:tempTS
142+
}
143+
incrementColorIdx();
144+
updateTimeSeries({ [tsID] : tsObj})
145+
const dimObj = {
146+
first:{
147+
name:thisDimNames[0],
148+
loc:dimCoords[0] ?? 0,
149+
units:thisDimUnits[0]
150+
},
151+
second:{
152+
name:thisDimNames[1],
153+
loc:dimCoords[1] ?? 0,
154+
units:thisDimUnits[1]
155+
},
156+
plot:{
157+
units:dimUnits[0]
158+
}
159+
}
160+
updateDimCoords({[tsID] : dimObj})
161+
}
169162
}
170-
}
171-
updateDimCoords({[tsID] : dimObj})
172-
}
173-
}
174163
// ----- SHADER MATERIAL ----- //
175164
const shaderMaterial = useMemo(()=>new THREE.ShaderMaterial({
176165
glslVersion: THREE.GLSL3,

‎src/components/textures/TextureMakers.tsx‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,8 @@ function StoreData(array: Array, valueScales?: {maxVal: number, minVal: number})
1414
const [minVal,maxVal] = valueScales ? [valueScales.minVal, valueScales.maxVal] : ArrayMinMax(data )
1515
const textureData = new Uint8Array(data.length)
1616
const range = (maxVal - minVal)
17-
const multiplier = 1/range
1817
for (let i = 0; i < data.length; i++){
19-
const normed = (data[i] - minVal) * multiplier;
18+
const normed = (data[i] - minVal) / range;
2019
if (isNaN(normed)){
2120
textureData[i] = 255;
2221
} else {

‎src/components/workers/chunkWorker.ts‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎src/components/zarr/GetArray.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ export async function GetArray(varOveride?: string) {
1212
const { idx4D, initStore, variable, setProgress, setStrides, setStatus } = useGlobalStore.getState();
1313
const { compress, xSlice, ySlice, zSlice, coarsen, kernelSize, kernelDepth, fetchNC, setCurrentChunks, setArraySize } = useZarrStore.getState();
1414
const { cache } = useCacheStore.getState();
15-
const useNC = initStore.startsWith("local") && fetchNC // In case a user has NetCDF switched but then goes to a remote
16-
const fetcher = useNC ? NCFetcher() : zarrFetcher()
15+
const fetcher = fetchNC ? NCFetcher() : zarrFetcher()
1716
const targetVariable = varOveride ?? variable;
1817
const meta = await fetcher.getMetadata(targetVariable);
1918
const { shape, chunkShape, fillValue, dtype } = meta;

‎src/components/zarr/utils.ts‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,7 @@ export function ToFloat16(array : Float32Array, scalingFactor: number | null) :
5959
initialScale
6060
denominator = Math.pow(10,newScalingFactor);
6161
multiplier = 1/denominator;
62-
// Need SharedArrayBuffer here or else chunks passed to workers will become detached on the main thread.
63-
const buffer = new SharedArrayBuffer(array.length * 2) // 2 for float16
64-
const newArray = new Float16Array(buffer)
62+
const newArray = new Float16Array(array.length)
6563
for (let i = 0; i < array.length; i++) {
6664
newArray[i] = array[i] * multiplier;
6765
}

‎src/utils/HelperFuncs.ts‎

Lines changed: 0 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -301,101 +301,6 @@ export function GetCurrentArray(overrideStore?:string){
301301
}
302302
}
303303

304-
export async function GetCurrentArrayWorkers(overrideStore?:string){
305-
const { variable, is4D, idx4D, initStore, strides, dataShape, setStatus }= useGlobalStore.getState()
306-
const { arraySize, currentChunks } = useZarrStore.getState()
307-
const {cache} = useCacheStore.getState();
308-
const store = overrideStore ? overrideStore : initStore
309-
310-
if (cache.has(is4D ? `${store}_${idx4D}_${variable}` : `${store}_${variable}`)){ // Non-chunked data
311-
const chunk = cache.get(is4D ? `${store}_${idx4D}_${variable}` : `${store}_${variable}`)
312-
const compressed = chunk.compressed
313-
setStatus(compressed ? "Decompressing data..." : null)
314-
const thisData = compressed ? DecompressArray(chunk.data) : chunk.data
315-
setStatus(null)
316-
return thisData
317-
}
318-
319-
const sharedBuffer = new SharedArrayBuffer(arraySize * Float16Array.BYTES_PER_ELEMENT)
320-
const hasZ = dataShape.length > 2;
321-
const [xStartIdx, xEndIdx] = currentChunks.x
322-
const [yStartIdx, yEndIdx] = currentChunks.y
323-
const [zStartIdx, zEndIdx] = currentChunks.z
324-
325-
type Task = {
326-
chunkData: Float16Array | Uint8Array
327-
chunkShape: number[]
328-
chunkStride: number[]
329-
chunkCoord: [number, number, number]
330-
compressed: boolean
331-
}
332-
const tasks: Task[] = []
333-
334-
for (let z = zStartIdx; z < zEndIdx; z++) {
335-
for (let y = yStartIdx; y < yEndIdx; y++) {
336-
for (let x = xStartIdx; x < xEndIdx; x++) {
337-
const chunkID = `z${z}_y${y}_x${x}`
338-
const cacheName = is4D ? `${store}_${variable}_${idx4D}_chunk_${chunkID}` : `${store}_${variable}_chunk_${chunkID}`
339-
const chunk = cache.get(cacheName);
340-
const compressed = chunk.compressed;
341-
tasks.push({
342-
chunkData: chunk.data,
343-
chunkShape: chunk.shape,
344-
chunkStride: chunk.stride,
345-
chunkCoord: [z, y, x],
346-
compressed,
347-
})
348-
}
349-
}
350-
}
351-
const POOL_COUNT = navigator.hardwareConcurrency || 4;
352-
const workers = Array.from({length: Math.min(POOL_COUNT, tasks.length)}, () =>
353-
new Worker(new URL('../components/workers/chunkWorker.ts', import.meta.url), { type: 'module' })
354-
)
355-
356-
const terminateAll = () => workers.forEach(w => w.terminate())
357-
358-
let taskIndex = 0
359-
let completed = 0
360-
361-
await new Promise<void>((resolve, reject) => {
362-
const dispatch = (worker: Worker) => {
363-
if (taskIndex >= tasks.length) return // this worker is done
364-
const task = tasks[taskIndex++] // Next isntruction set
365-
worker.onmessage = () => {
366-
completed++
367-
if (completed === tasks.length) {
368-
terminateAll()
369-
resolve()
370-
} else { // More tasks left. Send worker new isntructions
371-
dispatch(worker)
372-
}
373-
}
374-
worker.onerror = (e) => {
375-
terminateAll()
376-
reject(e)
377-
}
378-
worker.postMessage({
379-
sharedBuffer,
380-
chunkData: task.chunkData,
381-
chunkShape: task.chunkShape,
382-
chunkStride: task.chunkStride,
383-
dataShape,
384-
strides,
385-
hasZ,
386-
compressed:task.compressed,
387-
chunkCoord: task.chunkCoord,
388-
startCoords: [zStartIdx, yStartIdx, xStartIdx],
389-
})
390-
}
391-
workers.forEach(dispatch) // Distribute first tasks
392-
}).catch((e) => {
393-
if (e.message === 'cancelled') return
394-
throw e
395-
})
396-
setStatus(null)
397-
return new Float16Array(sharedBuffer)
398-
}
399304

400305
export function TwoDecimals(val: number){
401306
return Math.round(val * 100)/100

0 commit comments

Comments
 (0)