|
273 | 273 | "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]" |
274 | 274 | }, |
275 | 275 | "run_settings": { |
276 | | - "_COMMENT": "2025-07-01: running this dataset with a batch size of more than 1 results in some silent data corruption with dask v2. It is unclear why. Maybe similar to xarray/issues/8882. Spent too much time on this. Maybe race conditions. But proof that xarray/zarr... is not reliable", |
| 276 | + "_COMMENT": "2025-10-21: running this dataset with a batch size of more than 1 will use more than 32GB of memory on the host machine running coiled. It doesn't make sence, but this is an on-going issue with ffspec, or maybe xarray/engine", |
277 | 277 | "coiled_cluster_options": { |
278 | 278 | "n_workers": [ |
279 | | - 5, |
| 279 | + 1, |
280 | 280 | 20 |
281 | 281 | ], |
282 | | - "scheduler_vm_types": "m7i.xlarge", |
283 | | - "worker_vm_types": "m7i.2xlarge", |
| 282 | + "scheduler_vm_types": "m7i.large", |
| 283 | + "worker_vm_types": "m7i.8xlarge", |
284 | 284 | "allow_ingress_from": "me", |
285 | 285 | "compute_purchase_option": "spot_with_fallback", |
286 | 286 | "worker_options": { |
287 | | - "nthreads": 4, |
288 | | - "memory_limit": "16GB" |
| 287 | + "nthreads": 32, |
| 288 | + "memory_limit": "128GB" |
289 | 289 | } |
290 | 290 | }, |
291 | | - "batch_size": 2, |
| 291 | + "batch_size": 1, |
292 | 292 | "cluster": { |
293 | 293 | "mode": "coiled", |
294 | | - "restart_every_path": false |
| 294 | + "restart_every_path": true |
295 | 295 | }, |
296 | 296 | "paths": [ |
297 | 297 | { |
298 | | - "s3_uri": "s3://imos-data/IMOS/SRS/OC/radiometer/", |
| 298 | + "s3_uri": "s3://imos-data/IMOS/SRS/OC/radiometer/VMQ9273_Solander", |
| 299 | + "filter": [ |
| 300 | + ".*FV01.*\\.nc" |
| 301 | + ], |
| 302 | + "year_range": [] |
| 303 | + }, |
| 304 | + { |
| 305 | + "s3_uri": "s3://imos-data/IMOS/SRS/OC/radiometer/VLHJ_Southern-Surveyor", |
299 | 306 | "filter": [ |
300 | 307 | ".*FV01.*\\.nc" |
301 | 308 | ], |
|
365 | 372 | "dimensions": "TIME" |
366 | 373 | } |
367 | 374 | }, |
368 | | - "quality_control_version": { |
369 | | - "source": "@global_attribute:file_version", |
370 | | - "schema": { |
371 | | - "type": "<U49", |
372 | | - "units": "1", |
373 | | - "dimensions": "TIME" |
374 | | - } |
375 | | - }, |
376 | 375 | "platform_code": { |
377 | 376 | "source": "@global_attribute:platform_code", |
378 | 377 | "schema": { |
|
0 commit comments