Skip to content

Commit 52bc9e8

Browse files
committed
Fix: update vessel radiance config
1 parent 9ad0b70 commit 52bc9e8

1 file changed

Lines changed: 16 additions & 17 deletions

File tree

aodn_cloud_optimised/config/dataset/vessel_satellite_radiance_delayed_qc.json

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -273,29 +273,36 @@
273273
"Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
274274
},
275275
"run_settings": {
276-
"_COMMENT": "2025-07-01: running this dataset with a batch size of more than 1 results in some silent data corruption with dask v2. It is unclear why. Maybe similar to xarray/issues/8882. Spent too much time on this. Maybe race conditions. But proof that xarray/zarr... is not reliable",
276+
"_COMMENT": "2025-10-21: running this dataset with a batch size of more than 1 will use more than 32GB of memory on the host machine running coiled. It doesn't make sence, but this is an on-going issue with ffspec, or maybe xarray/engine",
277277
"coiled_cluster_options": {
278278
"n_workers": [
279-
5,
279+
1,
280280
20
281281
],
282-
"scheduler_vm_types": "m7i.xlarge",
283-
"worker_vm_types": "m7i.2xlarge",
282+
"scheduler_vm_types": "m7i.large",
283+
"worker_vm_types": "m7i.8xlarge",
284284
"allow_ingress_from": "me",
285285
"compute_purchase_option": "spot_with_fallback",
286286
"worker_options": {
287-
"nthreads": 4,
288-
"memory_limit": "16GB"
287+
"nthreads": 32,
288+
"memory_limit": "128GB"
289289
}
290290
},
291-
"batch_size": 2,
291+
"batch_size": 1,
292292
"cluster": {
293293
"mode": "coiled",
294-
"restart_every_path": false
294+
"restart_every_path": true
295295
},
296296
"paths": [
297297
{
298-
"s3_uri": "s3://imos-data/IMOS/SRS/OC/radiometer/",
298+
"s3_uri": "s3://imos-data/IMOS/SRS/OC/radiometer/VMQ9273_Solander",
299+
"filter": [
300+
".*FV01.*\\.nc"
301+
],
302+
"year_range": []
303+
},
304+
{
305+
"s3_uri": "s3://imos-data/IMOS/SRS/OC/radiometer/VLHJ_Southern-Surveyor",
299306
"filter": [
300307
".*FV01.*\\.nc"
301308
],
@@ -365,14 +372,6 @@
365372
"dimensions": "TIME"
366373
}
367374
},
368-
"quality_control_version": {
369-
"source": "@global_attribute:file_version",
370-
"schema": {
371-
"type": "<U49",
372-
"units": "1",
373-
"dimensions": "TIME"
374-
}
375-
},
376375
"platform_code": {
377376
"source": "@global_attribute:platform_code",
378377
"schema": {

0 commit comments

Comments
 (0)