Skip to content

Commit 614d3ba

Browse files
erikvansebilleerikvansebille
committed
Using to_windowed_arrays in docs
Instead of explicit ds.load()
1 parent d29d36b commit 614d3ba

11 files changed

Lines changed: 42 additions & 21 deletions

‎docs/user_guide/examples/explanation_kernelloop.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ import parcels.tutorial
5757
5858
# Load the CopernicusMarine data in the Agulhas region from the example_datasets
5959
ds_fields = parcels.tutorial.open_dataset("CopernicusMarine_data_for_Argo_tutorial/data")
60-
ds_fields.load() # load the dataset into memory
6160
6261
# Create an idealised wind field and add it to the dataset
6362
tdim, ydim, xdim = (len(ds_fields.time),len(ds_fields.latitude), len(ds_fields.longitude))
@@ -83,6 +82,9 @@ fieldset = parcels.FieldSet.from_sgrid_conventions(
8382
"Wind": ("UWind", "VWind")
8483
}
8584
)
85+
86+
# Convert the FieldSet to windowed arrays for better performance
87+
fieldset = fieldset.to_windowed_arrays()
8688
```
8789

8890
Now we define a wind Kernel that uses a forward Euler method to apply the wind forcing. Note that we update the `particles.dx` and `particles.dy` variables, rather than `particles.x` and `particles.y` directly.

‎docs/user_guide/examples/tutorial_delaystart.ipynb‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,14 @@
5656
"ds_fields = parcels.tutorial.open_dataset(\n",
5757
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
5858
")\n",
59-
"ds_fields.load() # load the dataset into memory\n",
6059
"\n",
6160
"# Convert to SGRID-compliant dataset and create FieldSet\n",
6261
"fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"]}\n",
6362
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
64-
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)"
63+
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
64+
"\n",
65+
"# Convert the FieldSet to windowed arrays for better performance\n",
66+
"fieldset = fieldset.to_windowed_arrays()"
6567
]
6668
},
6769
{
@@ -398,7 +400,7 @@
398400
],
399401
"metadata": {
400402
"kernelspec": {
401-
"display_name": "Parcels:docs (3.14.6)",
403+
"display_name": "Python 3",
402404
"language": "python",
403405
"name": "python3"
404406
},

‎docs/user_guide/examples/tutorial_diffusion.ipynb‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -472,8 +472,7 @@
472472
"# Load the CopernicusMarine data in the Agulhas region from the example_datasets\n",
473473
"ds_fields = parcels.tutorial.open_dataset(\n",
474474
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
475-
").isel(depth=slice(0, 1))\n",
476-
"ds_fields.load() # load the dataset into memory"
475+
").isel(depth=slice(0, 1))"
477476
]
478477
},
479478
{
@@ -528,7 +527,10 @@
528527
"ds_cell_areas[\"cell_areas\"] = ([\"lat\", \"lon\"], calc_cell_areas(ds_fields))\n",
529528
"fset2 = parcels.FieldSet.from_sgrid_conventions(ds_cell_areas)\n",
530529
"\n",
531-
"fieldset += fset2"
530+
"fieldset += fset2\n",
531+
"\n",
532+
"# Convert the FieldSet to windowed arrays for better performance\n",
533+
"fieldset = fieldset.to_windowed_arrays()"
532534
]
533535
},
534536
{

‎docs/user_guide/examples/tutorial_dt_integrators.ipynb‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
"ds_fields = parcels.tutorial.open_dataset(\n",
6868
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
6969
")\n",
70-
"ds_fields.load() # load the dataset into memory\n",
7170
"\n",
7271
"OUTPUT_FOLDER = Path(\"output\")\n",
7372
"OUTPUT_FOLDER.mkdir(exist_ok=True)\n",
@@ -77,6 +76,9 @@
7776
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
7877
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
7978
"\n",
79+
"# Convert the FieldSet to windowed arrays for better performance\n",
80+
"fieldset = fieldset.to_windowed_arrays()\n",
81+
"\n",
8082
"# Check field resolution in time and space\n",
8183
"print(\n",
8284
" f\"dt = {np.unique(np.diff(ds_fields.time.values).astype('timedelta64[h]'))} hours\"\n",

‎docs/user_guide/examples/tutorial_gsw_density.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,6 @@
4343
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
4444
")\n",
4545
"\n",
46-
"# TODO check how we can get good performance without loading full dataset in memory\n",
47-
"ds_fields.load() # load the dataset into memory\n",
48-
"\n",
4946
"# Convert to SGRID-compliant dataset and create FieldSet\n",
5047
"fields = {\n",
5148
" \"U\": ds_fields[\"uo\"],\n",
@@ -54,7 +51,10 @@
5451
" \"so\": ds_fields[\"so\"],\n",
5552
"}\n",
5653
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
57-
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)"
54+
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
55+
"\n",
56+
"# Convert the FieldSet to windowed arrays for better performance\n",
57+
"fieldset = fieldset.to_windowed_arrays()"
5858
]
5959
},
6060
{

‎docs/user_guide/examples/tutorial_manipulating_field_data.ipynb‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@
4949
"ds_fields = parcels.tutorial.open_dataset(\n",
5050
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
5151
")\n",
52-
"ds_fields.load() # load the dataset into memory\n",
5352
"\n",
5453
"# Create an idealised wind field and add it to the dataset\n",
5554
"ydim, xdim = len(ds_fields.latitude), len(ds_fields.longitude)\n",
@@ -122,6 +121,9 @@
122121
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
123122
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
124123
"\n",
124+
"# Convert the FieldSet to windowed arrays for better performance\n",
125+
"fieldset = fieldset.to_windowed_arrays()\n",
126+
"\n",
125127
"npart = 10\n",
126128
"z = np.repeat(ds_fields.depth[0].values, npart)\n",
127129
"lons = np.repeat(32.2, npart)\n",

‎docs/user_guide/examples/tutorial_sampling.ipynb‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,12 +60,14 @@
6060
"ds_fields = parcels.tutorial.open_dataset(\n",
6161
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
6262
")\n",
63-
"ds_fields.load() # load the dataset into memory\n",
6463
"\n",
6564
"# Convert to SGRID-compliant dataset and create FieldSet\n",
6665
"fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"], \"thetao\": ds_fields[\"thetao\"]}\n",
6766
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
68-
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)"
67+
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
68+
"\n",
69+
"# Convert the FieldSet to windowed arrays for better performance\n",
70+
"fieldset = fieldset.to_windowed_arrays()"
6971
]
7072
},
7173
{

‎docs/user_guide/examples/tutorial_write_in_kernel.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,12 +122,12 @@
122122
"ds_fields = parcels.tutorial.open_dataset(\n",
123123
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
124124
")\n",
125-
"ds_fields.load() # load the dataset into memory\n",
126125
"\n",
127126
"# Convert to SGRID-compliant dataset and create FieldSet\n",
128127
"fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"]}\n",
129128
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
130129
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
130+
"fieldset = fieldset.to_windowed_arrays()\n",
131131
"\n",
132132
"pset = parcels.ParticleSet(fieldset=fieldset, x=[31, 32], y=[-32, -31], z=[1, 1])\n",
133133
"\n",

‎docs/user_guide/getting_started/tutorial_output.ipynb‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,12 +58,14 @@
5858
"ds_fields = parcels.tutorial.open_dataset(\n",
5959
" \"CopernicusMarine_data_for_Argo_tutorial/data\"\n",
6060
")\n",
61-
"ds_fields.load() # load the dataset into memory\n",
6261
"\n",
6362
"# Convert to SGRID-compliant dataset and create FieldSet\n",
6463
"fields = {\"U\": ds_fields[\"uo\"], \"V\": ds_fields[\"vo\"]}\n",
6564
"ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)\n",
66-
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)"
65+
"fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)\n",
66+
"\n",
67+
"# Convert the FieldSet to windowed arrays for better performance\n",
68+
"fieldset = fieldset.to_windowed_arrays()"
6769
]
6870
},
6971
{

‎docs/user_guide/getting_started/tutorial_quickstart.md‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,6 @@ hydrodynamics fields in which the particles are tracked. Here we provide an exam
3131

3232
```{code-cell}
3333
ds_fields = parcels.tutorial.open_dataset("CopernicusMarine_data_for_Argo_tutorial/data")
34-
35-
ds_fields.load() # load the dataset into memory
3634
ds_fields
3735
```
3836

@@ -52,6 +50,15 @@ ds_fset = parcels.convert.copernicusmarine_to_sgrid(fields=fields)
5250
fieldset = parcels.FieldSet.from_sgrid_conventions(ds_fset)
5351
```
5452

53+
Now, in order to improve performance, we can convert the `parcels.FieldSet` to windowed arrays. This is especially useful for large datasets with many timeslices, as it allows Parcels to load only the necessary timeslices into memory during the simulation.
54+
55+
#TODO add link to performance explanation notebook here.
56+
57+
```{code-cell}
58+
# Convert the FieldSet to windowed arrays for better performance
59+
fieldset = fieldset.to_windowed_arrays()
60+
```
61+
5562
You can inspect the `parcels.FieldSet` object with the `describe` method in order to see which `parcels.Field`s are included, and which grid and interpolation method is used for each field. This also gives information on the type of mesh and the time interval of the `parcels.FieldSet`:
5663

5764
```{code-cell}

0 commit comments

Comments
 (0)