Skip to content

Commit f285965

Browse files
paren8esisparen8esis
committed
Merge branch 'fix_notebook_and_dataset_paths'
2 parents fe3a303 + 996d657 commit f285965

3 files changed

Lines changed: 108 additions & 111 deletions

File tree

‎check_dataset.ipynb‎

Lines changed: 13 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
"name": "stderr",
1111
"output_type": "stream",
1212
"text": [
13+
"/home/masdra/.local/lib/python3.9/site-packages/xarray/backends/cfgrib_.py:27: UserWarning: Failed to load cfgrib - most likely there is a problem accessing the ecCodes library. Try `import cfgrib` to get the full error message\n",
14+
" warnings.warn(\n",
1315
"Global seed set to 16\n",
1416
"Global seed set to 16\n"
1517
]
@@ -55,42 +57,30 @@
5557
},
5658
{
5759
"cell_type": "code",
58-
"execution_count": 3,
60+
"execution_count": 7,
5961
"id": "0869a4fc-6c74-4a5a-8863-567728057f32",
6062
"metadata": {},
6163
"outputs": [],
6264
"source": [
6365
"# Define paths\n",
64-
"root_path_coco = Path('/home/masdra/NOA/SAgNet/dataset')\n",
65-
"coco_train = root_path_coco / 'poc_coco_train.json'\n",
66-
"coco_val = root_path_coco / 'poc_coco_val.json'"
66+
"root_coco_path = Path('coco_files')\n",
67+
"coco_train = root_coco_path / 'poc_coco_train.json'\n",
68+
"coco_val = root_coco_path / 'poc_coco_val.json'\n",
69+
"\n",
70+
"root_data_path = Path('dataset/')\n",
71+
"netcdf_path = root_data_path / 'netcdf'"
6772
]
6873
},
6974
{
7075
"cell_type": "code",
71-
"execution_count": 6,
76+
"execution_count": null,
7277
"id": "b26b8c78-f65b-497b-8955-d7712ca20f72",
7378
"metadata": {},
74-
"outputs": [
75-
{
76-
"name": "stdout",
77-
"output_type": "stream",
78-
"text": [
79-
"loading annotations into memory...\n",
80-
"Done (t=0.03s)\n",
81-
"creating index...\n",
82-
"index created!\n",
83-
"loading annotations into memory...\n",
84-
"Done (t=0.02s)\n",
85-
"creating index...\n",
86-
"index created!\n"
87-
]
88-
}
89-
],
79+
"outputs": [],
9080
"source": [
9181
"# Create Data Module\n",
92-
"dm = PatchesDataModule(\n",
93-
" root_path_coco=root_path_coco,\n",
82+
"dm = PADDataModule(\n",
83+
" netcdf_path=netcdf_path,\n",
9484
" path_train=coco_train,\n",
9585
" path_val=coco_val,\n",
9686
" group_freq='1MS',\n",

‎utils/PAD_datamodule.py‎

Lines changed: 94 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ class PADDataModule(pl.LightningDataModule):
2222

2323
def __init__(
2424
self,
25-
root_path_coco: Union[str, Path] = Path(),
25+
netcdf_path: Union[str, Path] = Path(),
2626
path_train: Union[str, Path] = Path(),
2727
path_val: Union[str, Path] = Path(),
2828
path_test: Union[str, Path] = Path(),
@@ -52,11 +52,11 @@ def __init__(
5252
----------
5353
root_path_coco: Path or str
5454
The path containing the COCO files.
55-
path_train: Path or str, default Path('coco_train.json')
56-
The file path containing the training data.
57-
path_val: Path or str, default Path('coco_val.json')
55+
netcdf_path: Path or str
56+
The path containing the training data (netCDF files).
57+
path_val: Path or str
5858
The file path containing the validation data.
59-
path_test: Path or str, default Path('coco_test.json')
59+
path_test: Path or str
6060
The file path containing the testing data.
6161
bands: list of str, default None
6262
A list of the bands to use. If None, then all available bands are
@@ -114,7 +114,7 @@ def __init__(
114114

115115
super().__init__()
116116

117-
self.root_path_coco = root_path_coco
117+
self.netcdf_path = netcdf_path
118118

119119
self.path_train = Path(path_train)
120120
self.path_val = Path(path_val)
@@ -185,97 +185,103 @@ def setup(self, stage=None):
185185
coco_train = COCO(self.path_train)
186186
coco_val = COCO(self.path_val)
187187

188-
self.dataset_train = PADDataset(root_path_coco=self.root_path_coco,
189-
coco=coco_train,
190-
# transforms=transforms,
191-
group_freq=self.group_freq,
192-
compression=self.compression,
193-
prefix=self.prefix,
194-
bands=self.bands,
195-
linear_encoder=self.linear_encoder,
196-
saved_medians=self.saved_medians,
197-
window_len=self.window_len,
198-
fixed_window=self.fixed_window,
199-
requires_norm=self.requires_norm,
200-
return_masks=self.return_masks,
201-
clouds=self.clouds,
202-
cirrus=self.cirrus,
203-
shadow=self.shadow,
204-
snow=self.snow,
205-
output_size=self.output_size,
206-
binary_labels=self.binary_labels,
207-
mode='train',
208-
return_parcels=self.return_parcels
209-
)
188+
self.dataset_train = PADDataset(
189+
root_path_netcdf=self.netcdf_path,
190+
coco=coco_train,
191+
# transforms=transforms,
192+
group_freq=self.group_freq,
193+
compression=self.compression,
194+
prefix=self.prefix,
195+
bands=self.bands,
196+
linear_encoder=self.linear_encoder,
197+
saved_medians=self.saved_medians,
198+
window_len=self.window_len,
199+
fixed_window=self.fixed_window,
200+
requires_norm=self.requires_norm,
201+
return_masks=self.return_masks,
202+
clouds=self.clouds,
203+
cirrus=self.cirrus,
204+
shadow=self.shadow,
205+
snow=self.snow,
206+
output_size=self.output_size,
207+
binary_labels=self.binary_labels,
208+
mode='train',
209+
return_parcels=self.return_parcels
210+
)
210211

211-
self.dataset_eval = PADDataset(root_path_coco=self.root_path_coco,
212-
coco=coco_val,
213-
group_freq=self.group_freq,
214-
compression=self.compression,
215-
prefix=self.prefix,
216-
bands=self.bands,
217-
linear_encoder=self.linear_encoder,
218-
saved_medians=self.saved_medians,
219-
window_len=self.window_len,
220-
fixed_window=self.fixed_window,
221-
requires_norm=self.requires_norm,
222-
return_masks=self.return_masks,
223-
clouds=self.clouds,
224-
cirrus=self.cirrus,
225-
shadow=self.shadow,
226-
snow=self.snow,
227-
output_size=self.output_size,
228-
binary_labels=self.binary_labels,
229-
mode='val',
230-
return_parcels=self.return_parcels
231-
)
212+
self.dataset_eval = PADDataset(
213+
root_path_netcdf=self.netcdf_path,
214+
coco=coco_val,
215+
group_freq=self.group_freq,
216+
compression=self.compression,
217+
prefix=self.prefix,
218+
bands=self.bands,
219+
linear_encoder=self.linear_encoder,
220+
saved_medians=self.saved_medians,
221+
window_len=self.window_len,
222+
fixed_window=self.fixed_window,
223+
requires_norm=self.requires_norm,
224+
return_masks=self.return_masks,
225+
clouds=self.clouds,
226+
cirrus=self.cirrus,
227+
shadow=self.shadow,
228+
snow=self.snow,
229+
output_size=self.output_size,
230+
binary_labels=self.binary_labels,
231+
mode='val',
232+
return_parcels=self.return_parcels
233+
)
232234

233235
else:
234236
# Setup datasets for testing
235237
coco_test = COCO(self.path_test)
236238

237-
self.dataset_test = PADDataset(root_path_coco=self.root_path_coco,
238-
coco=coco_test,
239-
group_freq=self.group_freq,
240-
compression=self.compression,
241-
prefix=self.prefix,
242-
bands=self.bands,
243-
linear_encoder=self.linear_encoder,
244-
saved_medians=self.saved_medians,
245-
window_len=self.window_len,
246-
fixed_window=self.fixed_window,
247-
requires_norm=self.requires_norm,
248-
return_masks=self.return_masks,
249-
clouds=self.clouds,
250-
cirrus=self.cirrus,
251-
shadow=self.shadow,
252-
snow=self.snow,
253-
output_size=self.output_size,
254-
binary_labels=self.binary_labels,
255-
mode='test',
256-
return_parcels=self.return_parcels
257-
)
239+
self.dataset_test = PADDataset(
240+
root_path_netcdf=self.netcdf_path,
241+
coco=coco_test,
242+
group_freq=self.group_freq,
243+
compression=self.compression,
244+
prefix=self.prefix,
245+
bands=self.bands,
246+
linear_encoder=self.linear_encoder,
247+
saved_medians=self.saved_medians,
248+
window_len=self.window_len,
249+
fixed_window=self.fixed_window,
250+
requires_norm=self.requires_norm,
251+
return_masks=self.return_masks,
252+
clouds=self.clouds,
253+
cirrus=self.cirrus,
254+
shadow=self.shadow,
255+
snow=self.snow,
256+
output_size=self.output_size,
257+
binary_labels=self.binary_labels,
258+
mode='test',
259+
return_parcels=self.return_parcels
260+
)
258261

259262
def train_dataloader(self):
260-
return DataLoader(self.dataset_train,
261-
batch_size=self.batch_size,
262-
shuffle=True,
263-
num_workers=self.num_workers,
264-
pin_memory=True
265-
)
263+
return DataLoader(
264+
self.dataset_train,
265+
batch_size=self.batch_size,
266+
shuffle=True,
267+
num_workers=self.num_workers,
268+
pin_memory=True
269+
)
266270

267271
def val_dataloader(self):
268-
return DataLoader(self.dataset_eval,
269-
batch_size=self.batch_size,
270-
shuffle=False,
271-
num_workers=self.num_workers,
272-
pin_memory=True
273-
)
272+
return DataLoader(
273+
self.dataset_eval,
274+
batch_size=self.batch_size,
275+
shuffle=False,
276+
num_workers=self.num_workers,
277+
pin_memory=True
278+
)
274279

275280
def test_dataloader(self):
276-
return DataLoader(self.dataset_test,
277-
batch_size=self.batch_size,
278-
shuffle=False,
279-
num_workers=self.num_workers,
280-
pin_memory=True
281-
)
281+
return DataLoader(
282+
self.dataset_test,
283+
batch_size=self.batch_size,
284+
shuffle=False,
285+
num_workers=self.num_workers,
286+
pin_memory=True
287+
)

‎utils/PAD_dataset.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,6 +269,7 @@ def get_padding_offset(self):
269269

270270
return pad_top, pad_bot, pad_left, pad_right
271271

272+
272273
def get_medians(self, netcdf: netCDF4.Dataset, start_bin: int, window: int) -> np.ndarray:
273274
"""
274275
netcdf (.nc): NETCDF4 Dataset.

0 commit comments

Comments
 (0)