From f9e8ebfdb0397a9e0e731bd55f8ebe082618820c Mon Sep 17 00:00:00 2001 From: 97harsh Date: Sun, 15 Nov 2020 22:54:34 +0530 Subject: [PATCH 01/21] Adding baseic model --- .gitignore | 6 +- .../deep_ar/config/lstm_kwargs.json | 18 +++ flood_forecast/deep_ar/model.py | 109 ++++++++++++++++++ flood_forecast/model_dict_function.py | 4 +- 4 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 flood_forecast/deep_ar/config/lstm_kwargs.json create mode 100644 flood_forecast/deep_ar/model.py diff --git a/.gitignore b/.gitignore index 83d42b3cc..00b575eec 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,8 @@ tests/output/ data mypy .mypy_cache -*.png \ No newline at end of file +*.png +.vscode +.vscode/* +.idea +.idea/ \ No newline at end of file diff --git a/flood_forecast/deep_ar/config/lstm_kwargs.json b/flood_forecast/deep_ar/config/lstm_kwargs.json new file mode 100644 index 000000000..f2d86f88c --- /dev/null +++ b/flood_forecast/deep_ar/config/lstm_kwargs.json @@ -0,0 +1,18 @@ +{ + "learning_rate": 1e-3, + "batch_size": 64, + "lstm_layers": 3, + "num_epochs": 20, + "train_window": 192, + "test_window": 192, + "predict_start": 168, + "test_predict_start": 168, + "predict_steps": 24, + "num_class": 370, + "cov_dim": 4, + "lstm_hidden_dim": 40, + "embedding_dim": 20, + "sample_times": 200, + "lstm_dropout": 0.1, + "predict_batch": 256 +} \ No newline at end of file diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py new file mode 100644 index 000000000..1803b503a --- /dev/null +++ b/flood_forecast/deep_ar/model.py @@ -0,0 +1,109 @@ +import math +import logging + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + + +class DeepAR(nn.Module): + def __init__(self, params): + ''' + We define a recurrent network that predicts the future values of a time-dependent variable based on + past inputs and covariates. + ''' + super(DeepAR, self).__init__() + self.params = params + self.embedding = nn.Embedding(params["num_class"], params["embedding_dim"]) + + self.lstm = nn.LSTM(input_size=1+params["cov_dim"]+params["embedding_dim"], + hidden_size=params["lstm_hidden_dim"], + num_layers=params["lstm_layers"], + bias=True, + batch_first=False, + dropout=params["lstm_dropout"]) + '''self.lstm = nn.LSTM(input_size=1 + params.cov_dim, + hidden_size=params.lstm_hidden_dim, + num_layers=params.lstm_layers, + bias=True, + batch_first=False, + dropout=params.lstm_dropout)''' + # initialize LSTM forget gate bias to be 1 as recommanded by http://proceedings.mlr.press/v37/jozefowicz15.pdf + for names in self.lstm._all_weights: + for name in filter(lambda n: "bias" in n, names): + bias = getattr(self.lstm, name) + n = bias.size(0) + start, end = n // 4, n // 2 + bias.data[start:end].fill_(1.) + + self.relu = nn.ReLU() + self.distribution_mu = nn.Linear(params["lstm_hidden_dim"] * params["lstm_layers"], 1) + self.distribution_presigma = nn.Linear(params["lstm_hidden_dim"] * params["lstm_layers"], 1) + self.distribution_sigma = nn.Softplus() + + def forward(self, x, idx, hidden, cell): + ''' + Predict mu and sigma of the distribution for z_t. + Args: + x: ([1, batch_size, 1+cov_dim]): z_{t-1} + x_t, note that z_0 = 0 + idx ([1, batch_size]): one integer denoting the time series id + hidden ([lstm_layers, batch_size, lstm_hidden_dim]): LSTM h from time step t-1 + cell ([lstm_layers, batch_size, lstm_hidden_dim]): LSTM c from time step t-1 + Returns: + mu ([batch_size]): estimated mean of z_t + sigma ([batch_size]): estimated standard deviation of z_t + hidden ([lstm_layers, batch_size, lstm_hidden_dim]): LSTM h from time step t + cell ([lstm_layers, batch_size, lstm_hidden_dim]): LSTM c from time step t + ''' + onehot_embed = self.embedding(idx) #TODO: is it possible to do this only once per window instead of per step? + lstm_input = torch.cat((x, onehot_embed), dim=2) + output, (hidden, cell) = self.lstm(lstm_input, (hidden, cell)) + # use h from all three layers to calculate mu and sigma + hidden_permute = hidden.permute(1, 2, 0).contiguous().view(hidden.shape[1], -1) + pre_sigma = self.distribution_presigma(hidden_permute) + mu = self.distribution_mu(hidden_permute) + sigma = self.distribution_sigma(pre_sigma) # softplus to make sure standard deviation is positive + return torch.squeeze(mu), torch.squeeze(sigma), hidden, cell + + def init_hidden(self, input_size): + return torch.zeros(self.params.lstm_layers, input_size, self.params.lstm_hidden_dim, device=self.params.device) + + def init_cell(self, input_size): + return torch.zeros(self.params.lstm_layers, input_size, self.params.lstm_hidden_dim, device=self.params.device) + + def test(self, x, v_batch, id_batch, hidden, cell, sampling=False): + batch_size = x.shape[1] + if sampling: + samples = torch.zeros(self.params.sample_times, batch_size, self.params.predict_steps, + device=self.params.device) + for j in range(self.params.sample_times): + decoder_hidden = hidden + decoder_cell = cell + for t in range(self.params.predict_steps): + mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params.predict_start + t].unsqueeze(0), + id_batch, decoder_hidden, decoder_cell) + gaussian = torch.distributions.normal.Normal(mu_de, sigma_de) + pred = gaussian.sample() # not scaled + samples[j, :, t] = pred * v_batch[:, 0] + v_batch[:, 1] + if t < (self.params.predict_steps - 1): + x[self.params.predict_start + t + 1, :, 0] = pred + + sample_mu = torch.median(samples, dim=0)[0] + sample_sigma = samples.std(dim=0) + return samples, sample_mu, sample_sigma + + else: + decoder_hidden = hidden + decoder_cell = cell + sample_mu = torch.zeros(batch_size, self.params.predict_steps, device=self.params.device) + sample_sigma = torch.zeros(batch_size, self.params.predict_steps, device=self.params.device) + for t in range(self.params.predict_steps): + mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params.predict_start + t].unsqueeze(0), + id_batch, decoder_hidden, decoder_cell) + sample_mu[:, t] = mu_de * v_batch[:, 0] + v_batch[:, 1] + sample_sigma[:, t] = sigma_de * v_batch[:, 0] + if t < (self.params.predict_steps - 1): + x[self.params.predict_start + t + 1, :, 0] = mu_de + return sample_mu, sample_sigma diff --git a/flood_forecast/model_dict_function.py b/flood_forecast/model_dict_function.py index 95317a823..22eac3b83 100644 --- a/flood_forecast/model_dict_function.py +++ b/flood_forecast/model_dict_function.py @@ -14,6 +14,7 @@ from flood_forecast.transformer_xl.transformer_bottleneck import DecoderTransformer from flood_forecast.custom.dilate_loss import DilateLoss from flood_forecast.meta_models.basic_ae import AE +from flood_forecast.deep_ar.model import DeepAR import torch """ @@ -29,7 +30,8 @@ "CustomTransformerDecoder": CustomTransformerDecoder, "DARNN": DARNN, "DecoderTransformer": DecoderTransformer, - "BasicAE": AE + "BasicAE": AE, + "DeepAR": DeepAR } From 6bf9fff624bd13acfafe67aca09a252289385e84 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Sun, 15 Nov 2020 23:16:57 +0530 Subject: [PATCH 02/21] Initial commit for DeepAR --- flood_forecast/deep_ar/model.py | 78 ++++++++++++++++++++------------- tests/DeepAR_test.json | 72 ++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 31 deletions(-) create mode 100644 tests/DeepAR_test.json diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index 1803b503a..32359d12b 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -1,29 +1,42 @@ -import math -import logging -import numpy as np import torch import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Variable class DeepAR(nn.Module): - def __init__(self, params): + def __init__(self, + num_class, + embedding_dim, + lstm_hidden_dim, + lstm_layers, + device, + sample_times, + predict_steps, + predict_start + ): ''' We define a recurrent network that predicts the future values of a time-dependent variable based on past inputs and covariates. ''' super(DeepAR, self).__init__() - self.params = params - self.embedding = nn.Embedding(params["num_class"], params["embedding_dim"]) + self.params = {} + self.params["num_class"] = num_class + self.params["embedding_dim"] = embedding_dim + self.params["lstm_hidden_dim"] = lstm_hidden_dim + self.params["lstm_layers"] = lstm_layers + self.params["device"] = device + self.params["sample_times"] = sample_times + self.params["predict_steps"] = predict_steps + self.params["predict_start"] = predict_start + # self.params = params + self.embedding = nn.Embedding(self.params["num_class"], self.params["embedding_dim"]) - self.lstm = nn.LSTM(input_size=1+params["cov_dim"]+params["embedding_dim"], - hidden_size=params["lstm_hidden_dim"], - num_layers=params["lstm_layers"], + self.lstm = nn.LSTM(input_size=1 + self.params["cov_dim"] + self.params["embedding_dim"], + hidden_size=self.params["lstm_hidden_dim"], + num_layers=self.params["lstm_layers"], bias=True, batch_first=False, - dropout=params["lstm_dropout"]) + dropout=self.params["lstm_dropout"]) '''self.lstm = nn.LSTM(input_size=1 + params.cov_dim, hidden_size=params.lstm_hidden_dim, num_layers=params.lstm_layers, @@ -39,8 +52,8 @@ def __init__(self, params): bias.data[start:end].fill_(1.) self.relu = nn.ReLU() - self.distribution_mu = nn.Linear(params["lstm_hidden_dim"] * params["lstm_layers"], 1) - self.distribution_presigma = nn.Linear(params["lstm_hidden_dim"] * params["lstm_layers"], 1) + self.distribution_mu = nn.Linear(self.params["lstm_hidden_dim"] * self.params["lstm_layers"], 1) + self.distribution_presigma = nn.Linear(self.params["lstm_hidden_dim"] * self.params["lstm_layers"], 1) self.distribution_sigma = nn.Softplus() def forward(self, x, idx, hidden, cell): @@ -57,7 +70,7 @@ def forward(self, x, idx, hidden, cell): hidden ([lstm_layers, batch_size, lstm_hidden_dim]): LSTM h from time step t cell ([lstm_layers, batch_size, lstm_hidden_dim]): LSTM c from time step t ''' - onehot_embed = self.embedding(idx) #TODO: is it possible to do this only once per window instead of per step? + onehot_embed = self.embedding(idx) # TODO: is it possible to do this only once per window instead of per step? lstm_input = torch.cat((x, onehot_embed), dim=2) output, (hidden, cell) = self.lstm(lstm_input, (hidden, cell)) # use h from all three layers to calculate mu and sigma @@ -68,27 +81,30 @@ def forward(self, x, idx, hidden, cell): return torch.squeeze(mu), torch.squeeze(sigma), hidden, cell def init_hidden(self, input_size): - return torch.zeros(self.params.lstm_layers, input_size, self.params.lstm_hidden_dim, device=self.params.device) + return torch.zeros(self.params["lstm_layers"], input_size, self.params["lstm_hidden_dim"], + device=self.params["device"]) def init_cell(self, input_size): - return torch.zeros(self.params.lstm_layers, input_size, self.params.lstm_hidden_dim, device=self.params.device) + return torch.zeros(self.params["lstm_layers"], input_size, self.params["lstm_hidden_dim"], + device=self.params["device"]) def test(self, x, v_batch, id_batch, hidden, cell, sampling=False): batch_size = x.shape[1] if sampling: - samples = torch.zeros(self.params.sample_times, batch_size, self.params.predict_steps, - device=self.params.device) - for j in range(self.params.sample_times): + samples = torch.zeros(self.params["sample_times"], batch_size, self.params["predict_steps"], + device=self.params["device"]) + for j in range(self.params["sample_times"]): decoder_hidden = hidden decoder_cell = cell - for t in range(self.params.predict_steps): - mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params.predict_start + t].unsqueeze(0), - id_batch, decoder_hidden, decoder_cell) + for t in range(self.params["predict_steps"]): + mu_de, sigma_de, decoder_hidden, decoder_cell = self( + x[self.params["predict_start"] + t].unsqueeze(0), + id_batch, decoder_hidden, decoder_cell) gaussian = torch.distributions.normal.Normal(mu_de, sigma_de) pred = gaussian.sample() # not scaled samples[j, :, t] = pred * v_batch[:, 0] + v_batch[:, 1] - if t < (self.params.predict_steps - 1): - x[self.params.predict_start + t + 1, :, 0] = pred + if t < (self.params["predict_steps"] - 1): + x[self.params["predict_start"] + t + 1, :, 0] = pred sample_mu = torch.median(samples, dim=0)[0] sample_sigma = samples.std(dim=0) @@ -97,13 +113,13 @@ def test(self, x, v_batch, id_batch, hidden, cell, sampling=False): else: decoder_hidden = hidden decoder_cell = cell - sample_mu = torch.zeros(batch_size, self.params.predict_steps, device=self.params.device) - sample_sigma = torch.zeros(batch_size, self.params.predict_steps, device=self.params.device) - for t in range(self.params.predict_steps): - mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params.predict_start + t].unsqueeze(0), + sample_mu = torch.zeros(batch_size, self.params["predict_steps"], device=self.params["device"]) + sample_sigma = torch.zeros(batch_size, self.params["predict_steps"], device=self.params["device"]) + for t in range(self.params["predict_steps"]): + mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params["predict_start"] + t].unsqueeze(0), id_batch, decoder_hidden, decoder_cell) sample_mu[:, t] = mu_de * v_batch[:, 0] + v_batch[:, 1] sample_sigma[:, t] = sigma_de * v_batch[:, 0] - if t < (self.params.predict_steps - 1): - x[self.params.predict_start + t + 1, :, 0] = mu_de + if t < (self.params["predict_steps"] - 1): + x[self.params["predict_start"] + t + 1, :, 0] = mu_de return sample_mu, sample_sigma diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json new file mode 100644 index 000000000..5b9200712 --- /dev/null +++ b/tests/DeepAR_test.json @@ -0,0 +1,72 @@ +{ + "model_name": "DeepAR", + "model_type": "PyTorch", + "model_params": { + + }, + "dataset_params": + { "class": "default", + "training_path": "tests/test_data/keag_small.csv", + "validation_path": "tests/test_data/keag_small.csv", + "test_path": "tests/test_data/keag_small.csv", + "batch_size":4, + "forecast_history":5, + "forecast_length":1, + "train_end": 100, + "valid_start":301, + "valid_end": 401, + "test_start":125, + "test_end":500, + "target_col": ["cfs"], + "relevant_cols": ["cfs", "precip", "temp"], + "scaler": "StandardScaler", + "interpolate": false + }, + "early_stopping":{ + "patience":1 + + }, + "training_params": + { + "criterion":"MAPE", + "optimizer": "Adam", + "optim_params": + { + + }, + "lr": 0.3, + "epochs": 20, + "batch_size":4 + + }, + "GCS": false, + + "wandb": { + "name": "flood_forecast_circleci", + "project": "repo-flood_forecast", + "tags": ["dummy_run", "circleci"] + }, + "forward_params":{}, + "metrics":["MSE"], + "inference_params": + { + "datetime_start":"2016-05-31", + "hours_to_forecast":336, + "test_csv_path":"tests/test_data/keag_small.csv", + "decoder_params":{ + "decoder_function": "simple_decode", + "unsqueeze_dim": 1}, + "dataset_params":{ + "file_path": "tests/test_data/keag_small.csv", + "forecast_history":5, + "forecast_length":1, + "relevant_cols": ["cfs", "precip", "temp"], + "target_col": ["cfs"], + "scaling": "StandardScaler", + "interpolate_param": false + } + } + + +} + \ No newline at end of file From e8dd08f892cef8b941e554f93859aaaa45d6e665 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Sun, 15 Nov 2020 23:32:00 +0530 Subject: [PATCH 03/21] Initial commit for DeepAR --- .../deep_ar/config/lstm_kwargs.json | 20 ++++---- flood_forecast/deep_ar/model.py | 4 ++ tests/DeepAR_test.json | 50 +++++++------------ 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/flood_forecast/deep_ar/config/lstm_kwargs.json b/flood_forecast/deep_ar/config/lstm_kwargs.json index f2d86f88c..69f373d86 100644 --- a/flood_forecast/deep_ar/config/lstm_kwargs.json +++ b/flood_forecast/deep_ar/config/lstm_kwargs.json @@ -1,18 +1,18 @@ { - "learning_rate": 1e-3, "batch_size": 64, + "cov_dim": 4, + "embedding_dim": 20, + "learning_rate": 1e-3, + "lstm_dropout": 0.1, + "lstm_hidden_dim": 40, "lstm_layers": 3, + "num_class": 370, "num_epochs": 20, - "train_window": 192, - "test_window": 192, + "predict_batch": 256, "predict_start": 168, - "test_predict_start": 168, "predict_steps": 24, - "num_class": 370, - "cov_dim": 4, - "lstm_hidden_dim": 40, - "embedding_dim": 20, "sample_times": 200, - "lstm_dropout": 0.1, - "predict_batch": 256 + "test_predict_start": 168, + "test_window": 192, + "train_window": 192 } \ No newline at end of file diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index 32359d12b..97087769e 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -6,6 +6,8 @@ class DeepAR(nn.Module): def __init__(self, num_class, + cov_dim, + lstm_dropout, embedding_dim, lstm_hidden_dim, lstm_layers, @@ -21,6 +23,8 @@ def __init__(self, super(DeepAR, self).__init__() self.params = {} self.params["num_class"] = num_class + self.params["cov_dim"] = cov_dim + self.params["lstm_dropout"] = lstm_dropout self.params["embedding_dim"] = embedding_dim self.params["lstm_hidden_dim"] = lstm_hidden_dim self.params["lstm_layers"] = lstm_layers diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 5b9200712..c7633995b 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -2,6 +2,21 @@ "model_name": "DeepAR", "model_type": "PyTorch", "model_params": { + "cov_dim": 1, + "embedding_dim": 1, + "learning_rate": 1e-3, + "lstm_dropout": 0.1, + "lstm_hidden_dim": 40, + "lstm_layers": 1, + "num_class": 1, + "num_epochs": 1, + "predict_batch": 256, + "predict_start": 5, + "predict_steps": 24, + "sample_times": 200, + "test_predict_start": 168, + "test_window": 192, + "train_window": 192 }, "dataset_params": @@ -15,12 +30,10 @@ "train_end": 100, "valid_start":301, "valid_end": 401, - "test_start":125, + "test_predict_start":125, "test_end":500, "target_col": ["cfs"], - "relevant_cols": ["cfs", "precip", "temp"], - "scaler": "StandardScaler", - "interpolate": false + "relevant_cols": ["cfs", "precip", "temp"] }, "early_stopping":{ "patience":1 @@ -30,11 +43,7 @@ { "criterion":"MAPE", "optimizer": "Adam", - "optim_params": - { - - }, - "lr": 0.3, + "lr": 1e-3, "epochs": 20, "batch_size":4 @@ -46,27 +55,6 @@ "project": "repo-flood_forecast", "tags": ["dummy_run", "circleci"] }, - "forward_params":{}, - "metrics":["MSE"], - "inference_params": - { - "datetime_start":"2016-05-31", - "hours_to_forecast":336, - "test_csv_path":"tests/test_data/keag_small.csv", - "decoder_params":{ - "decoder_function": "simple_decode", - "unsqueeze_dim": 1}, - "dataset_params":{ - "file_path": "tests/test_data/keag_small.csv", - "forecast_history":5, - "forecast_length":1, - "relevant_cols": ["cfs", "precip", "temp"], - "target_col": ["cfs"], - "scaling": "StandardScaler", - "interpolate_param": false - } - } - - + "metrics":["MSE"] } \ No newline at end of file From 5bf8088eef047211fa5af9c9b5cd51eed4302859 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 00:25:00 +0530 Subject: [PATCH 04/21] config file updation --- .circleci/config.yml | 2 ++ flood_forecast/deep_ar/model.py | 20 ++++++++++---------- tests/DeepAR_test.json | 4 ++-- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bd1066a4..0e6200102 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,6 +229,8 @@ jobs: name: Trainer tests when: always command: | + echo -e 'running Deep_AR_test \n' + coverage run flood_forecast/trainer.py -p tests/DeepAR_test.json echo -e 'running da-meta data unit test' coverage run flood_forecast/trainer.py -p tests/da_meta.json echo -e 'running transformer bottleneck' diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index 97087769e..fb5651654 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -5,16 +5,16 @@ class DeepAR(nn.Module): def __init__(self, - num_class, - cov_dim, - lstm_dropout, - embedding_dim, - lstm_hidden_dim, - lstm_layers, - device, - sample_times, - predict_steps, - predict_start + num_class: int, + cov_dim: int, + lstm_dropout: float, + embedding_dim: int, + lstm_hidden_dim: int, + lstm_layers: int, + device: str, + sample_times: int, + predict_steps: int, + predict_start: int ): ''' We define a recurrent network that predicts the future values of a time-dependent variable based on diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index c7633995b..f5644c059 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -14,7 +14,7 @@ "predict_start": 5, "predict_steps": 24, "sample_times": 200, - "test_predict_start": 168, + "test_predict_start": 5, "test_window": 192, "train_window": 192 @@ -53,7 +53,7 @@ "wandb": { "name": "flood_forecast_circleci", "project": "repo-flood_forecast", - "tags": ["dummy_run", "circleci"] + "tags": ["dummy_run", "circleci","DeepAR"] }, "metrics":["MSE"] } From f9dc29930ca2b6adc6682847ba78dcbd90e6d11e Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 00:34:02 +0530 Subject: [PATCH 05/21] trainer_test updated --- tests/DeepAR_test.json | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index f5644c059..b3051ceb2 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -4,19 +4,13 @@ "model_params": { "cov_dim": 1, "embedding_dim": 1, - "learning_rate": 1e-3, "lstm_dropout": 0.1, "lstm_hidden_dim": 40, "lstm_layers": 1, "num_class": 1, - "num_epochs": 1, - "predict_batch": 256, "predict_start": 5, "predict_steps": 24, - "sample_times": 200, - "test_predict_start": 5, - "test_window": 192, - "train_window": 192 + "sample_times": 200 }, "dataset_params": From 03aef74862dece2339de9b34cf82af971ffa88ed Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 00:46:27 +0530 Subject: [PATCH 06/21] trainer_test updated1 --- tests/DeepAR_test.json | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index b3051ceb2..546d84428 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -11,7 +11,6 @@ "predict_start": 5, "predict_steps": 24, "sample_times": 200 - }, "dataset_params": { "class": "default", From 716361795edbb0a90b8ec35b9676606d3f7259b0 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 00:57:56 +0530 Subject: [PATCH 07/21] removed device argument --- flood_forecast/deep_ar/model.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index fb5651654..367c0693d 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -11,7 +11,6 @@ def __init__(self, embedding_dim: int, lstm_hidden_dim: int, lstm_layers: int, - device: str, sample_times: int, predict_steps: int, predict_start: int @@ -28,7 +27,6 @@ def __init__(self, self.params["embedding_dim"] = embedding_dim self.params["lstm_hidden_dim"] = lstm_hidden_dim self.params["lstm_layers"] = lstm_layers - self.params["device"] = device self.params["sample_times"] = sample_times self.params["predict_steps"] = predict_steps self.params["predict_start"] = predict_start @@ -85,18 +83,15 @@ def forward(self, x, idx, hidden, cell): return torch.squeeze(mu), torch.squeeze(sigma), hidden, cell def init_hidden(self, input_size): - return torch.zeros(self.params["lstm_layers"], input_size, self.params["lstm_hidden_dim"], - device=self.params["device"]) + return torch.zeros(self.params["lstm_layers"], input_size, self.params["lstm_hidden_dim"]) def init_cell(self, input_size): - return torch.zeros(self.params["lstm_layers"], input_size, self.params["lstm_hidden_dim"], - device=self.params["device"]) + return torch.zeros(self.params["lstm_layers"], input_size, self.params["lstm_hidden_dim"]) def test(self, x, v_batch, id_batch, hidden, cell, sampling=False): batch_size = x.shape[1] if sampling: - samples = torch.zeros(self.params["sample_times"], batch_size, self.params["predict_steps"], - device=self.params["device"]) + samples = torch.zeros(self.params["sample_times"], batch_size, self.params["predict_steps"]) for j in range(self.params["sample_times"]): decoder_hidden = hidden decoder_cell = cell @@ -117,8 +112,8 @@ def test(self, x, v_batch, id_batch, hidden, cell, sampling=False): else: decoder_hidden = hidden decoder_cell = cell - sample_mu = torch.zeros(batch_size, self.params["predict_steps"], device=self.params["device"]) - sample_sigma = torch.zeros(batch_size, self.params["predict_steps"], device=self.params["device"]) + sample_mu = torch.zeros(batch_size, self.params["predict_steps"]) + sample_sigma = torch.zeros(batch_size, self.params["predict_steps"]) for t in range(self.params["predict_steps"]): mu_de, sigma_de, decoder_hidden, decoder_cell = self(x[self.params["predict_start"] + t].unsqueeze(0), id_batch, decoder_hidden, decoder_cell) From 09ace45427a9a5afaeeb0aa556e8a7aed11c8a41 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 01:03:47 +0530 Subject: [PATCH 08/21] changed number of layers to include dropout functionality --- flood_forecast/deep_ar/model.py | 16 ++++++++++++---- tests/DeepAR_test.json | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index 367c0693d..ff3c2ba20 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -15,10 +15,18 @@ def __init__(self, predict_steps: int, predict_start: int ): - ''' - We define a recurrent network that predicts the future values of a time-dependent variable based on - past inputs and covariates. - ''' + """Initialize the DeepAR model. + + :param num_class: Number of classes + :param cov_dim: Number of covariates + :param lstm_dropout: drop out rate + :param embedding_dim: dimension of embedding layer + :param lstm_hidden_dim: hidden dimension of LSTM + :param lstm_layers: Number of LSTM layers + :param sample_times: sample time steps + :param predict_steps: Number of steps to predict + :param predict_start: Step to start prediction at + """ super(DeepAR, self).__init__() self.params = {} self.params["num_class"] = num_class diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 546d84428..a0072d7ec 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -6,7 +6,7 @@ "embedding_dim": 1, "lstm_dropout": 0.1, "lstm_hidden_dim": 40, - "lstm_layers": 1, + "lstm_layers": 2, "num_class": 1, "predict_start": 5, "predict_steps": 24, From 1c918263b79fa1fc1fcfeeaa4a17886fc104a464 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 01:15:41 +0530 Subject: [PATCH 09/21] added back inference params which is required in test --- tests/DeepAR_test.json | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index a0072d7ec..7926691f3 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -2,8 +2,8 @@ "model_name": "DeepAR", "model_type": "PyTorch", "model_params": { - "cov_dim": 1, - "embedding_dim": 1, + "cov_dim": 3, + "embedding_dim": 10, "lstm_dropout": 0.1, "lstm_hidden_dim": 40, "lstm_layers": 2, @@ -11,7 +11,7 @@ "predict_start": 5, "predict_steps": 24, "sample_times": 200 - }, + }, "dataset_params": { "class": "default", "training_path": "tests/test_data/keag_small.csv", @@ -48,6 +48,22 @@ "project": "repo-flood_forecast", "tags": ["dummy_run", "circleci","DeepAR"] }, - "metrics":["MSE"] + "metrics":["MSE"], + "inference_params": + { "num_prediction_samples": 10, + "datetime_start":"2016-05-31", + "hours_to_forecast":336, + "test_csv_path":"tests/test_data/keag_small.csv", + "dataset_params":{ + "file_path": "tests/test_data/keag_small.csv", + "forecast_history":5, + "forecast_length":1, + "relevant_cols": ["cfs", "precip", "temp"], + "target_col": ["cfs"], + "scaling": "StandardScaler", + "interpolate_param": false + } + } + } \ No newline at end of file From a8c4cb7216e232d120ef2d62c1579da379c21e5b Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 01:28:04 +0530 Subject: [PATCH 10/21] added back inference params which is required in test --- tests/DeepAR_test.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 7926691f3..5f4102235 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -59,11 +59,8 @@ "forecast_history":5, "forecast_length":1, "relevant_cols": ["cfs", "precip", "temp"], - "target_col": ["cfs"], - "scaling": "StandardScaler", - "interpolate_param": false + "target_col": ["cfs"] } } - } \ No newline at end of file From 7db3cb07fc83702953d33523c00b222bf578f396 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 01:29:40 +0530 Subject: [PATCH 11/21] removed scaling in dataset --- tests/DeepAR_test.json | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 5f4102235..287d42ade 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -63,4 +63,3 @@ } } } - \ No newline at end of file From 737827205da38bde82c7a8afe5fbdeac1261c888 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 01:41:57 +0530 Subject: [PATCH 12/21] interpolate param added in test file --- tests/DeepAR_test.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 287d42ade..cc78cac8e 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -26,7 +26,8 @@ "test_predict_start":125, "test_end":500, "target_col": ["cfs"], - "relevant_cols": ["cfs", "precip", "temp"] + "relevant_cols": ["cfs", "precip", "temp"], + "interpolate_param": false }, "early_stopping":{ "patience":1 From b719e2389857584fe76c32743870739f915e78e4 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 01:56:12 +0530 Subject: [PATCH 13/21] added interpolate parameter --- tests/DeepAR_test.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index cc78cac8e..74bd9e89a 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -27,6 +27,7 @@ "test_end":500, "target_col": ["cfs"], "relevant_cols": ["cfs", "precip", "temp"], + "interpolate": false, "interpolate_param": false }, "early_stopping":{ @@ -60,7 +61,8 @@ "forecast_history":5, "forecast_length":1, "relevant_cols": ["cfs", "precip", "temp"], - "target_col": ["cfs"] + "target_col": ["cfs"], + "interpolate_param": false } } } From 90d3b5bfc7a9e7e859379a9ab2ba1479f98feca9 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 02:03:45 +0530 Subject: [PATCH 14/21] removed tag from wandb in test --- tests/DeepAR_test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 74bd9e89a..5e502cf89 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -48,7 +48,7 @@ "wandb": { "name": "flood_forecast_circleci", "project": "repo-flood_forecast", - "tags": ["dummy_run", "circleci","DeepAR"] + "tags": ["dummy_run", "circleci"] }, "metrics":["MSE"], "inference_params": From 6d5b86053208b2d23d947d87c8d1756eb46d048f Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 02:12:54 +0530 Subject: [PATCH 15/21] trying things in test to pass wandb issue --- tests/DeepAR_test.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 5e502cf89..9ee97c951 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -46,10 +46,10 @@ "GCS": false, "wandb": { - "name": "flood_forecast_circleci", - "project": "repo-flood_forecast", - "tags": ["dummy_run", "circleci"] - }, + "name": "flood_forecast_circleci", + "tags": ["dummy_run", "circleci", "DeepAR"], + "project": "repo-flood_forecast" + }, "metrics":["MSE"], "inference_params": { "num_prediction_samples": 10, From b4ef9bf9100bb9ccc496d7c7f05a92e21997a4eb Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 02:20:58 +0530 Subject: [PATCH 16/21] changed order in circle config file --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e6200102..ea42ca563 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,10 +229,10 @@ jobs: name: Trainer tests when: always command: | - echo -e 'running Deep_AR_test \n' - coverage run flood_forecast/trainer.py -p tests/DeepAR_test.json echo -e 'running da-meta data unit test' coverage run flood_forecast/trainer.py -p tests/da_meta.json + echo -e 'running Deep_AR_test \n' + coverage run flood_forecast/trainer.py -p tests/DeepAR_test.json echo -e 'running transformer bottleneck' coverage run flood_forecast/trainer.py -p tests/transformer_bottleneck.json echo -e 'running da_rnn probabilistic test' From 006a5d78d71e49daa30324e7e0622e5455671137 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 12:16:32 +0530 Subject: [PATCH 17/21] testing changes1 added wandb api key --- flood_forecast/deep_ar/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index ff3c2ba20..8c3d59aac 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -29,6 +29,7 @@ def __init__(self, """ super(DeepAR, self).__init__() self.params = {} + self.params["num_class"] = num_class self.params["cov_dim"] = cov_dim self.params["lstm_dropout"] = lstm_dropout From 54f8057b8c2e68191649d3f40c642bcdc04d30c8 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 12:28:57 +0530 Subject: [PATCH 18/21] added forward params in test json --- tests/DeepAR_test.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 9ee97c951..9c6c26f44 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -50,6 +50,7 @@ "tags": ["dummy_run", "circleci", "DeepAR"], "project": "repo-flood_forecast" }, + "forward_params":{}, "metrics":["MSE"], "inference_params": { "num_prediction_samples": 10, From 17bbd68d1e069495c4b044de843d4154f7400442 Mon Sep 17 00:00:00 2001 From: 97harsh Date: Mon, 16 Nov 2020 12:36:32 +0530 Subject: [PATCH 19/21] added blank optim params in test --- tests/DeepAR_test.json | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/DeepAR_test.json b/tests/DeepAR_test.json index 9c6c26f44..504f5cfdd 100644 --- a/tests/DeepAR_test.json +++ b/tests/DeepAR_test.json @@ -34,15 +34,21 @@ "patience":1 }, + "training_params": { - "criterion":"MAPE", + "criterion":"NegativeLogLikelihood", + "probabilistic": true, "optimizer": "Adam", - "lr": 1e-3, - "epochs": 20, + "optim_params": + { + }, + "lr": 0.3, + "epochs": 1, "batch_size":4 - - }, + + }, + "GCS": false, "wandb": { From c2f010badbf902641d67f42ad62796baa3cb8665 Mon Sep 17 00:00:00 2001 From: isaacmg Date: Fri, 29 Jan 2021 02:42:42 -0400 Subject: [PATCH 20/21] add things --- flood_forecast/deep_ar/model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index 8c3d59aac..426b68da2 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -66,8 +66,10 @@ def __init__(self, self.distribution_mu = nn.Linear(self.params["lstm_hidden_dim"] * self.params["lstm_layers"], 1) self.distribution_presigma = nn.Linear(self.params["lstm_hidden_dim"] * self.params["lstm_layers"], 1) self.distribution_sigma = nn.Softplus() + self.cell = self.init_cell(1 + self.params["cov_dim"] + self.params["embedding_dim"]) + self.hidden = self.init_hidden(1 + self.params["cov_dim"] + self.params["embedding_dim"]) - def forward(self, x, idx, hidden, cell): + def forward(self, x, idx=0): ''' Predict mu and sigma of the distribution for z_t. Args: @@ -83,7 +85,9 @@ def forward(self, x, idx, hidden, cell): ''' onehot_embed = self.embedding(idx) # TODO: is it possible to do this only once per window instead of per step? lstm_input = torch.cat((x, onehot_embed), dim=2) - output, (hidden, cell) = self.lstm(lstm_input, (hidden, cell)) + output, (hidden, cell) = self.lstm(lstm_input, (self.hidden, self.cell)) + self.cell = cell + self.hidden = hidden # use h from all three layers to calculate mu and sigma hidden_permute = hidden.permute(1, 2, 0).contiguous().view(hidden.shape[1], -1) pre_sigma = self.distribution_presigma(hidden_permute) From 2d71a4e4eda8909071dba33e3d4b8bb2ebd00336 Mon Sep 17 00:00:00 2001 From: isaacmg Date: Fri, 29 Jan 2021 03:19:08 -0400 Subject: [PATCH 21/21] small adjustment --- flood_forecast/deep_ar/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flood_forecast/deep_ar/model.py b/flood_forecast/deep_ar/model.py index 426b68da2..4b8cc8af2 100644 --- a/flood_forecast/deep_ar/model.py +++ b/flood_forecast/deep_ar/model.py @@ -69,7 +69,7 @@ def __init__(self, self.cell = self.init_cell(1 + self.params["cov_dim"] + self.params["embedding_dim"]) self.hidden = self.init_hidden(1 + self.params["cov_dim"] + self.params["embedding_dim"]) - def forward(self, x, idx=0): + def forward(self, x, idx=torch.Tensor(0)): ''' Predict mu and sigma of the distribution for z_t. Args: