Adicionar script de construção do dataset e update da branch

.env

@@ -0,0 +1,19 @@
+INMET_API_BASE_URL=https://apitempo.inmet.gov.br
+WS_INMET_DATA_DIR=./data/ws/inmet/
+WS_ALERTARIO_DATA_DIR=./data/ws/alertario/ws/
+GS_ALERTARIO_DATA_DIR=./data/ws/alertario/rain_gauge_era5_fused/
+CEMADEN_DATA_DIR=.data/ws/cemaden/
+SUMARE_DATA_DIR=.data/radar_sumare
+GOES16_DATA_DIR=./data/goes16/CMI/
+GOES16_FEATURES_DIR=./data/goes16/features/
+NWP_DATA_DIR=./data/NWP/
+AS_DATA_DIR=./data/as/
+DSI_DATA_DIR=./data/goes16/DSI
+TPW_DATA_DIR=./data/goes16/wsoi
+DATASETS_DIR=./data/datasets/
+MODELS_DIR=./models/
+WEBSIRENE_DATA_DIR=./data/ws/websirene/
+REGION_LAT_MIN=-23.801876626302175
+REGION_LON_MIN=-45.05290312102409
+REGION_LAT_MAX=-21.699774257353113
+REGION_LON_MAX=-42.35676996062447

.gitignore

@@ -1,9 +1,16 @@
-.vscode/
+clea.vscode/
 data/
+datasets/
 models/
 backup/
  __pycache__/
  *.pyc
+ *.log
+
+src/spatiotemporal_builder/.github
+
+.DS_Store
+src/.DS_Store
 
 src/__pycache__/globals.cpython-38.pyc
 src/__pycache__/globals.cpython-310.pyc
@@ -17,3 +24,38 @@ src/__pycache__/util.cpython-310.pyc
 src/__pycache__/globals.cpython-310.pyc
 src/__pycache__/util.cpython-310.pyc
 src/__pycache__/goes16_utils.cpython-310.pyc
+nohup.out
+src/__pycache__/
+src/surface_stations/__pycache__/
+*.out
+*.gif
+*.png
+*.nc
+*.parquet
+*.log
+src/spatiotemporal_builder/__pycache__/
+src/spatiotemporal_builder/features/
+anim/dF_dt_2024_01_13/animation.mp4
+notebooks/GPM/imerg_animation_2024_01_01.mp4
+notebooks/GPM/imerg_rio_animation.mp4
+notebooks/GPM/imerg_rio_animation_2024_01_01.mp4
+anim/PN_GLM/animation.mp4
+notebooks/RJ-30m-DEM.tif
+*.tgz
+src/config/__pycache__/__init__.cpython-310.pyc
+src/config/__pycache__/globals.cpython-310.pyc
+src/goes16/__pycache__/__init__.cpython-310.pyc
+src/goes16/__pycache__/features.cpython-310.pyc
+src/goes16/__pycache__/utils.cpython-310.pyc
+src/goes16/__pycache__/goes16_cmi_feature_extractor_extended.cpython-311.pyc
+src/goes16/__pycache__/goes16_cmi_feature_extractor.cpython-311.pyc
+src/goes16/features/__pycache__/__init__.cpython-310.pyc
+src/goes16/features/__pycache__/fa.cpython-310.pyc
+src/goes16/features/__pycache__/gtn.cpython-310.pyc
+src/goes16/features/__pycache__/li_proxy.cpython-310.pyc
+src/goes16/features/__pycache__/pn_std.cpython-310.pyc
+src/goes16/features/__pycache__/pn.cpython-310.pyc
+src/goes16/features/__pycache__/toct.cpython-310.pyc
+src/goes16/features/__pycache__/wv_grad.cpython-310.pyc
+src/utils/__pycache__/__init__.cpython-310.pyc
+src/utils/__pycache__/util.cpython-310.pyc

.pre-commit-config.yaml

@@ -0,0 +1,13 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.4.2
+    hooks:
+      - id: black
+        args: ["--line-length", "88"]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.6
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format

Makefile

@@ -0,0 +1,112 @@
+# === AtmoSeer Makefile ===
+
+# Diretório base para os dados
+DATA_DIR=data
+
+# Diretório base para o código-fonte
+SRC_DIR=src
+
+# ========== Surfaces stations - CEMADEN ========================
+cemaden_retrieve_data:
+	PYTHONPATH=src python3 src/surface_stations/retrieve_data_cemaden.py \
+	  $(if $(IBGE),--ibge $(IBGE)) \
+	  $(if $(START),--inicio $(START)) \
+	  $(if $(END),--fim $(END)) \
+	  $(if $(EMAIL),--email $(EMAIL)) \
+	  $(if $(SENHA),--senha $(SENHA)) \
+	  $(if $(ESTACAO),--estacao $(ESTACAO))
+
+cemaden_retrieve_ws:
+	PYTHONPATH=src python3 src/surface_stations/retrieve_ws_cemaden.py \
+	  $(if $(ESTADO),-e $(ESTADO)) \
+	  $(if $(IDESTACAO),-c $(IDESTACAO)) \
+	  $(if $(INICIO),-start $(INICIO)) \
+	  $(if $(FIM),-end $(FIM)) \
+	  $(if $(SAIDA),-o $(SAIDA)) \
+
+cemaden_mapa:
+	PYTHONPATH=src python3 src/surface_stations/cemaden_mapa.py 
+
+cemaden_mapa_chuva:
+	PYTHONPATH=src python3 src/surface_stations/cemaden_mapa_chuva.py
+
+# ========== Surfaces stations - INMET ========================
+inmet_retrieve_ws:
+	PYTHONPATH=src python3 src/surface_stations/retrieve_ws_inmet.py \
+	  $(if $(API_TOKEN),-t $(API_TOKEN)) \
+	  $(if $(STATION_ID),-s $(STATION_ID)) \
+	  $(if $(BEGIN_YEAR),-b $(BEGIN_YEAR)) \
+	  $(if $(END_YEAR),-e $(END_YEAR))
+# Example usage:
+# make inmet_retrieve_ws API_TOKEN=your_token STATION_ID=A601 BEGIN_YEAR=2020 END_YEAR=2023
+
+surface_stations_preprocess:
+	PYTHONPATH=src python3 src/surface_stations/preprocess.py \
+	  $(if $(STATION_ID),--station_id $(STATION_ID)) \
+	  $(if $(SYSTEM),--station_system $(SYSTEM)) 
+# Example usage:
+# make surface_stations_preprocess STATION_ID=A601 SYSTEM=INMET	
+
+
+# === GOES-16 Downloader/Cropper ===
+goes16-download-crop:
+	PYTHONPATH=src python src/goes16/goes16_download_crop.py \
+	  --start_date $(START) \
+	  --end_date $(END) \
+	  --channel $(CHANNEL) \
+	  --crop_dir $(DIR) \
+	  --spatial_resolution 0.1 \
+	  --vars CMI
+
+# === GOES-16 Feature Extractor ===
+goes16-features:
+	PYTHONPATH=src python src/goes16/main_goes16_features.py $(FEATS)
+
+sumare-crop-images:
+	PYTHONPATH=src python3 src/sumare_radar/crop_image.py
+
+# Validates a GOES-16 feature based on the specified channels
+validate-feature:
+	@echo "Validating feature '$(FEAT)' with channels $(CANAIS)"
+	PYTHONPATH=src python src/goes16/validate_features.py --feature $(FEAT) --canais $(CANAIS)
+
+# Summarize GOES-16 feature extraction log files
+analyze-logs:
+	@echo "Analyzing log files in src/goes16/features/"
+	@PYTHONPATH=src python src/goes16/features/analyze_logs.py
+
+# === GPM Download/Cropper ===
+gpm-download-crop:
+	PYTHONPATH=src python src/gpm/gpm_download_crop.py \
+		--begin_date "$(BEGIN)" \
+		--end_date "$(END)" \
+		--user "$(USER)" \
+		--pwd "$(PWD)" \
+		$(DEBUG) \
+		$(IGNORED)
+
+
+# === Alerta Rio Parser (placeholder) ===
+alerta-rio-process:
+	PYTHONPATH=$(SRC_DIR) python $(SRC_DIR)/surface_stations/alerta_rio_parser.py
+
+
+# === Clean Outputs ===
+clean:
+	rm -rf $(DATA_DIR)/goes16/features/*
+
+.PHONY: format lint lint-fix pre-commit-install
+
+format:
+	ruff format src tests
+	black src tests
+
+lint:
+	ruff check src tests
+
+lint-fix:
+	ruff check src tests --fix
+	black src tests
+
+pre-commit-install:
+	pre-commit install

README.md

@@ -1,49 +1,142 @@
 # AtmoSeer
 
-## About
+AtmoSeer is a set of Python scripts for downloading, processing, and analyzing atmospheric and meteorological data from sources such as NASA GPM and GOES-16. It supports environment setup via Conda and requires authentication for NASA Earthdata access. This project also provides a pipeline to build rainfall forecast models. The pipeline can be configured with different meteorological data sources.
 
-This project provides a pipeline to build rainfall forecast models. The pipeline can be configured with different meteorological data sources.
+---
 
-## Install
+## Installation
 
-In the root directory of this repository, type the following command (you must have conda installed in your system):
 
+### 1. Clone the repository
+
+```bash
+git clone https://github.com/your-org/atmoseer.git
+cd atmoseer
+````
+
+### 2. Set NASA Earthdata Credentials
+
+Before running the setup script, you must set your NASA Earthdata credentials as environment variables:
+
+```bash
+export EARTHDATA_USER="your_username"
+export EARTHDATA_PASS="your_password"
+```
+
+> You can register for a free NASA Earthdata account at [https://urs.earthdata.nasa.gov](https://urs.earthdata.nasa.gov).
+
+### 3. Run the Setup Script
+
+The `setup.sh` script will:
+
+* Remove any existing Conda environment named `atmoseer`
+* Create a new Conda environment using `config/environment.yml`
+* Create necessary data directories
+* Configure `.netrc` and `.urs_cookies` in your home directory for Earthdata access
+
+```bash
 ./setup.sh
+```
+
+---
+
+## Usage
+
+After the environment is set up, activate it:
+
+```bash
+conda activate atmoseer
+```
+
+Now, you can start running AtmoSeer scripts.
+
+## Running Scripts with Make
+
+To simplify execution, AtmoSeer supports running preprocessing tasks using `make`. Each data module has its own Makefile targets and README with details.
+
+### ✅ Examples
+
+- **GOES-16 Feature Extraction**
+
+```bash
+make goes16-features FEATS="--pn --fa --toct --verbose"
+```
+
+See full usage in [`src/goes16/README.md`](src/goes16/README.md)
+
+- **GPM Download and Crop**
+
+```bash
+make gpm-download-crop BEGIN=2024/01/01 END=2024/01/03 USER=your_username PWD=your_password
+```
+
+See full usage in [`src/gpm/README.md`](src/gpm/README.md)
+
+---
+
+## 📁 Directory Structure
+
+```
+atmoseer/
+├── config/                 # Conda environment and config files
+│   └── environment.yml
+├── data/                  # Automatically created data directories
+│   ├── datasets/
+│   ├── era5/
+│   ├── goes16/
+    ├── gpm/
+│   ├── sounding/
+│   ├── surface_stations/
+│   └── 
+├── src/                   # Scripts for downloading and processing data, and for model training
+│   ├── era5/
+│   ├── goes16/
+    ├── gpm/
+│   ├── sounding/
+│   ├── surface_stations/
+    └── ...
+├── setup.sh               # Installation script
+└── README.md              # This file
+```
 
-## Project pipeline
+---
 
-The project pipeline is defined as a sequence of three steps: (1) data retrieving, (2) data pre-processing and (3) model training. These steps are implemented as Python scripts in the `./src` directory.
+## 🛠 Requirements
 
-#### Data retrieval
+* [Miniconda](https://docs.conda.io/en/latest/miniconda.html) or [Anaconda](https://www.anaconda.com/)
+* Linux or WSL (recommended)
+* Internet access for installing packages and accessing NASA endpoints
 
-All datasets retrieved and/or generated by the scripts will be stored in the `./data` folder.
+---
 
-- **_retrieve_ws_cor.py_**: This script retrieves observation from a user-provided weather station.
-- **_retrieve_ws_inmet.py_**: This script retrieves observations for from a user-provided weather station.
-- **_retrieve_as.py_**: this script retrieves atmospheric sounding data. 
-- **_retrieve_ERA5.py_**: this script retrieves numerical simulation data from the ERA5 portal. 
+## Guide to Collaborators
 
+Welcome! Please follow these guidelines to keep the project consistent and secure.
 
-##### Script **_gen_sounding_indices.py_** 
+- Communicate and document **only in English** so every contributor can follow along.
+- Never commit secrets, credentials, or private datasets. Store sensitive values in `.env` files that stay out of version control.
+- Use GitFlow conventions: branch from `develop` (or `main` if specified), follow feature/hotfix/release prefixes, and open pull requests before merging.
+- Prefer small, focused commits with clear messages; reference issues or tasks when possible.
+- Keep coding style consistent with the existing modules; run linting/formatting tools provided by the project before submitting code (see the tooling notes below).
+- Add or update tests whenever you change behaviour, and ensure the full test suite passes locally.
+- Discuss significant architectural or API changes with maintainers before implementation.
+- Respect code reviews: respond to feedback promptly and revise your changes as needed.
 
-This script will generate atmospheric instability indices for the data retrieveed by the script **_retrieve_as.py_**. Data from the SBGL sounding (located at the Galeão Airport, Rio de Janeiro - Brazil) will be used to calculate atmospheric instability indices, generating a new dataset. This new dataset contains one entry per sounding probe. SBGL sounding station produces two probes per day (at 00:00h and 12:00h UTC). Each entry in the produced contains the values of the computed instability indices for one probe. The following instability indices are computed:
+### Code Style Tooling
 
-- CAPE
-- CIN
-- Lift
-- k
-- Total totals
-- Show alter
+1. Install/update the Conda environment so that `black`, `ruff`, and `pre-commit` are available.
+2. Run `make format` for automatic formatting (`ruff format` + `black`) and `make lint` to check for style and static-analysis issues.
+3. Optional but recommended: `make pre-commit-install` to set up git hooks that run Ruff and Black before each commit.
 
-#### Preprocessing
+---
 
-The preprocessing scripts are responsible for performing several operations on the original dataset, such as creating variables or aggregating data, which can be interesting for model training and its final result. 
+## 🧾 License
 
-#### Dataset building
+This project is licensed under the MIT License. See `LICENSE` for details.
 
-These scripts will build the train, validation and test dataset from the times series produced in the previous steps. These are the datasets to be given as input to the model training step.
+---
 
-#### Model training and evaluation
+## 👥 Acknowledgments
 
-The model generation script is responsible for performing the training and exporting the results obtained by the model after testing. 
-# r2t
+* NASA GES DISC — [https://disc.gsfc.nasa.gov](https://disc.gsfc.nasa.gov)
+* Earthdata Login Services — [https://urs.earthdata.nasa.gov](https://urs.earthdata.nasa.gov)