feat: include a python API to dados.gov.br API (#241)

* feat: include a python API to dados.gov.br API

* refactor: refactor pysus structure

BREAKING CHANGE: version 2.0.0

* remove circular imports

* move individual database files to databases.py

* normalize FileDescription to prepare for ducklake implementation

* start building the dadosgov models to extract & describe the files within the datasets

* BREAKING CHANGE: refactor local data files; implement groups for pysus dep; async everything

* refactor(ducklake): implement abstract classes on the ducklake api classes

* refactor(ducklake): implement abstract classes on the ducklake api classes

* refactor(ftp): implement base models to ftp api

* finish implementing it on all ftp databases

* include some tests

* update tqdm

* implement interfaces on dadosgov modules

* create a PySUS orchestrator that will be a bridge between remote clients and local files

* clean package/prepare to v2 release; start refining tui

* restric dependency import error only to DBC

* start testing the programatically upload of the files on s3

* fix all the mypy linting errors

* include more tests now that catalog.db is filled

* fix tests

Author: luabida

.github/workflows/python-package.yml

@@ -31,14 +31,22 @@ jobs:
         auto-update-conda: true
         conda-solver: libmamba
 
-    - name: Run jupyterlab with PySUS
-      run: |
-        make run-jupyter-pysus
+    # - name: Run jupyterlab with PySUS
+    #   run: |
+    #     make run-jupyter-pysus
         # make test-jupyter-pysus  ## takes too long
 
     - name: Linting & Tests
       run: |
-        export CI=1
-        poetry install
+        pip install poetry poetry-plugin-export
+
+        poetry config virtualenvs.create false
+
+        poetry export --with dev --extras dbc --format requirements.txt --output reqs.txt --without-hashes
+
+        pip install -r reqs.txt
+        pip install -e ".[dbc]"
+
         pre-commit run --all-files
+
         make test-pysus

.gitignore

@@ -179,6 +179,7 @@ dmypy.json
 
 # pytype static type analyzer
 .pytype/
+.pylintrc
 
 # Cython debug symbols
 cython_debug/
@@ -189,3 +190,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
+pyrightconfig.json

.pre-commit-config.yaml

@@ -2,43 +2,57 @@ default_stages: [commit, push]
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.5.0
     hooks:
+      - id: trailing-whitespace
       - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-ast
 
-  - repo: local
+  - repo: https://github.com/psf/black
+    rev: 24.2.0
     hooks:
-    - entry: black
-      id: black
-      name: black
-      exclude: |
-        (?x)(
-          docs
-        )
-      files: ""
-      language: system
-      pass_filenames: true
-      types:
-        - python
-        - file
-        - python
+      - id: black
+        args: [--line-length=80]
+        exclude: ^docs/
 
-    - entry: flake8
-      exclude: ^$
-      files: ""
-      id: flake8
-      language: python
-      name: flake8
-      pass_filenames: true
-      types:
-        - python
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        args: [--profile=black, --line-length=80]
+        exclude: ^.*/js/.*$
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+        args: [--max-line-length=80, --extend-ignore=E203]
+        additional_dependencies: [
+            'flake8-blind-except',
+            'flake8-bugbear',
+            'flake8-comprehensions',
+            'flake8-implicit-str-concat',
+            'pydocstyle>=6.3.0',
+        ]
 
-    - entry: isort
-      exclude: "^.*/js/.*$"
-      files: ""
-      id: isort
-      language: python
-      name: isort
-      pass_filenames: true
-      types:
-        - python
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.8.0
+    hooks:
+      - id: mypy
+        additional_dependencies: [
+            'types-python-dateutil',
+            'types-requests',
+            'types-setuptools',
+            'pandas-stubs',
+            'pydantic>=2.0.0',
+        ]
+        args: [--ignore-missing-imports, --explicit-package-bases]
+        exclude: ^docs/
+
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+      - id: pyupgrade
+        args: [--py310-plus]

README.md

@@ -54,7 +54,7 @@ First, clone the Pysus repository:
 
 ```bash
 git clone https://github.com/AlertaDengue/PySUS.git
-``` 
+```
 
 then from within the PySUS directory build the container
 

conda/dev.yaml

@@ -7,5 +7,4 @@ dependencies:
   - python>=3.10,<3.14
   - jupyter
   - make
-  - poetry
   - pip

condarecipe/pysus/meta.yaml

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # PySUS documentation build configuration file, created by
 # sphinx-quickstart on Thu Aug 25 10:37:19 2016.