Upgrade ruff rules and more (#183)

Add ruff pre-commit hook.
Add pyupgrade pre-commit hook.
Add blacken-docs pre-commit hook.

Update ruff and mypy rules according to scienfic-python.org.
Specifically according to the rules here:
https://learn.scientific-python.org/development/guides/style/

Make end2end_test.py executable.

Author: CasperWA

.github/utils/end2end_test.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 """End-to-end testing in CI for OTEAPI OPTIMADE (using OTELib)."""
+from __future__ import annotations
+
 import importlib
 import json
 import os
@@ -28,13 +30,15 @@ def _check_service_availability(service_url: str) -> None:
     """
     import requests
 
+    error_message = f"Cannot connect to {service_url} !"
+
     try:
         response = requests.get(f"{service_url}/docs", allow_redirects=True, timeout=30)
     except (requests.ConnectionError, requests.ConnectTimeout) as exc_:
-        raise RuntimeError(f"Cannot connect to {service_url} !") from exc_
+        raise RuntimeError(error_message) from exc_
 
     if not response.ok:
-        raise RuntimeError(f"Cannot connect to {service_url} !")
+        raise RuntimeError(error_message)
 
 
 def main(oteapi_url: str) -> None:
@@ -69,12 +73,12 @@ def main(oteapi_url: str) -> None:
 
     session = source.get()
 
+    error_message = "Could not parse returned session as an OPTIMADEResourceStrategy."
+
     try:
         session = OPTIMADEResourceSession(**json.loads(session))
     except ValidationError as exc_:
-        raise RuntimeError(
-            "Could not parse returned session as an OPTIMADEResourceStrategy."
-        ) from exc_
+        raise RuntimeError(error_message) from exc_
 
     assert session.optimade_resource_model == f"{Structure.__module__}:Structure"
     assert len(session.optimade_resources) == 2
@@ -98,9 +102,7 @@ def main(oteapi_url: str) -> None:
         # Should be an OPTIMADEResourceSession because `source` is last in the pipeline
         session = OPTIMADEResourceSession(**json.loads(session))
     except ValidationError as exc_:
-        raise RuntimeError(
-            "Could not parse returned session as an OPTIMADEResourceStrategy."
-        ) from exc_
+        raise RuntimeError(error_message) from exc_
 
     assert session.optimade_resource_model == f"{Structure.__module__}:Structure"
     assert len(session.optimade_resources) == 4
@@ -130,10 +132,10 @@ def main(oteapi_url: str) -> None:
     # Configuration
     PORT = os.getenv("OTEAPI_PORT", "8080")
     OTEAPI_SERVICE_URL = f"http://localhost:{PORT}"
-    OTEAPI_PREFIX = os.getenv("OTEAPI_prefix", "/api/v1")
+    OTEAPI_PREFIX = os.getenv("OTEAPI_prefix", "/api/v1")  # noqa: SIM112
     if "OTEAPI_prefix" not in os.environ:
         # Set environment variables
-        os.environ["OTEAPI_prefix"] = OTEAPI_PREFIX
+        os.environ["OTEAPI_prefix"] = OTEAPI_PREFIX  # noqa: SIM112
 
     try:
         _check_service_availability(service_url=OTEAPI_SERVICE_URL)

.github/workflows/ci_tests.yml

@@ -33,14 +33,6 @@ jobs:
       package_dirs: oteapi_optimade
       full_docs_dirs: "models"
 
-  ruff:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - uses: chartboost/ruff-action@v1
-
   pytest:
     name: pytest (${{ matrix.os[1] }}-py${{ matrix.python-version }})
     runs-on: ${{ matrix.os[0] }}
@@ -74,11 +66,14 @@ jobs:
       run: pytest -vvv --cov-report=xml
 
     - name: Upload coverage to Codecov
-      if: matrix.python-version == '3.9' && github.repository == 'SINTEF/oteapi-optimade'
+      if: github.repository == 'SINTEF/oteapi-optimade'
       uses: codecov/codecov-action@v3
       with:
-        files: coverage.xml
-        flags: ${{ matrix.os[1] }}
+        env_vars: OS,PYTHON
+        flags: local
+      env:
+        OS: ${{ matrix.os[0] }}
+        PYTHON: ${{ matrix.python-version }}
 
   pytest-real-backend:
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -1,31 +1,45 @@
-# To install the git pre-commit hook run:
-#   pre-commit install
-# To update the pre-commit hooks run:
-#   pre-commit autoupdate
+default_language_version:
+  python: python3.9
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:
-    - id: end-of-file-fixer
-    - id: debug-statements
-    - id: check-yaml
-      name: Check YAML
     - id: check-toml
       name: Check TOML
+    - id: check-yaml
+      name: Check YAML
+    - id: debug-statements
+    - id: end-of-file-fixer
+    - id: mixed-line-ending
+    - id: name-tests-test
+      args: ["--pytest-test-first"]
     - id: trailing-whitespace
       args: [--markdown-linebreak-ext=md]
 
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
     hooks:
-    - id: ruff
-      args: ["--fix", "--exit-non-zero-on-fix"]
+    - id: pyupgrade
+      args: [--py39-plus, --keep-runtime-typing]
 
   - repo: https://github.com/ambv/black
     rev: 23.11.0
     hooks:
     - id: black
 
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: 1.16.0
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.5
+    hooks:
+    - id: ruff
+      args: ["--fix", "--exit-non-zero-on-fix", "--show-fixes"]
+
   - repo: https://github.com/PyCQA/bandit
     rev: 1.7.5
     hooks:

oteapi_optimade/__init__.py

@@ -5,6 +5,7 @@
 Authored by Casper Welzel Andersen, SINTEF, 2022
 Created from cookiecutter-oteapi-plugin, SINTEF, 2022
 """
+from __future__ import annotations
 
 __version__ = "0.4.2"
 __author__ = "Casper Welzel Andersen"

oteapi_optimade/dlite/parse.py

@@ -1,4 +1,6 @@
 """OTEAPI strategy for parsing OPTIMADE structure resources to DLite instances."""
+from __future__ import annotations
+
 import logging
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -17,7 +19,7 @@
 from oteapi_optimade.strategies.parse import OPTIMADEParseStrategy
 
 if TYPE_CHECKING:  # pragma: no cover
-    from typing import Any, Dict, Optional, Union
+    from typing import Any
 
 
 LOGGER = logging.getLogger("oteapi_optimade.dlite")
@@ -40,9 +42,7 @@ class OPTIMADEDLiteParseStrategy:
 
     parse_config: OPTIMADEDLiteParseConfig
 
-    def initialize(
-        self, session: "Optional[Dict[str, Any]]" = None
-    ) -> DLiteSessionUpdate:
+    def initialize(self, session: dict[str, Any] | None = None) -> DLiteSessionUpdate:
         """Initialize strategy.
 
         This method will be called through the `/initialize` endpoint of the OTE-API
@@ -59,7 +59,7 @@ def initialize(
         return DLiteSessionUpdate(collection_id=get_collection(session).uuid)
 
     def get(
-        self, session: "Optional[Union[SessionUpdate, Dict[str, Any]]]" = None
+        self, session: SessionUpdate | dict[str, Any] | None = None
     ) -> OPTIMADEParseSession:
         """Request and parse an OPTIMADE response using OPT.
 
@@ -110,14 +110,19 @@ def get(
             f"yaml://{entities_path}/OPTIMADEStructureSpecies.yaml"
         )
 
+        error_message_supporting_only_structures = (
+            "The DLite OPTIMADE Parser currently only supports structures entities."
+        )
+
         if self.parse_config.configuration.return_object:
             # The response is given as a "proper" pydantic data model instance
 
             if "optimade_response_object" not in session:
-                raise ValueError(
+                error_message = (
                     "'optimade_response_object' was expected to be present in the "
                     "session."
                 )
+                raise ValueError(error_message)
 
             # Currently, only "structures" entries are supported and handled
             if isinstance(session.optimade_response_object, StructureResponseMany):
@@ -152,26 +157,23 @@ def get(
                         "Could not determine what to do with `data`. Type %s.",
                         type(session.optimade_response_object.data),
                     )
-                    raise OPTIMADEParseError(
-                        "Could not parse `data` entry in response."
-                    )
+                    error_message = "Could not parse `data` entry in response."
+                    raise OPTIMADEParseError(error_message)
             else:
                 LOGGER.debug(
                     "Got currently unsupported response type %s. Only structures are "
                     "supported.",
                     session.optimade_response_object.__class__.__name__,
                 )
-                raise OPTIMADEParseError(
-                    "The DLite OPTIMADE Parser currently only supports structures "
-                    "entities."
-                )
+                raise OPTIMADEParseError(error_message_supporting_only_structures)
         else:
             # The response is given as pure Python dictionary
 
             if "optimade_response" not in session:
-                raise ValueError(
+                error_message = (
                     "'optimade_response' was expected to be present in the session."
                 )
+                raise ValueError(error_message)
 
             if not session.optimade_response or "data" not in session.optimade_response:
                 LOGGER.debug("Not a successful response - no 'data' entry found.")
@@ -187,30 +189,25 @@ def get(
                         "Could not parse list of 'data' entries as structures."
                     )
                     raise OPTIMADEParseError(
-                        "The DLite OPTIMADE Parser currently only supports structures "
-                        "entities."
+                        error_message_supporting_only_structures
                     ) from exc
             elif session.optimade_response is not None:
                 try:
                     structures = [Structure(session.optimade_response["data"])]
                 except ValidationError as exc:
                     LOGGER.debug("Could not parse single 'data' entry as a structure.")
                     raise OPTIMADEParseError(
-                        "The DLite OPTIMADE Parser currently only supports structures "
-                        "entities."
+                        error_message_supporting_only_structures
                     ) from exc
             else:
-                LOGGER.debug("Could not parse 'data' entries as structures.")
-                raise OPTIMADEParseError(
-                    "The DLite OPTIMADE Parser currently only supports structures "
-                    "entities."
-                )
+                LOGGER.debug("Could not parse 'data' entries as structures.")  # type: ignore[unreachable]
+                raise OPTIMADEParseError(error_message_supporting_only_structures)
 
         dlite_collection = get_collection(session)
 
         # DLite-fy OPTIMADE structures
         for structure in structures:
-            new_structure_attributes: dict[str, "Any"] = {}
+            new_structure_attributes: dict[str, Any] = {}
 
             # Most inner layer: assemblies & species
             if structure.attributes.assemblies:
@@ -221,19 +218,21 @@ def get(
 
                 for assembly in structure.attributes.assemblies:
                     # Ensure we're dealing with a normal Python dict
-                    assembly = (
+                    assembly_dict = (
                         assembly.dict(exclude_none=True)
                         if isinstance(assembly, BaseModel)
                         else assembly
                     )
 
                     dimensions = {
-                        "ngroups": len(assembly.get("group_probabilities", []) or []),
-                        "nsites": len(assembly.get("sites_in_groups", []) or []),
+                        "ngroups": len(
+                            assembly_dict.get("group_probabilities", []) or []
+                        ),
+                        "nsites": len(assembly_dict.get("sites_in_groups", []) or []),
                     }
                     new_structure_attributes["assemblies"].append(
                         OPTIMADEStructureAssembly(
-                            dimensions=dimensions, properties=assembly
+                            dimensions=dimensions, properties=assembly_dict
                         )
                     )
 
@@ -245,24 +244,24 @@ def get(
 
                 for species_individual in structure.attributes.species:
                     # Ensure we're dealing with a normal Python dict
-                    species_individual = (
+                    species_individual_dict = (
                         species_individual.dict(exclude_none=True)
                         if isinstance(species_individual, BaseModel)
                         else species_individual
                     )
 
                     dimensions = {
                         "nelements": len(
-                            species_individual.get("chemical_symbols", []) or []
+                            species_individual_dict.get("chemical_symbols", []) or []
                         ),
                         "nattached_elements": len(
-                            species_individual.get("attached", []) or []
+                            species_individual_dict.get("attached", []) or []
                         ),
                     }
                     new_structure_attributes["species"].append(
                         OPTIMADEStructureSpecies(
                             dimensions=dimensions,
-                            properties=species_individual,
+                            properties=species_individual_dict,
                         )
                     )
 

oteapi_optimade/exceptions.py

@@ -1,4 +1,5 @@
 """OTE-API OPTIMADE-specific Python exceptions."""
+from __future__ import annotations
 
 
 class BaseOteapiOptimadeException(Exception):

oteapi_optimade/models/__init__.py

@@ -1,4 +1,6 @@
 """`oteapi_optimade.models` module - pydantic data models."""
+from __future__ import annotations
+
 from .strategies import (
     OPTIMADEDLiteParseConfig,
     OPTIMADEFilterConfig,

oteapi_optimade/models/config.py

@@ -1,4 +1,6 @@
 """General OPTIMADE configuration models."""
+from __future__ import annotations
+
 from typing import Literal, Optional
 
 from oteapi.models import AttrDict, DataCacheConfig
@@ -13,7 +15,7 @@
 """Set the `expireTime` and `tag` to default values for the data cache."""
 
 
-class OPTIMADEConfig(AttrDict):
+class OPTIMADEConfig(AttrDict):  # type: ignore[misc]
     """OPTIMADE configuration."""
 
     version: str = Field(