pyproject.toml checks (ecmwf#1042)

* adding chceks for toml files into actions

* lint fixes

* lint checks

* link fixes

* changes

* disabling ruff check

* change to path info instead

* adding E501 and E721 to be ignored for now

---------

Co-authored-by: Tim Hunter <tim.hunter@ecmwf.int>

Author: enssow

.github/workflows/ci.yml

@@ -28,6 +28,9 @@ jobs:
         # Run temporarily on a sub directory before the main restyling.
         run: ./scripts/actions.sh lint-check
 
+      - name: TOML checks
+        run: ./scripts/actions.sh toml-check
+
       - name: Type checker (pyrefly, experimental)
         # Do not attempt to install the default dependencies, this is much faster.
         # Run temporarily on a sub directory before the main restyling.

packages/common/pyproject.toml

@@ -3,7 +3,7 @@ name = "weathergen-common"
 version = "0.1.0"
 description = "The WeatherGenerator Machine Learning Earth System Model"
 readme = "../../README.md"
-requires-python = ">=3.11,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     "xarray>=2025.6.1",
     "dask>=2024.9.1",
@@ -42,12 +42,6 @@ not-callable = false
 
 
 
-[tool.black]
-
-# Wide rows
-line-length = 100
-
-
 # The linting configuration
 [tool.ruff]
 
@@ -70,7 +64,11 @@ select = [
     # isort
     "I",
     # Banned imports
-    "TID"
+    "TID",
+    # Naming conventions
+    "N",
+    # print
+    "T201"
 ]
 
 # These rules are sensible and should be enabled at a later stage.
@@ -82,11 +80,20 @@ ignore = [
   "SIM118",
   "SIM102",
   "SIM401",
-  "UP040", # TODO: enable later
   # To ignore, not relevant for us
-  "SIM108" # in case additional norm layer supports are added in future  
+  "SIM108", # in case additional norm layer supports are added in future
+  "N817", # we use heavy acronyms, e.g., allowing 'import LongModuleName as LMN' (LMN is accepted)
+  "E731", # overly restrictive and less readable code
+  "N812", # prevents us following the convention for importing torch.nn.functional as F
 ]
 
+[tool.ruff.lint.flake8-tidy-imports.banned-api]
+"numpy.ndarray".msg = "Do not use 'ndarray' to describe a numpy array type, it is a function. Use numpy.typing.NDArray or numpy.typing.NDArray[np.float32] for example"
+
+[tool.ruff.format]
+# Use Unix `\n` line endings for all files
+line-ending = "lf"
+
 
 
 [build-system]

packages/common/src/weathergen/common/io.py

@@ -23,16 +23,13 @@
 
 # experimental value, should be inferred more intelligently
 CHUNK_N_SAMPLES = 16392
-DType: typing.TypeAlias = np.float32
+type DType = np.float32
 type NPDT64 = datetime64
 
 
 _logger = logging.getLogger(__name__)
 
 
-np.ndarray(3)
-
-
 @dataclasses.dataclass
 class IOReaderData:
     """

packages/evaluate/pyproject.toml

@@ -3,7 +3,7 @@ name = "weathergen-evaluate"
 version = "0.1.0"
 description = "The WeatherGenerator Machine Learning Earth System Model"
 readme = "../../README.md"
-requires-python = ">=3.11,<3.13"
+requires-python = ">=3.12,<3.13"
 dependencies = [
     "cartopy>=0.24.1",
     "xskillscore",
@@ -25,6 +25,12 @@ dev = [
 evaluation = "weathergen.evaluate.run_evaluation:evaluate"
 export = "weathergen.evaluate.export_inference:export"
 
+# The linting configuration
+[tool.ruff]
+
+# Wide rows
+line-length = 100
+
 [tool.ruff.lint]
 # All disabled until the code is formatted.
 select = [
@@ -40,24 +46,40 @@ select = [
     "SIM",
     # isort
     "I",
+    # Banned imports
+    "TID",
+    # Naming conventions
+    "N",
+    # print
+    "T201"
 ]
 
 # These rules are sensible and should be enabled at a later stage.
 ignore = [
-  "E501",
-  "E721",
-  "E722",
   # "B006",
   "B011",
   "UP008",
   "SIM117",
   "SIM118",
   "SIM102",
   "SIM401",
+  "E501", # to be removed 
+  "E721",
   # To ignore, not relevant for us
-  "E741",
+  "SIM108", # in case additional norm layer supports are added in future
+  "N817", # we use heavy acronyms, e.g., allowing 'import LongModuleName as LMN' (LMN is accepted)
+  "E731", # overly restrictive and less readable code
+  "N812", # prevents us following the convention for importing torch.nn.functional as F
 ]
 
+[tool.ruff.lint.flake8-tidy-imports.banned-api]
+"numpy.ndarray".msg = "Do not use 'ndarray' to describe a numpy array type, it is a function. Use numpy.typing.NDArray or numpy.typing.NDArray[np.float32] for example"
+
+[tool.ruff.format]
+# Use Unix `\n` line endings for all files
+line-ending = "lf"
+
+
 [tool.pyrefly]
 project-includes = ["src/"]
 project-excludes = [

packages/evaluate/src/weathergen/evaluate/derived_channels.py

@@ -105,9 +105,7 @@ def get_channel(self, data_tars, data_preds, tag, level, calc_func) -> None:
 
                 data_updated.append(conc)
 
-                self.channels = self.channels + (
-                    [tag] if tag not in self.channels else []
-                )
+                self.channels = self.channels + ([tag] if tag not in self.channels else [])
 
             else:
                 data_updated.append(data)

packages/evaluate/src/weathergen/evaluate/io_reader.py

@@ -67,9 +67,7 @@ class DataAvailability:
 
 
 class Reader:
-    def __init__(
-        self, eval_cfg: dict, run_id: str, private_paths: dict[str, str] | None = None
-    ):
+    def __init__(self, eval_cfg: dict, run_id: str, private_paths: dict[str, str] | None = None):
         """
         Generic data reader class.
 
@@ -254,19 +252,15 @@ def _get_channels_fsteps_samples(self, stream: str, mode: str) -> DataAvailabili
         )
 
         stream_cfg = self.get_stream(stream)
-        assert stream_cfg.get(mode, False), (
-            "Mode does not exist in stream config. Please add it."
-        )
+        assert stream_cfg.get(mode, False), "Mode does not exist in stream config. Please add it."
 
         samples = stream_cfg[mode].get("sample", None)
         fsteps = stream_cfg[mode].get("forecast_step", None)
         channels = stream_cfg.get("channels", None)
 
         return DataAvailability(
             score_availability=True,
-            channels=None
-            if (channels == "all" or channels is None)
-            else list(channels),
+            channels=None if (channels == "all" or channels is None) else list(channels),
             fsteps=None if (fsteps == "all" or fsteps is None) else list(fsteps),
             samples=None if (samples == "all" or samples is None) else list(samples),
         )
@@ -286,9 +280,7 @@ def __init__(self, eval_cfg: dict, run_id: str, private_paths: dict | None = Non
 
         if not self.results_base_dir:
             self.results_base_dir = Path(get_shared_wg_path("results"))
-            _logger.info(
-                f"Results directory obtained from private config: {self.results_base_dir}"
-            )
+            _logger.info(f"Results directory obtained from private config: {self.results_base_dir}")
         else:
             _logger.info(f"Results directory parsed: {self.results_base_dir}")
 
@@ -306,19 +298,15 @@ def __init__(self, eval_cfg: dict, run_id: str, private_paths: dict | None = Non
         )
         # for backward compatibility allow metric_dir to be specified in the run config
         self.metrics_dir = Path(
-            self.eval_cfg.get(
-                "metrics_dir", self.metrics_base_dir / self.run_id / "evaluation"
-            )
+            self.eval_cfg.get("metrics_dir", self.metrics_base_dir / self.run_id / "evaluation")
         )
 
         self.fname_zarr = self.results_dir.joinpath(
             f"validation_epoch{self.epoch:05d}_rank{self.rank:04d}.zarr"
         )
 
         if not self.fname_zarr.exists() or not self.fname_zarr.is_dir():
-            _logger.error(
-                f"Zarr file {self.fname_zarr} does not exist or is not a directory."
-            )
+            _logger.error(f"Zarr file {self.fname_zarr} does not exist or is not a directory.")
             raise FileNotFoundError(
                 f"Zarr file {self.fname_zarr} does not exist or is not a directory."
             )
@@ -401,9 +389,7 @@ def get_data(
             # TODO: Avoid conversion of fsteps and sample to integers (as obtained from the ZarrIO)
             fsteps = sorted([int(fstep) for fstep in fsteps])
             samples = sorted(
-                [int(sample) for sample in self.get_samples()]
-                if samples is None
-                else samples
+                [int(sample) for sample in self.get_samples()] if samples is None else samples
             )
             channels = channels or stream_cfg.get("channels", all_channels)
             channels = to_list(channels)
@@ -429,15 +415,11 @@ def get_data(
             fsteps_final = []
 
             for fstep in fsteps:
-                _logger.info(
-                    f"RUN {self.run_id} - {stream}: Processing fstep {fstep}..."
-                )
+                _logger.info(f"RUN {self.run_id} - {stream}: Processing fstep {fstep}...")
                 da_tars_fs, da_preds_fs = [], []
                 pps = []
 
-                for sample in tqdm(
-                    samples, desc=f"Processing {self.run_id} - {stream} - {fstep}"
-                ):
+                for sample in tqdm(samples, desc=f"Processing {self.run_id} - {stream} - {fstep}"):
                     out = zio.get_data(sample, stream, fstep)
                     target, pred = out.target.as_xarray(), out.prediction.as_xarray()
 
@@ -474,17 +456,13 @@ def get_data(
                         da_tars_fs = da_tars_fs.assign_coords(
                             sample=(
                                 "ipoint",
-                                np.repeat(
-                                    da_tars_fs.sample.values, len(da_tars_fs.ipoint)
-                                ),
+                                np.repeat(da_tars_fs.sample.values, len(da_tars_fs.ipoint)),
                             )
                         )
                         da_preds_fs = da_preds_fs.assign_coords(
                             sample=(
                                 "ipoint",
-                                np.repeat(
-                                    da_preds_fs.sample.values, len(da_preds_fs.ipoint)
-                                ),
+                                np.repeat(da_preds_fs.sample.values, len(da_preds_fs.ipoint)),
                             )
                         )
 
@@ -506,12 +484,8 @@ def get_data(
                         points_per_sample.loc[{"forecast_step": fstep}] = np.array(pps)
 
             # Safer than a list
-            da_tars = {
-                fstep: da for fstep, da in zip(fsteps_final, da_tars, strict=True)
-            }
-            da_preds = {
-                fstep: da for fstep, da in zip(fsteps_final, da_preds, strict=True)
-            }
+            da_tars = {fstep: da for fstep, da in zip(fsteps_final, da_tars, strict=True)}
+            da_preds = {fstep: da for fstep, da in zip(fsteps_final, da_preds, strict=True)}
 
             return ReaderOutput(
                 target=da_tars, prediction=da_preds, points_per_sample=points_per_sample
@@ -522,7 +496,7 @@ def get_data(
     def get_stream(self, stream: str):
         """
         returns the dictionary associated to a particular stream.
-        Returns an empty dictionary if the stream does not exist in the Zarr file.  
+        Returns an empty dictionary if the stream does not exist in the Zarr file.
 
         Parameters
         ----------

packages/evaluate/src/weathergen/evaluate/plot_utils.py

@@ -103,9 +103,7 @@ def plot_metric_region(
                 run_ids.append(run_id)
 
             if selected_data:
-                _logger.info(
-                    f"Creating plot for {metric} - {region} - {stream} - {ch}."
-                )
+                _logger.info(f"Creating plot for {metric} - {region} - {stream} - {ch}.")
                 name = "_".join([metric, region] + sorted(set(run_ids)) + [stream, ch])
                 plotter.plot(
                     selected_data,
@@ -157,9 +155,7 @@ def score_card_metric_region(
             if channels_common is None:
                 channels_common = set(channels_per_run)
             else:
-                channels_common = set(channels_common).intersection(
-                    set(channels_per_run)
-                )
+                channels_common = set(channels_common).intersection(set(channels_per_run))
 
         if not channels_common:
             continue
@@ -207,9 +203,7 @@ def get_marker_size(cls, stream_name: str) -> float:
         float
             The default marker size for the stream.
         """
-        return cls._marker_size_stream.get(
-            stream_name.lower(), cls._default_marker_size
-        )
+        return cls._marker_size_stream.get(stream_name.lower(), cls._default_marker_size)
 
     @classmethod
     def list_streams(cls):