Merge pull request #71 from CDCgov/copilot/update-ruff-config-and-pre-commit

Sync ruff config between ruff.toml and pre-commit; upgrade to v0.15.8

Author: ryanraaschCDC

.pre-commit-config.yaml

@@ -14,17 +14,12 @@ repos:
 #####
 # Python
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.8
+  rev: v0.15.8
   hooks:
     # Sort imports
-    - id: ruff
-      args: ['check', '--select', 'I', '--fix']
-    # Run the linter
-    - id: ruff
-      args: ['--line-length', '79']
+    - id: ruff-check
     # Run the formatter
     - id: ruff-format
-      args: ['--line-length', '79']
 #####
 # R
 # -   repo: https://github.com/lorenzwalthert/precommit

cfa/dataops/catalog.py

@@ -3,18 +3,18 @@
 import json
 import os
 import pkgutil
+from collections.abc import Sequence
 from configparser import ConfigParser
 from importlib import import_module
 from io import BytesIO
 from pathlib import PurePosixPath
 from types import SimpleNamespace
-from typing import Any, List, Literal, Sequence, overload
+from typing import Any, Literal, overload
 
 import pandas as pd
 import polars as pl
 import tomli
 from azure.identity import ManagedIdentityCredential
-
 from cfa.cloudops.blob_helpers import (
     read_blob_stream,
     walk_blobs_in_container,
@@ -53,9 +53,7 @@ def get_all_catalogs() -> list:
     catalog_nspace = _config.get("DEFAULT", "catalog_namespaces")
     try:
         catalog_pkg = import_module(catalog_nspace)
-        for module_finder, modname, ispkg in pkgutil.iter_modules(
-            catalog_pkg.__path__
-        ):
+        for module_finder, modname, ispkg in pkgutil.iter_modules(catalog_pkg.__path__):
             if ispkg:
                 catalogs.append((catalog_nspace, modname, module_finder.path))
     except ModuleNotFoundError:
@@ -76,9 +74,7 @@ def get_all_catalogs() -> list:
     report_mod = import_module(f"{cns}.{cat_name}.reports")
     all_dataset_ns_map.update(dataset_mod.dataset_ns_map)
     all_reports_ns_map.update(report_mod.report_ns_map)
-    with open(
-        os.path.join(cat_path, cat_name, "catalog_defaults.toml"), "rb"
-    ) as f:
+    with open(os.path.join(cat_path, cat_name, "catalog_defaults.toml"), "rb") as f:
         defaults = tomli.load(f)
     for k in dataset_mod.dataset_ns_map.keys():
         all_defaults.update({k: defaults})
@@ -111,13 +107,9 @@ def __init__(self, config_path: str, defaults: dict, ns: str):
                 account = v.get("account", "")
                 container = v.get("container", "")
                 if account == "":
-                    self.config[k]["account"] = self.defaults["storage"][
-                        "account"
-                    ]
+                    self.config[k]["account"] = self.defaults["storage"]["account"]
                 if container == "":
-                    self.config[k]["container"] = self.defaults["storage"][
-                        "container"
-                    ]
+                    self.config[k]["container"] = self.defaults["storage"]["container"]
         self.validate_dataset_config(config_path)
         self._ledger_location = {
             "account": self.defaults["storage"]["account"],
@@ -144,8 +136,7 @@ def validate_dataset_config(self, config_path) -> None:
             config_models = {}
             for c_key, c_value in self.config.items():
                 if (
-                    c_key.startswith("stage_")
-                    or c_key in ["load", "extract", "data"]
+                    c_key.startswith("stage_") or c_key in ["load", "extract", "data"]
                 ) and c_value is not None:
                     config_models[c_key] = StorageEndpointValidation(**c_value)
                 elif c_key == "properties":
@@ -207,9 +198,7 @@ def write_blob(
             append (bool, optional): whether to append to existing file (only for single file writes).
         """
         if auto_version and not append:
-            path_after_prefix = (
-                f"{get_timestamp()}/{path_after_prefix.lstrip('/')}"
-            )
+            path_after_prefix = f"{get_timestamp()}/{path_after_prefix.lstrip('/')}"
         path_after_prefix = path_after_prefix.lstrip("/")
         full_path = f"{self.prefix}/{path_after_prefix}"
         if isinstance(file_buffer, bytes):
@@ -231,9 +220,7 @@ def write_blob(
         self.ledger_entry(action="write")
         # print(f"file written to: {full_path}")
 
-    def read_blobs(
-        self, version: str = "latest", newest: bool = True
-    ) -> List[bytes]:
+    def read_blobs(self, version: str = "latest", newest: bool = True) -> list[bytes]:
         """Read a blob in as bytes so it can be loaded into a dataframe
 
         Args:
@@ -295,18 +282,16 @@ def get_file_ext(self, version: str = "latest") -> str:
         Returns:
             str: the file extension
         """
-        return self._get_version_blobs(version=version, print_version=False)[
-            0
-        ]["name"].split(".")[-1]
+        return self._get_version_blobs(version=version, print_version=False)[0][
+            "name"
+        ].split(".")[-1]
 
     def _get_version_blobs(
         self, version: str = "latest", newest=True, print_version=True
     ) -> list:
         if not self.is_ledger:
             available_versions = self.get_versions()
-            version = version_matcher(
-                version, available_versions, newest=newest
-            )
+            version = version_matcher(version, available_versions, newest=newest)
             if not version:
                 raise ValueError(
                     f"Version {version} not found in available versions: {available_versions}"
@@ -414,9 +399,7 @@ def get_dataframe(
 
     def get_dataframe(
         self,
-        output: Literal[
-            "pandas", "pd", "polars", "pl", "pl_lazy", "lazy"
-        ] = "pandas",
+        output: Literal["pandas", "pd", "polars", "pl", "pl_lazy", "lazy"] = "pandas",
         version: str = "latest",
         newest: bool = True,
     ) -> pd.DataFrame | pl.DataFrame | pl.LazyFrame:
@@ -489,9 +472,7 @@ def get_dataframe(
                 self.ledger_entry(action="read")
                 return df
             else:
-                raise ValueError(
-                    f"Lazy loading not supported for {file_ext} files."
-                )
+                raise ValueError(f"Lazy loading not supported for {file_ext} files.")
         blobs = self.read_blobs(version, newest=newest)
         blob_bytes = [blob.content_as_bytes() for blob in blobs]
         blob_files = [BytesIO(pq) for pq in blob_bytes]
@@ -523,9 +504,7 @@ def get_dataframe(
             return df
         elif file_ext == "jsonl" or file_ext == "ndjson":
             if output in ["pandas", "pd"]:
-                df = pd.concat(
-                    [pd.read_json(blob, lines=True) for blob in blob_files]
-                )
+                df = pd.concat([pd.read_json(blob, lines=True) for blob in blob_files])
                 df.reset_index(inplace=True, drop=True)
             else:
                 df = pl.concat(
@@ -535,9 +514,7 @@ def get_dataframe(
             return df
         elif file_ext == "parquet" or file_ext == "parq":
             if output in ["pandas", "pd"]:
-                df = pd.concat(
-                    [pd.read_parquet(pq_file) for pq_file in blob_files]
-                )
+                df = pd.concat([pd.read_parquet(pq_file) for pq_file in blob_files])
                 df.reset_index(inplace=True, drop=True)
             else:
                 df = pl.concat(
@@ -593,9 +570,7 @@ def save_dataframe(
             raise ValueError(
                 f"File format {file_format} not supported. Use 'parquet', 'csv', 'json', or 'jsonl'."
             )
-        if file_format in ["json", "jsonl"] and path_after_prefix.endswith(
-            ".json"
-        ):
+        if file_format in ["json", "jsonl"] and path_after_prefix.endswith(".json"):
             path_after_prefix = path_after_prefix[:-5] + ".jsonl"
             print("Changing file extension to .jsonl for line-delimited JSON.")
         if isinstance(df, pd.DataFrame):
@@ -618,9 +593,7 @@ def save_dataframe(
                     auto_version=auto_version,
                 )
             elif file_format in ["json", "jsonl"]:
-                json_bytes = df.to_json(orient="records", lines=True).encode(
-                    "utf-8"
-                )
+                json_bytes = df.to_json(orient="records", lines=True).encode("utf-8")
                 self.write_blob(
                     file_buffer=json_bytes,
                     path_after_prefix=path_after_prefix
@@ -762,19 +735,15 @@ def dict_to_sn(d: Any, defaults: dict = None, ns: str = "") -> SimpleNamespace:
 rc = []
 for k in all_reports_ns_map.keys():
     rc.append(report_dict_to_sn({k: all_reports_ns_map[k]}))
-combined_reports_dict = {
-    key: value for ns in rc for key, value in vars(ns).items()
-}
+combined_reports_dict = {key: value for ns in rc for key, value in vars(ns).items()}
 
 datacat = SimpleNamespace(**combined_dict)
 datacat.__setattr__("__namespace_list__", dataset_namespaces)
 reportcat = SimpleNamespace(**combined_reports_dict)
 reportcat.__setattr__("__namespace_list__", report_namespaces)
 
 
-def _attach_schema_mock_functions(
-    datacat: SimpleNamespace, catalogs: list
-) -> None:
+def _attach_schema_mock_functions(datacat: SimpleNamespace, catalogs: list) -> None:
     """Recursively walk the datacat namespace and attach mock_data functions to
     the extract and load BlobEndpoints of each DatasetEndpoint, sourced from
     a schema module co-located with the dataset.
@@ -812,14 +781,11 @@ def _walk(ns: SimpleNamespace) -> None:
                     # strip cat_name prefix -> "stf.nhsn_hrd_prelim"
                     # then split into team ("stf") and dataset ("nhsn_hrd_prelim")
                     # so the schema lives at: datasets.stf.schemas.nhsn_hrd_prelim
-                    ns_within_datasets = val.__ns_str__.removeprefix(
-                        f"{cat_name}."
-                    )
+                    ns_within_datasets = val.__ns_str__.removeprefix(f"{cat_name}.")
                     ns_parts = ns_within_datasets.rsplit(".", 1)
                     team_path = ns_parts[0] if len(ns_parts) > 1 else ""
                     schema_mod_path = (
-                        f"{cns}.{cat_name}.datasets"
-                        f".{team_path}.schemas.{dataset_name}"
+                        f"{cns}.{cat_name}.datasets.{team_path}.schemas.{dataset_name}"
                         if team_path
                         else f"{cns}.{cat_name}.datasets.schemas.{dataset_name}"
                     )

cfa/dataops/command.py

@@ -58,9 +58,7 @@ def get_available_data():
     Retrieve a list of available datasets for CFA.
     """
     parser = ArgumentParser(description="Get list of available datasets")
-    parser.add_argument(
-        "-p", "--prefix", help="optional prefix filter", default=None
-    )
+    parser.add_argument("-p", "--prefix", help="optional prefix filter", default=None)
     args = parser.parse_args()
     datasets = datacat.__namespace_list__
     if args.prefix:
@@ -125,9 +123,7 @@ def save_data_locally():
     parser.add_argument(
         "--stage", "-s", help="specific stage to get version for", default=None
     )
-    parser.add_argument(
-        "--version", "-v", help="specific version to get", default=None
-    )
+    parser.add_argument("--version", "-v", help="specific version to get", default=None)
     parser.add_argument(
         "--force", "-f", help="force re-download of data", action="store_true"
     )

cfa/dataops/config_validator.py

@@ -1,7 +1,6 @@
 """The dataset config validator."""
 
 from enum import Enum
-from typing import List, Optional
 
 from pydantic import (
     BaseModel,
@@ -34,15 +33,15 @@ class PropertiesValidation(BaseModel):
         ...,
         description="The type of the dataset.",
     )
-    schemas: Optional[str] = Field(
+    schemas: str | None = Field(
         None,
         description="the relative path to where the data schema(s) are defined.",
     )
-    transform_templates: Optional[List[str]] = Field(
+    transform_templates: list[str] | None = Field(
         None,
         description="A list of transform templates to applied to raw data.",
     )
-    automate: Optional[bool] = Field(
+    automate: bool | None = Field(
         False,
         description="Whether to automate workflows that cache raw data and transform it.",
     )
@@ -80,9 +79,9 @@ class ConfigValidator(BaseModel):
 
     model_config = ConfigDict(extra="allow")
     properties: PropertiesValidation
-    load: Optional[StorageEndpointValidation] = None
-    source: Optional[SourceValidation] = None
-    extract: Optional[StorageEndpointValidation] = None
+    load: StorageEndpointValidation | None = None
+    source: SourceValidation | None = None
+    extract: StorageEndpointValidation | None = None
 
     # Any stage_xx field will be validated dynamically; no need for explicit field or nesting.
     @classmethod

cfa/dataops/create_catalog/command.py

@@ -19,9 +19,7 @@
 _here_dir = os.path.split(os.path.abspath(__file__))[0]
 template_dirs = [os.path.join(_here_dir, "repo_templates")]
 repo_files_dirs = os.path.join(_here_dir, "repo_files")
-repo_template_lookup = TemplateLookup(
-    directories=template_dirs, cache_enabled=False
-)
+repo_template_lookup = TemplateLookup(directories=template_dirs, cache_enabled=False)
 
 
 def main():
@@ -84,9 +82,7 @@ def main():
         with open(
             os.path.join(library_modules_root, "catalog_defaults.toml"), "w"
         ) as f:
-            template = repo_template_lookup.get_template(
-                "catalog_defaults.toml.mako"
-            )
+            template = repo_template_lookup.get_template("catalog_defaults.toml.mako")
             f.write(
                 template.render(
                     unique_name=unique_name,
@@ -129,9 +125,7 @@ def main():
         sys.exit(0)
 
     if args.expanded_repo_for_cfa:
-        print(
-            "Including CFA expanded repository pre-commit and workflow files."
-        )
+        print("Including CFA expanded repository pre-commit and workflow files.")
         for copy_i in [
             ".pre-commit-config.yaml",
             ".secrets.baseline",

cfa/dataops/create_catalog/repo_files/_cfa_only/.pre-commit-config.yaml

@@ -13,17 +13,12 @@ repos:
 #####
 # Python
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.8
+  rev: v0.15.8
   hooks:
     # Sort imports
-    - id: ruff
-      args: ['check', '--select', 'I', '--fix']
-    # Run the linter
-    - id: ruff
-      args: ['--line-length', '79']
+    - id: ruff-check
     # Run the formatter
     - id: ruff-format
-      args: ['--line-length', '79']
 # Secrets
 - repo: https://github.com/Yelp/detect-secrets
   rev: v1.4.0

cfa/dataops/create_catalog/repo_files/_cfa_only/ruff.toml

@@ -1,3 +1,3 @@
 extend-include = ["*.ipynb"]
-line-length = 79
-ignore = []
+fix = true
+lint.select = ["I", "E4", "E7", "E9", "F", "UP"]

cfa/dataops/create_catalog/repo_files/datasets/__init__.py

@@ -2,8 +2,8 @@
 
 import glob
 import os
+from collections.abc import Callable
 from os import path
-from typing import Optional
 
 import tomli
 
@@ -44,7 +44,7 @@ def get_dataset_name(p: str) -> str:
 
 
 def get_fs_ns_map(
-    base_dir: str, file_ext: str, endpoint_func: Optional[callable] = None
+    base_dir: str, file_ext: str, endpoint_func: Callable[[str], str] | None = None
 ) -> dict:
     """Get a nested dictionary representing the filesystem structure starting from base_dir.
 
@@ -60,9 +60,7 @@ def get_fs_ns_map(
     """
     base_dir = os.path.abspath(base_dir)
     file_ext = file_ext.lstrip(".")
-    fs_paths = glob.glob(
-        os.path.join(base_dir, "**", f"*.{file_ext}"), recursive=True
-    )
+    fs_paths = glob.glob(os.path.join(base_dir, "**", f"*.{file_ext}"), recursive=True)
     fs_map = {}
     for p_i in fs_paths:
         if p_i.startswith(base_dir):