Package updates & linter/typing fixes

- Removed empty config.cfg hack (it was an ancient workaround for a problem that no longer exists)
- Updated ruff formatter config to format as python3.12 (the version of python used by the project)
- Fixed some outdated readme text
- Upgraded mypy version
- Fixed type hint complaints from mypy
- Updated pytest & pytest plugin versions (fixes several noisy deprecation warnings)

Author: induane

README.md

@@ -5,7 +5,7 @@
 
 *<p style="text-align: center;">DataOps Observability is part of DataKitchen's Open Source Data Observability. DataOps Observability monitors every data journey from data source to customer value, from any team development environment into production, across every tool, team, environment, and customer so that problems are detected, localized, and understood immediately.</p>*
 
-[![DatKitchen Open Source Data Observability](https://datakitchen.io/wp-content/uploads/2024/04/both-products.png)](https://datakitchen.storylane.io/share/g01ss0plyamz)
+[![DataKitchen Open Source Data Observability](https://datakitchen.io/wp-content/uploads/2024/04/both-products.png)](https://datakitchen.storylane.io/share/g01ss0plyamz)
 [Interactive Product Tour](https://datakitchen.storylane.io/share/g01ss0plyamz)
 
 ## Developer Setup
@@ -100,9 +100,7 @@ We enforce the use of certain linting tools. To not get caught by the build-syst
 
 The following hooks are enabled in pre-commit:
 
-- `black`: The black formatter is enforced on the project. We use a basic configuration. Ideally this should solve any and all
-formatting questions we might encounter.
-- `isort`: the isort import-sorter is enforced on the project. We use it with the `black` profile.
+- `ruff`: Handles code formatting, import sorting, and linting
 
 To enable pre-commit from within your virtual environment, simply run:
 

agent_api/config/defaults.py

@@ -5,13 +5,12 @@
 """
 
 import os
-from typing import Optional
 
 # Flask specific settings: https://flask.palletsprojects.com/en/latest/config/#builtin-configuration-values
 from common.entities import Service
 
-PROPAGATE_EXCEPTIONS: Optional[bool] = None
-SERVER_NAME: Optional[str] = os.environ.get("AGENT_API_HOSTNAME")  # Use flask defaults if none set
+PROPAGATE_EXCEPTIONS: bool | None = None
+SERVER_NAME: str | None = os.environ.get("AGENT_API_HOSTNAME")  # Use flask defaults if none set
 USE_X_SENDFILE: bool = False  # If we serve files enable this in production settings when webserver support configured
 
 # Application settings

agent_api/config/local.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 # Flask specific settings: https://flask.palletsprojects.com/en/latest/config/#builtin-configuration-values
-PROPAGATE_EXCEPTIONS: Optional[bool] = True
+PROPAGATE_EXCEPTIONS: bool | None = True
 SECRET_KEY: str = "NOT_VERY_SECRET"

agent_api/config/minikube.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 # Flask specific settings: https://flask.palletsprojects.com/en/latest/config/#builtin-configuration-values
-TESTING: Optional[bool] = True
+TESTING: bool | None = True
 SECRET_KEY: str = "NOT_VERY_SECRET"

agent_api/endpoints/v1/heartbeat.py

@@ -1,7 +1,7 @@
 import logging
-from datetime import datetime, timezone
+from datetime import datetime, UTC
 from http import HTTPStatus
-from typing import Optional, Union, cast
+from typing import Union, cast
 from uuid import UUID
 
 from flask import Response, g, make_response
@@ -23,7 +23,7 @@ def _update_or_create(
     version: str,
     project_id: Union[str, UUID],
     latest_heartbeat: datetime,
-    latest_event_timestamp: Optional[datetime],
+    latest_event_timestamp: datetime | None,
 ) -> None:
     try:
         agent = Agent.select().where(Agent.key == key, Agent.tool == tool, Agent.project_id == project_id).get()
@@ -57,7 +57,7 @@ class Heartbeat(BaseView):
 
     def post(self) -> Response:
         data = self.parse_body(schema=HeartbeatSchema())
-        data["latest_heartbeat"] = datetime.now(tz=timezone.utc)
+        data["latest_heartbeat"] = datetime.now(tz=UTC)
         data["project_id"] = g.project.id
         _update_or_create(**data)
         return make_response("", HTTPStatus.NO_CONTENT)

agent_api/tests/integration/v1_endpoints/test_heartbeat.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timezone
+from datetime import datetime, timezone, UTC
 from http import HTTPStatus
 
 import pytest
@@ -9,7 +9,7 @@
 
 @pytest.mark.integration
 def test_agent_heartbeat(client, database_ctx, headers):
-    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=timezone.utc)
+    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=UTC)
     data = {
         "key": "test-key",
         "tool": "test-tool",
@@ -35,7 +35,7 @@ def test_agent_heartbeat_no_event_timestamp(client, database_ctx, headers):
 
 @pytest.mark.integration
 def test_agent_heartbeat_update(client, database_ctx, headers):
-    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=timezone.utc)
+    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=UTC)
     data = {
         "key": "test-key",
         "tool": "test-tool",
@@ -47,7 +47,7 @@ def test_agent_heartbeat_update(client, database_ctx, headers):
     assert HTTPStatus.NO_CONTENT == response_1.status_code, response_1.json
 
     # The latest_event_timestamp should be older than "now"
-    now = datetime.now(timezone.utc)
+    now = datetime.now(UTC)
     agent_1 = Agent.select().get()
     assert agent_1.latest_heartbeat < now
     assert agent_1.status == AgentStatus.ONLINE
@@ -62,7 +62,7 @@ def test_agent_heartbeat_update(client, database_ctx, headers):
 
 @pytest.mark.integration
 def test_agent_heartbeat_existing_update(client, database_ctx, headers):
-    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=timezone.utc)
+    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=UTC)
     data_1 = {
         "key": "test-key",
         "tool": "test-tool",
@@ -79,7 +79,7 @@ def test_agent_heartbeat_existing_update(client, database_ctx, headers):
 
     data_2 = data_1.copy()
     data_2["version"] = "12.0.3"
-    data_2["latest_event_timestamp"] = datetime(2023, 10, 20, 4, 44, 44, tzinfo=timezone.utc).isoformat()
+    data_2["latest_event_timestamp"] = datetime(2023, 10, 20, 4, 44, 44, tzinfo=UTC).isoformat()
 
     response_2 = client.post("/agent/v1/heartbeat", json=data_2, headers=headers)
     assert HTTPStatus.NO_CONTENT == response_2.status_code, response_2.json

cli/base.py

@@ -4,7 +4,7 @@
 from argparse import ArgumentParser
 from logging.config import dictConfig
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any
 from collections.abc import Callable
 
 from log_color import ColorFormatter, ColorStripper
@@ -80,7 +80,7 @@ def __init__(self, **kwargs: Any) -> None:
         LOG.info("#g<\u2714> Established #c<%s> connection to #c<%s>", DB.obj.__class__.__name__, DB.obj.database)
 
 
-def logging_init(*, level: str, logfile: Optional[str] = None) -> None:
+def logging_init(*, level: str, logfile: str | None = None) -> None:
     """Given the log level and an optional logging file location, configure all logging."""
     # Don't bother with a file handler if we're not logging to a file
     handlers = ["console", "filehandler"] if logfile else ["console"]

cli/entry_points/database_schema.py

@@ -1,6 +1,6 @@
 import re
 from argparse import ArgumentParser
-from typing import Any, Optional
+from typing import Any
 from re import Pattern
 from collections.abc import Iterable
 
@@ -19,7 +19,7 @@ class MysqlPrintDatabase(MySQLDatabase):
     def __init__(self) -> None:
         super().__init__("")
 
-    def execute_sql(self, sql: str, params: Optional[Iterable[Any]] = None, commit: Optional[bool] = None) -> None:
+    def execute_sql(self, sql: str, params: Iterable[Any] | None = None, commit: bool | None = None) -> None:
         if params:
             raise Exception(f"Params are not expected to be needed to run DDL SQL, but found {params}")
         if match := self._create_table_re.match(sql):

cli/entry_points/gen_events.py

@@ -7,7 +7,7 @@
 import time
 from argparse import Action, ArgumentParser, Namespace
 from datetime import datetime
-from typing import Any, Optional, Union
+from typing import Any, Union
 from collections.abc import Sequence
 
 from requests_extensions import get_session
@@ -90,7 +90,7 @@ def __call__(
             parser: ArgumentParser,
             namespace: Namespace,
             values: Union[str, Sequence[Any], None],
-            option_string: Optional[str] = None,
+            option_string: str | None = None,
         ) -> None:
             event_data = {}
             remove_fields = []

cli/entry_points/graph_schema.py

@@ -2,6 +2,7 @@
 import sys
 from argparse import ArgumentParser
 from pathlib import Path
+from typing import Any
 
 from jinja2 import Environment, FileSystemLoader
 from peewee import Field, ForeignKeyField, ManyToManyField, Model
@@ -54,7 +55,7 @@ def subcmd_entry_point(self) -> None:
         dot_parts = [head.render({})]
 
         # Initial context/config
-        model_context = []
+        model_context: list[dict[str, Any]] = []
 
         LOG.info("#m<Graphing models...>")
         for name, model in model_map.items():