Fix all ruff lint errors in CI

Author: mostafam-dev

src/scraperguard/alerts/models.py

@@ -3,11 +3,11 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 
 def _utcnow() -> datetime:
-    return datetime.now(timezone.utc)
+    return datetime.now(UTC)
 
 
 @dataclass

src/scraperguard/api/routes.py

@@ -126,9 +126,6 @@ async def get_report(
     # If url not specified, find the first snapshot for this run
     if url is None:
         # Query snapshots associated with this run — we need to find a URL
-        # Use a simple approach: get the run's snapshots via a direct query
-        snapshot = None
-        # Try to get a snapshot from this run via storage
         # The storage doesn't have a list-by-run method, so use the connection directly
         if hasattr(storage, '_conn'):
             cursor = storage._conn.execute(
@@ -145,12 +142,13 @@ async def get_report(
             content={"error": "No snapshots found for this run"},
         )
 
-    snapshot = storage.get_latest_snapshot(url)
+    storage.get_latest_snapshot(url)
     validation_result = storage.get_latest_validation_result(url, schema_name="")
     # Try to find any schema name for this URL
     if validation_result is None and hasattr(storage, '_conn'):
         cursor = storage._conn.execute(
-            "SELECT schema_name FROM validation_results WHERE url = ? ORDER BY timestamp DESC LIMIT 1",
+            "SELECT schema_name FROM validation_results"
+            " WHERE url = ? ORDER BY timestamp DESC LIMIT 1",
             (url,),
         )
         row = cursor.fetchone()

src/scraperguard/cli/main.py

@@ -13,7 +13,7 @@
 import json
 import time
 import urllib.request
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from pathlib import Path
 
 import click
@@ -54,7 +54,10 @@ def cli() -> None:
 @cli.command()
 @click.argument("target")
 @click.option("--schema", default=None, help="Path to a Python file with a BaseSchema subclass.")
-@click.option("--config", "config_path", default=None, help="Path to scraperguard.yaml config file.")
+@click.option(
+    "--config", "config_path", default=None,
+    help="Path to scraperguard.yaml config file.",
+)
 @click.option("--run-id", default=None, help="Run ID to group with (creates new if not provided).")
 @click.option("--selectors", default=None, help="Comma-separated CSS selectors to track.")
 @click.option("--store-raw-html", is_flag=True, default=False, help="Store raw HTML in snapshot.")
@@ -121,7 +124,7 @@ def run(
         metadata = SnapshotMetadata(
             http_status=http_status,
             latency_ms=latency_ms,
-            timestamp=datetime.now(timezone.utc),
+            timestamp=datetime.now(UTC),
             headers=headers,
             response_size_bytes=len(html.encode("utf-8")),
         )
@@ -156,7 +159,8 @@ def run(
                     click.echo(f"Warning: Drift analysis failed: {exc}", err=True)
                 storage.save_validation_result(validation_result)
                 click.echo(
-                    f"Schema validation: {validation_result.passed_count}/{validation_result.total_items} passed"
+                    f"Schema validation: "
+                    f"{validation_result.passed_count}/{validation_result.total_items} passed"
                 )
             except SchemaLoadError as exc:
                 click.echo(f"Warning: Schema load failed: {exc}", err=True)
@@ -168,7 +172,6 @@ def run(
         if selector_list:
             try:
                 current_tree = parse_to_tree(snapshot.normalized_html)
-                prev_snapshot = storage.get_latest_snapshot(url)
                 # get_latest_snapshot might return the one we just saved; get the one before
                 snapshots = storage.list_snapshots(url, limit=2)
                 prev_tree = None
@@ -193,7 +196,9 @@ def run(
                     if s.id != snapshot.id:
                         prev_snapshot_obj = s
                         break
-                if prev_snapshot_obj and should_diff(snapshot.fingerprint, prev_snapshot_obj.fingerprint):
+                if prev_snapshot_obj and should_diff(
+                    snapshot.fingerprint, prev_snapshot_obj.fingerprint,
+                ):
                     before_tree = parse_to_tree(prev_snapshot_obj.normalized_html)
                     after_tree = parse_to_tree(snapshot.normalized_html)
                     dom_changes = diff_trees(before_tree, after_tree)
@@ -208,7 +213,9 @@ def run(
                     if s.id != snapshot.id:
                         prev_snapshot_obj = s
                         break
-                if prev_snapshot_obj and should_diff(snapshot.fingerprint, prev_snapshot_obj.fingerprint):
+                if prev_snapshot_obj and should_diff(
+                    snapshot.fingerprint, prev_snapshot_obj.fingerprint,
+                ):
                     before_tree = parse_to_tree(prev_snapshot_obj.normalized_html)
                     after_tree = parse_to_tree(snapshot.normalized_html)
                     dom_changes = diff_trees(before_tree, after_tree)

src/scraperguard/config.py

@@ -17,7 +17,6 @@
 
 from scraperguard.storage.base import StorageBackend
 
-
 # ---------------------------------------------------------------------------
 # Configuration dataclasses
 # ---------------------------------------------------------------------------

src/scraperguard/core/classify/classifier.py

@@ -9,14 +9,14 @@
 
 import re
 from dataclasses import dataclass, field
-from enum import Enum
+from enum import StrEnum
 
-from scraperguard.core.dom_diff.differ import DOMChange, ChangeType
+from scraperguard.core.dom_diff.differ import ChangeType, DOMChange
 from scraperguard.core.dom_diff.selector_tracker import SelectorStatus
 from scraperguard.storage.models import ValidationResult
 
 
-class FailureType(str, Enum):
+class FailureType(StrEnum):
     """Known failure root causes."""
 
     SELECTOR_BREAK = "selector_break"
@@ -96,7 +96,10 @@ def _check_captcha(inp: ClassificationInput) -> Classification | None:
         confidence=confidence,
         evidence=[f"Found CAPTCHA signature: '{sig}'" for sig in found],
         affected_fields=[],
-        recommended_action="Target site is serving a CAPTCHA. Consider using a CAPTCHA-solving service or rotating IP addresses.",
+        recommended_action=(
+            "Target site is serving a CAPTCHA."
+            " Consider using a CAPTCHA-solving service or rotating IP addresses."
+        ),
         severity="critical",
     )
 
@@ -126,9 +129,15 @@ def _check_js_challenge(inp: ClassificationInput) -> Classification | None:
         return Classification(
             failure_type=FailureType.JS_CHALLENGE,
             confidence=0.80,
-            evidence=[f"Page has minimal text content ({len(visible_text)} chars) but contains {script_count} script tags"],
+            evidence=[
+                f"Page has minimal text content ({len(visible_text)} chars)"
+                f" but contains {script_count} script tags",
+            ],
             affected_fields=[],
-            recommended_action="Page requires JavaScript rendering. Use a browser-based scraper (Playwright, Selenium).",
+            recommended_action=(
+                "Page requires JavaScript rendering."
+                " Use a browser-based scraper (Playwright, Selenium)."
+            ),
             severity="critical",
         )
     return None
@@ -258,9 +267,15 @@ def _check_dom_restructure(inp: ClassificationInput) -> Classification | None:
         return Classification(
             failure_type=FailureType.DOM_RESTRUCTURE,
             confidence=confidence,
-            evidence=[f"Detected {total} structural DOM changes ({high_count} high severity)"],
+            evidence=[
+                f"Detected {total} structural DOM changes"
+                f" ({high_count} high severity)",
+            ],
             affected_fields=[],
-            recommended_action="Major structural change detected. Review page layout and update scraper selectors.",
+            recommended_action=(
+                "Major structural change detected."
+                " Review page layout and update scraper selectors."
+            ),
             severity=severity,
         )
     return None
@@ -286,7 +301,10 @@ def _check_ab_variant(inp: ClassificationInput) -> Classification | None:
         return Classification(
             failure_type=FailureType.AB_VARIANT,
             confidence=0.55,
-            evidence=[f"Partial selector failure ({broken_count}/{total_count}) with moderate structural changes suggests A/B variant"],
+            evidence=[
+                f"Partial selector failure ({broken_count}/{total_count})"
+                " with moderate structural changes suggests A/B variant",
+            ],
             affected_fields=[],
             recommended_action="Possible A/B test variant. Monitor over multiple runs to confirm.",
             severity="info",
@@ -305,7 +323,10 @@ def _check_partial_extraction(inp: ClassificationInput) -> Classification | None
             confidence=0.65,
             evidence=[f"{vr.failed_count}/{vr.total_items} items failed validation"],
             affected_fields=[],
-            recommended_action="Partial extraction failure. Some items are extracting correctly. Check specific failure patterns.",
+            recommended_action=(
+                "Partial extraction failure."
+                " Some items are extracting correctly. Check specific failure patterns."
+            ),
             severity="warning",
         )
     return None

src/scraperguard/core/dom_diff/differ.py

@@ -8,12 +8,12 @@
 
 from collections import Counter
 from dataclasses import dataclass, field
-from enum import Enum
+from enum import StrEnum
 
 from scraperguard.core.dom_diff.parser import DOMNode, find_nodes_by_selector
 
 
-class ChangeType(str, Enum):
+class ChangeType(StrEnum):
     """Categories of structural DOM changes."""
 
     NODE_REMOVED = "node_removed"
@@ -202,7 +202,10 @@ def _diff_children(before: DOMNode, after: DOMNode, changes: list[DOMChange]) ->
                 severity="medium",
                 details={
                     "before_order": [before_children[bi].tag for bi, _ in matches],
-                    "after_order": [after_children[ai].tag for _, ai in sorted(matches, key=lambda m: m[1])],
+                    "after_order": [
+                        after_children[ai].tag
+                        for _, ai in sorted(matches, key=lambda m: m[1])
+                    ],
                 },
                 message=f"Children reordered at {before.path}",
             ))

src/scraperguard/core/snapshot/normalizer.py

@@ -13,7 +13,6 @@
 import lxml.html
 from lxml.html import HtmlElement, tostring
 
-
 # ---------------------------------------------------------------------------
 # Configurable attribute removal / retention patterns
 # ---------------------------------------------------------------------------

src/scraperguard/health.py

@@ -12,7 +12,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 
 from scraperguard.core.classify.classifier import Classification
 from scraperguard.core.dom_diff.differ import DOMChange
@@ -54,7 +54,7 @@ class HealthReport:
     components: list[HealthComponent]
     classifications: list[Classification]
     drift_events: list[DriftEvent]
-    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
     run_id: str = ""
     url: str = ""
 
@@ -186,7 +186,10 @@ def compute_structural_stability(
         name="Structural Stability",
         score=score,
         weight=weight,
-        details=f"{len(dom_changes)} structural changes detected ({high} high, {medium} medium, {low} low severity)",
+        details=(
+            f"{len(dom_changes)} structural changes detected"
+            f" ({high} high, {medium} medium, {low} low severity)"
+        ),
     )
 
 

src/scraperguard/integrations/playwright/observer.py

@@ -12,9 +12,10 @@
 
 import logging
 import time
+from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
-from datetime import datetime, timezone
-from typing import TYPE_CHECKING, AsyncGenerator
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
 
 from scraperguard.core.classify.classifier import ClassificationInput, classify_failure
 from scraperguard.core.dom_diff.differ import diff_trees
@@ -130,7 +131,7 @@ async def _run_pipeline(self) -> None:
         metadata = SnapshotMetadata(
             http_status=200,
             latency_ms=latency_ms,
-            timestamp=datetime.now(timezone.utc),
+            timestamp=datetime.now(UTC),
             headers={},
             response_size_bytes=len(self._raw_html.encode("utf-8")),
         )

src/scraperguard/integrations/scrapy/middleware.py

@@ -9,7 +9,7 @@
 
 import logging
 import time
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING
 
 from scraperguard.config import ScraperGuardConfig, get_storage_backend, load_config
@@ -89,7 +89,10 @@ def from_crawler(cls, crawler: Crawler) -> ScraperGuardObserverMiddleware:
 
             return instance
         except Exception:
-            logger.exception("ScraperGuard: Failed to initialize middleware, creating no-op instance")
+            logger.exception(
+                "ScraperGuard: Failed to initialize middleware,"
+                " creating no-op instance",
+            )
             # Return a minimal instance that will pass-through everything
             instance = cls.__new__(cls)
             instance.config = ScraperGuardConfig()
@@ -133,7 +136,7 @@ def process_response(self, request: Request, response: Response, spider: Spider)
             metadata = SnapshotMetadata(
                 http_status=response.status,
                 latency_ms=latency_ms,
-                timestamp=datetime.now(timezone.utc),
+                timestamp=datetime.now(UTC),
                 headers=headers_dict,
                 response_size_bytes=len(response.body),
             )