Merge origin/main into claude/express-checkup-cli-DBMWi

NikolayS · NikolayS · commit bc6f5cb73d9a · 2025-12-27T11:24:52.000-08:00
Resolve conflict in reporter/postgres_reports.py docstring:
- Keep detailed scope documentation from our branch
- Add new check types (K004-K008) from main branch
diff --git a/README.md b/README.md
@@ -275,7 +275,7 @@ postgres_ai monitoring generates automated health check reports based on [postgr
 | Check ID | Title |
 |----------|-------|
 | K001 | Globally aggregated query metrics |
-| K003 | Top queries by total_time |
+| K003 | Top queries by total time (total_exec_time + total_plan_time) |
 | K004 | Top queries by temp bytes written |
 | K005 | Top queries by WAL generation |
 | K006 | Top queries by shared blocks read |
diff --git a/reporter/postgres_reports.py b/reporter/postgres_reports.py
diff --git a/reporter/requirements-dev.txt b/reporter/requirements-dev.txt
@@ -4,3 +4,4 @@ pytest-postgresql==7.0.2
 coverage==7.6.10
 pytest-cov==6.0.0
 jsonschema==4.23.0
+PyYAML==6.0.2
diff --git a/reporter/schemas/K003.schema.json b/reporter/schemas/K003.schema.json
@@ -9,7 +9,7 @@
     "build_ts": { "type": ["string", "null"] },
     "generation_mode": { "type": ["string", "null"] },
     "checkId": { "const": "K003" },
-    "checkTitle": { "const": "Top queries by total_time" },
+    "checkTitle": { "const": "Top queries by total time (total_exec_time + total_plan_time)" },
     "timestamptz": { "type": "string" },
     "nodes": { "$ref": "#/$defs/nodes" },
     "results": {
diff --git a/reporter/schemas/K004.schema.json b/reporter/schemas/K004.schema.json
@@ -9,7 +9,7 @@
     "build_ts": { "type": ["string", "null"] },
     "generation_mode": { "type": ["string", "null"] },
     "checkId": { "const": "K004" },
-    "checkTitle": { "const": "Check K004" },
+    "checkTitle": { "const": "Top queries by temp bytes written" },
     "timestamptz": { "type": "string" },
     "nodes": { "$ref": "#/$defs/nodes" },
     "results": {
diff --git a/reporter/schemas/K005.schema.json b/reporter/schemas/K005.schema.json
@@ -9,7 +9,7 @@
     "build_ts": { "type": ["string", "null"] },
     "generation_mode": { "type": ["string", "null"] },
     "checkId": { "const": "K005" },
-    "checkTitle": { "const": "Check K005" },
+    "checkTitle": { "const": "Top queries by WAL generation" },
     "timestamptz": { "type": "string" },
     "nodes": { "$ref": "#/$defs/nodes" },
     "results": {
diff --git a/reporter/schemas/K006.schema.json b/reporter/schemas/K006.schema.json
@@ -9,7 +9,7 @@
     "build_ts": { "type": ["string", "null"] },
     "generation_mode": { "type": ["string", "null"] },
     "checkId": { "const": "K006" },
-    "checkTitle": { "const": "Check K006" },
+    "checkTitle": { "const": "Top queries by shared blocks read" },
     "timestamptz": { "type": "string" },
     "nodes": { "$ref": "#/$defs/nodes" },
     "results": {
diff --git a/reporter/schemas/K007.schema.json b/reporter/schemas/K007.schema.json
@@ -9,7 +9,7 @@
     "build_ts": { "type": ["string", "null"] },
     "generation_mode": { "type": ["string", "null"] },
     "checkId": { "const": "K007" },
-    "checkTitle": { "const": "Check K007" },
+    "checkTitle": { "const": "Top queries by shared blocks hit" },
     "timestamptz": { "type": "string" },
     "nodes": { "$ref": "#/$defs/nodes" },
     "results": {
diff --git a/reporter/schemas/K008.schema.json b/reporter/schemas/K008.schema.json
@@ -0,0 +1,107 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "K008 report schema",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["checkId", "checkTitle", "timestamptz", "nodes", "results"],
+  "properties": {
+    "version": { "type": ["string", "null"] },
+    "build_ts": { "type": ["string", "null"] },
+    "checkId": { "const": "K008" },
+    "checkTitle": { "const": "Top queries by shared blocks hit+read" },
+    "timestamptz": { "type": "string" },
+    "nodes": { "$ref": "#/$defs/nodes" },
+    "results": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": { "$ref": "#/$defs/nodeResult" }
+    }
+  },
+  "$defs": {
+    "nodes": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["primary", "standbys"],
+      "properties": {
+        "primary": { "type": "string" },
+        "standbys": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "postgresVersion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["version", "server_version_num", "server_major_ver", "server_minor_ver"],
+      "properties": {
+        "version": { "type": "string" },
+        "server_version_num": { "type": "string" },
+        "server_major_ver": { "type": "string" },
+        "server_minor_ver": { "type": "string" }
+      }
+    },
+    "timeline": {
+      "type": "array",
+      "items": { "type": "integer" }
+    },
+    "hourlyNumbers": {
+      "type": "array",
+      "items": { "type": "number" }
+    },
+    "queryEntry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["queryid", "total_shared_hit_read_bytes", "hourly_shared_hit_read_bytes"],
+      "properties": {
+        "queryid": { "type": "string" },
+        "total_shared_hit_read_bytes": { "type": "number" },
+        "hourly_shared_hit_read_bytes": { "$ref": "#/$defs/hourlyNumbers" }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "queries_returned",
+        "total_shared_hit_read_bytes",
+        "total_shared_hit_read_bytes_tracked_queries",
+        "total_shared_hit_read_bytes_other",
+        "time_range_hours",
+        "hourly_timestamps",
+        "limit"
+      ],
+      "properties": {
+        "queries_returned": { "type": "integer" },
+        "total_shared_hit_read_bytes": { "type": "number" },
+        "total_shared_hit_read_bytes_tracked_queries": { "type": "number" },
+        "total_shared_hit_read_bytes_other": { "type": "number" },
+        "time_range_hours": { "type": "integer" },
+        "hourly_timestamps": { "$ref": "#/$defs/timeline" },
+        "limit": { "type": "integer" }
+      }
+    },
+    "dbEntry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["top_queries", "other_shared_hit_read_bytes_hourly", "summary"],
+      "properties": {
+        "top_queries": { "type": "array", "items": { "$ref": "#/$defs/queryEntry" } },
+        "other_shared_hit_read_bytes_hourly": { "$ref": "#/$defs/hourlyNumbers" },
+        "summary": { "$ref": "#/$defs/summary" }
+      }
+    },
+    "data": {
+      "type": "object",
+      "additionalProperties": { "$ref": "#/$defs/dbEntry" }
+    },
+    "nodeResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["data"],
+      "properties": {
+        "data": { "$ref": "#/$defs/data" },
+        "postgres_version": { "$ref": "#/$defs/postgresVersion" }
+      }
+    }
+  }
+}
+
+
diff --git a/tests/reporter/test_extract_queryids_from_reports_unit.py b/tests/reporter/test_extract_queryids_from_reports_unit.py
@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import pytest
+
+from reporter.postgres_reports import PostgresReportGenerator
+
+
+@pytest.fixture(name="generator")
+def fixture_generator() -> PostgresReportGenerator:
+    return PostgresReportGenerator(
+        prometheus_url="http://prom.test",
+        postgres_sink_url="",
+    )
+
+
+@pytest.mark.unit
+def test_extract_queryids_from_reports_includes_query_metrics_and_top_queries(
+    generator: PostgresReportGenerator,
+) -> None:
+    reports = {
+        # K001-style report: query_metrics
+        "K001": {
+            "results": {
+                "node-1": {
+                    "data": {
+                        "db1": {
+                            "query_metrics": [
+                                {"queryid": "1"},
+                                {"queryid": "0"},  # excluded
+                                {"queryid": 2},  # int form
+                            ]
+                        }
+                    }
+                }
+            }
+        },
+        # K003-style report: top_queries
+        "K003": {
+            "results": {
+                "node-1": {
+                    "data": {
+                        "db1": {
+                            "top_queries": [
+                                {"queryid": "3"},
+                                {"queryid": "-4"},
+                            ]
+                        },
+                        "db2": {
+                            "top_queries": [
+                                {"queryid": "5"},
+                            ]
+                        },
+                    }
+                }
+            }
+        },
+        # D004 has sample_queries but should NOT be used for per-query file generation.
+        "D004": {
+            "results": {
+                "node-1": {
+                    "data": {
+                        "pg_stat_statements_status": {
+                            "sample_queries": [
+                                {"queryid": "999"},
+                            ]
+                        }
+                    }
+                }
+            }
+        },
+    }
+
+    out = generator.extract_queryids_from_reports(reports)
+
+    assert out["db1"] == {"1", "2", "3", "-4"}
+    assert out["db2"] == {"5"}
+    assert "999" not in (out["db1"] | out["db2"])
+
+
+@pytest.mark.unit
+def test_extract_queryids_from_reports_n001_includes_nonzero_query_id_only(
+    generator: PostgresReportGenerator,
+) -> None:
+    reports = {
+        "N001": {
+            "results": {
+                "node-1": {
+                    "data": {
+                        "db1": {
+                            "wait_event_types": {
+                                "CPU*": {
+                                    "queries_list": [
+                                        {"query_id": "0"},  # excluded
+                                        {"query_id": "10"},
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    out = generator.extract_queryids_from_reports(reports)
+
+    assert out == {"db1": {"10"}}
+
+
+@pytest.mark.unit
+def test_extract_queryids_from_reports_d004_only_is_empty(
+    generator: PostgresReportGenerator,
+) -> None:
+    reports = {
+        "D004": {
+            "results": {
+                "node-1": {
+                    "data": {
+                        "pg_stat_statements_status": {
+                            "sample_queries": [
+                                {"queryid": "-1100697950502680692"},
+                                {"queryid": "-115926913472768758"},
+                            ]
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    out = generator.extract_queryids_from_reports(reports)
+    assert out == {}
+
+
diff --git a/tests/reporter/test_generators_hourly_unit.py b/tests/reporter/test_generators_hourly_unit.py
@@ -139,6 +139,26 @@ def test_generate_k007_computes_totals(monkeypatch: pytest.MonkeyPatch, generato
     assert db["summary"]["total_shared_hit_bytes"] == pytest.approx(2.0 + sum(other))
 
 
+@pytest.mark.unit
+def test_generate_k008_computes_totals(monkeypatch: pytest.MonkeyPatch, generator: PostgresReportGenerator, fixed_pg_version) -> None:
+    monkeypatch.setattr(generator, "_get_postgres_version_info", lambda *args, **kwargs: fixed_pg_version)
+    monkeypatch.setattr(generator, "get_all_databases", lambda *args, **kwargs: ["db1"])
+
+    per_query = {"1": [6.0, 1.0], "2": [0.0, 3.0], "3": [2.0, 2.0]}
+    other = [1.0, 10.0]
+    timeline = [100, 200]
+
+    monkeypatch.setattr(generator, "_get_hourly_topk_pgss_data_sum2", lambda *args, **kwargs: (per_query, other, timeline))
+
+    report = generator.generate_k008_shared_hit_read_report("local", "node-1", time_range_minutes=120, limit=50)
+    db = report["results"]["node-1"]["data"]["db1"]
+
+    tracked = sum(sum(v) for v in per_query.values())
+    assert db["summary"]["total_shared_hit_read_bytes_tracked_queries"] == pytest.approx(tracked)
+    assert db["summary"]["total_shared_hit_read_bytes_other"] == pytest.approx(sum(other))
+    assert db["summary"]["total_shared_hit_read_bytes"] == pytest.approx(tracked + sum(other))
+
+
 @pytest.mark.unit
 def test_generate_m001_computes_mean(monkeypatch: pytest.MonkeyPatch, generator: PostgresReportGenerator, fixed_pg_version) -> None:
     monkeypatch.setattr(generator, "_get_postgres_version_info", lambda *args, **kwargs: fixed_pg_version)
diff --git a/tests/reporter/test_generators_unit.py b/tests/reporter/test_generators_unit.py
@@ -882,6 +882,7 @@ def _(*args, **kwargs):
         "generate_k005_wal_bytes_report",
         "generate_k006_shared_read_report",
         "generate_k007_shared_hit_report",
+        "generate_k008_shared_hit_read_report",
         "generate_m001_mean_time_report",
         "generate_m002_rows_report",
         "generate_m003_io_time_report",
diff --git a/tests/reporter/test_hourly_topk_pgss_multi_unit.py b/tests/reporter/test_hourly_topk_pgss_multi_unit.py
diff --git a/tests/reporter/test_report_schemas_hourly.py b/tests/reporter/test_report_schemas_hourly.py
