From a87e82a1ed709c6ba9cd9a95e31d8bd40c38daac Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 16 Feb 2026 17:21:11 +0200 Subject: [PATCH 1/9] Add `btree_bloat` metric Based on the bloat estimation queries from https://github.com/ioguix/pgsql-bloat-estimation --- internal/metrics/metrics.yaml | 120 +++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 3 deletions(-) diff --git a/internal/metrics/metrics.yaml b/internal/metrics/metrics.yaml index d5aec37c08..a9d7551ad3 100644 --- a/internal/metrics/metrics.yaml +++ b/internal/metrics/metrics.yaml @@ -4187,9 +4187,123 @@ metrics: pg_ls_archive_statusdir() a where name ~ '[0-9A-F]{24}.ready'; - - - + btree_bloat: + description: + Estimates B-tree indexes bloat by comparing actual index size against expected size derived from pg_stats and pg_class catalog metadata. + It reports bloat percentage, real size, extra bloat bytes, and fillfactor for each index, helping identify candidates for REINDEX or maintenance. + Based on the bloat estimation queries from https://github.com/ioguix/pgsql-bloat-estimation. + sqls: + 14: |- + SELECT /* pgwatch_generated */ + (extract(epoch from now()) * 1e9)::bigint as epoch_ns, + current_database() AS tag_dbname, nspname AS tag_schemaname, tblname AS tag_tblname, idxname as tag_idxname, + bs*(relpages)::bigint AS real_size, + bs*(relpages-est_pages)::bigint AS extra_size, + 100 * (relpages-est_pages)::float / relpages AS extra_pct, + fillfactor, + CASE WHEN relpages > est_pages_ff + THEN bs*(relpages-est_pages_ff) + ELSE 0 + END AS bloat_size, + 100 * (relpages-est_pages_ff)::float / relpages AS bloat_pct, + is_na + -- , 100-(pst).avg_leaf_density AS pst_avg_bloat, est_pages, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples, relpages -- (DEBUG INFO) + FROM ( + SELECT coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)/(4+nulldatahdrwidth)::float)), 0 -- ItemIdData size + computed avg size of a tuple (nulldatahdrwidth) + ) AS est_pages, + coalesce(1 + + ceil(reltuples/floor((bs-pageopqdata-pagehdr)*fillfactor/(100*(4+nulldatahdrwidth)::float))), 0 + ) AS est_pages_ff, + bs, nspname, tblname, idxname, relpages, fillfactor, is_na + -- , pgstatindex(idxoid) AS pst, index_tuple_hdr_bm, maxalign, pagehdr, nulldatawidth, nulldatahdrwidth, reltuples -- (DEBUG INFO) + FROM ( + SELECT maxalign, bs, nspname, tblname, idxname, reltuples, relpages, idxoid, fillfactor, + ( index_tuple_hdr_bm + + maxalign - CASE -- Add padding to the index tuple header to align on MAXALIGN + WHEN index_tuple_hdr_bm%maxalign = 0 THEN maxalign + ELSE index_tuple_hdr_bm%maxalign + END + + nulldatawidth + maxalign - CASE -- Add padding to the data to align on MAXALIGN + WHEN nulldatawidth = 0 THEN 0 + WHEN nulldatawidth::integer%maxalign = 0 THEN maxalign + ELSE nulldatawidth::integer%maxalign + END + )::numeric AS nulldatahdrwidth, pagehdr, pageopqdata, is_na + -- , index_tuple_hdr_bm, nulldatawidth -- (DEBUG INFO) + FROM ( + SELECT n.nspname, i.tblname, i.idxname, i.reltuples, i.relpages, + i.idxoid, i.fillfactor, current_setting('block_size')::numeric AS bs, + CASE -- MAXALIGN: 4 on 32bits, 8 on 64bits (and mingw32 ?) + WHEN version() ~ 'mingw32' OR version() ~ '64-bit|x86_64|ppc64|ia64|amd64' THEN 8 + ELSE 4 + END AS maxalign, + /* per page header, fixed size: 20 for 7.X, 24 for others */ + 24 AS pagehdr, + /* per page btree opaque data */ + 16 AS pageopqdata, + /* per tuple header: add IndexAttributeBitMapData if some cols are null-able */ + CASE WHEN max(coalesce(s.null_frac,0)) = 0 + THEN 8 -- IndexTupleData size + ELSE 8 + (( 32 + 8 - 1 ) / 8) -- IndexTupleData size + IndexAttributeBitMapData size ( max num filed per index + 8 - 1 /8) + END AS index_tuple_hdr_bm, + /* data len: we remove null values save space using it fractionnal part from stats */ + sum( (1-coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 1024)) AS nulldatawidth, + max( CASE WHEN i.atttypid = 'pg_catalog.name'::regtype THEN 1 ELSE 0 END ) > 0 AS is_na + FROM ( + SELECT ct.relname AS tblname, ct.relnamespace, ic.idxname, ic.attpos, ic.indkey, ic.indkey[ic.attpos], ic.reltuples, ic.relpages, ic.tbloid, ic.idxoid, ic.fillfactor, + coalesce(a1.attnum, a2.attnum) AS attnum, coalesce(a1.attname, a2.attname) AS attname, coalesce(a1.atttypid, a2.atttypid) AS atttypid, + CASE WHEN a1.attnum IS NULL + THEN ic.idxname + ELSE ct.relname + END AS attrelname + FROM ( + SELECT idxname, reltuples, relpages, tbloid, idxoid, fillfactor, indkey, + pg_catalog.generate_series(1,indnatts) AS attpos + FROM ( + SELECT ci.relname AS idxname, ci.reltuples, ci.relpages, i.indrelid AS tbloid, + i.indexrelid AS idxoid, + coalesce(substring( + array_to_string(ci.reloptions, ' ') + from 'fillfactor=([0-9]+)')::smallint, 90) AS fillfactor, + i.indnatts, + pg_catalog.string_to_array(pg_catalog.textin( + pg_catalog.int2vectorout(i.indkey)),' ')::int[] AS indkey + FROM pg_catalog.pg_index i + JOIN pg_catalog.pg_class ci ON ci.oid = i.indexrelid + WHERE ci.relam=(SELECT oid FROM pg_am WHERE amname = 'btree') + AND ci.relpages > 0 + ) AS idx_data + ) AS ic + JOIN pg_catalog.pg_class ct ON ct.oid = ic.tbloid + LEFT JOIN pg_catalog.pg_attribute a1 ON + ic.indkey[ic.attpos] <> 0 + AND a1.attrelid = ic.tbloid + AND a1.attnum = ic.indkey[ic.attpos] + LEFT JOIN pg_catalog.pg_attribute a2 ON + ic.indkey[ic.attpos] = 0 + AND a2.attrelid = ic.idxoid + AND a2.attnum = ic.attpos + ) i + JOIN pg_catalog.pg_namespace n ON n.oid = i.relnamespace + JOIN pg_catalog.pg_stats s ON s.schemaname = n.nspname + AND s.tablename = i.attrelname + AND s.attname = i.attname + GROUP BY 1,2,3,4,5,6,7,8,9,10,11 + ) AS rows_data_stats + ) AS rows_hdr_pdg_stats + ) AS relation_stats + WHERE (bs * relpages::float / (1024 * 1024)) > 1 /* exclude indexes below 1 MiB */ + ORDER BY is_na, bloat_pct desc + LIMIT 100 + gauges: + - real_size + - extra_size + - extra_pct + - fillfactor + - bloat_size + - bloat_pct + - is_na presets: aiven: description: aiven database metrics From 1e0c812f66d61c7c531eecfba4d82f1007bf285a Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 16 Feb 2026 17:26:43 +0200 Subject: [PATCH 2/9] Add `btree_bloat: 7200` to presets --- internal/metrics/metrics.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/metrics/metrics.yaml b/internal/metrics/metrics.yaml index a9d7551ad3..83965cbc88 100644 --- a/internal/metrics/metrics.yaml +++ b/internal/metrics/metrics.yaml @@ -4293,7 +4293,9 @@ metrics: ) AS rows_data_stats ) AS rows_hdr_pdg_stats ) AS relation_stats - WHERE (bs * relpages::float / (1024 * 1024)) > 1 /* exclude indexes below 1 MiB */ + WHERE + (bs * relpages::float / (1024 * 1024)) > 1 /* exclude indexes below 1 MiB */ + AND relpages > est_pages_ff ORDER BY is_na, bloat_pct desc LIMIT 100 gauges: @@ -4336,6 +4338,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 300 table_stats: 300 wal: 60 @@ -4363,6 +4366,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 600 wal: 60 wal_receiver: 60 @@ -4401,6 +4405,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 600 table_stats: 300 wal: 60 @@ -4449,6 +4454,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 600 table_stats: 300 wal: 60 @@ -4475,6 +4481,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 600 table_stats: 300 wal: 60 @@ -4528,6 +4535,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 600 table_stats: 300 wal: 60 @@ -4571,6 +4579,7 @@ presets: stat_statements: 180 stat_statements_calls: 60 table_bloat_approx_summary_sql: 7200 + btree_bloat: 7200 table_io_stats: 600 table_stats: 300 wal: 60 @@ -4638,6 +4647,7 @@ presets: table_bloat_approx_stattuple: 30 table_bloat_approx_summary: 30 table_bloat_approx_summary_sql: 30 + btree_bloat: 30 table_hashes: 30 table_io_stats: 30 table_stats: 30 From 2ef8b249b31abaa2e8c5414d052e605e07f9ca6e Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 16 Feb 2026 19:38:18 +0200 Subject: [PATCH 3/9] Add `Top $n indexes by estimated bloat %` panel to pg index overview dashboard --- grafana/postgres/v12/index-overview.json | 263 ++++++++++++++++++++++- 1 file changed, 252 insertions(+), 11 deletions(-) diff --git a/grafana/postgres/v12/index-overview.json b/grafana/postgres/v12/index-overview.json index 97f4af32d4..5fc2adc03e 100644 --- a/grafana/postgres/v12/index-overview.json +++ b/grafana/postgres/v12/index-overview.json @@ -189,7 +189,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "WITH index_scan_changes AS (\n SELECT\n time,\n dbname,\n tag_data ->> 'index_full_name' AS index_full_name,\n tag_data ->> 'table_full_name' AS table_full_name,\n (data ->> 'idx_scan')::int8 - LEAD((data ->> 'idx_scan')::int8) OVER w AS idx_scans,\n (data ->> 'index_size_b')::int8 AS index_size_b,\n ROW_NUMBER() OVER w AS rn\n FROM\n index_stats\n WHERE\n dbname IN ($dbname)\n WINDOW w AS (\n PARTITION BY dbname,\n tag_data ->> 'table_full_name',\n tag_data ->> 'index_full_name'\n ORDER BY time DESC\n )\n),\ntotal_scans AS (\n SELECT\n dbname,\n table_full_name,\n index_full_name,\n SUM(COALESCE(idx_scans, 0)) AS total_idx_scans,\n MAX(index_size_b) FILTER (WHERE rn = 1) AS latest_index_size\n FROM\n index_scan_changes\n WHERE\n $__timeFilter(time)\n GROUP BY\n dbname, table_full_name, index_full_name\n)\nSELECT\n dbname AS \"Source Name\",\n index_full_name AS \"Index Name\",\n table_full_name AS \"Table Name\",\n latest_index_size AS \"Index Size\",\n total_idx_scans AS \"Scans\"\nFROM\n total_scans\nORDER BY\n latest_index_size DESC\nLIMIT $top", + "rawSql": "WITH index_scan_changes AS (\n SELECT\n time,\n dbname,\n tag_data ->> 'index_full_name' AS index_full_name,\n tag_data ->> 'table_full_name' AS table_full_name,\n (data ->> 'idx_scan')::int8 - LEAD((data ->> 'idx_scan')::int8) OVER w AS idx_scans,\n (data ->> 'index_size_b')::int8 AS index_size_b,\n ROW_NUMBER() OVER w AS rn\n FROM\n index_stats\n WHERE\n dbname IN ($dbname)\n WINDOW w AS (\n PARTITION BY dbname,\n tag_data ->> 'table_full_name',\n tag_data ->> 'index_full_name'\n ORDER BY time DESC\n )\n),\ntotal_scans AS (\n SELECT\n dbname,\n table_full_name,\n index_full_name,\n SUM(COALESCE(idx_scans, 0)) AS total_idx_scans,\n MAX(index_size_b) FILTER (WHERE rn = 1) AS latest_index_size\n FROM\n index_scan_changes\n WHERE\n $__timeFilter(time)\n GROUP BY\n dbname, table_full_name, index_full_name\n)\nSELECT\n dbname AS \"Source Name\",\n table_full_name AS \"Table Name\",\n index_full_name AS \"Index Name\",\n latest_index_size AS \"Index Size\",\n total_idx_scans AS \"Scans\"\nFROM\n total_scans\nORDER BY\n latest_index_size DESC\nLIMIT $top", "refId": "A", "resultFormat": "table", "select": [ @@ -245,6 +245,247 @@ ], "type": "table" }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "decimals": 2, + "displayName": "", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Table Name" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Opens 'Table details' dashboard", + "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" + } + ] + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.width", + "value": 265 + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Index Size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "custom.width", + "value": 301 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Index Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 306 + }, + { + "id": "custom.inspect", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Source Name" + }, + "properties": [ + { + "id": "custom.width", + "value": 186 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Estimated Bloat size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Estimated Bloat %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "pgwatch-metrics" + }, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "WITH last_fetch_time(time) AS (\n SELECT time\n FROM btree_bloat\n WHERE time <= $__timeTo()\n ORDER BY time DESC\n LIMIT 1\n)\n\nSELECT\n dbname AS \"Source Name\",\n tag_data->>'tblname' AS \"Table Name\",\n tag_data->>'idxname' AS \"Index Name\",\n data->>'real_size' AS \"Index Size\",\n data->>'bloat_pct' AS \"Estimated Bloat %\",\n data->>'bloat_size' AS \"Estimated Bloat size\"\nFROM\n btree_bloat\nWHERE time = (SELECT time FROM last_fetch_time)\nORDER BY (data->>'bloat_pct')::float8 DESC\nLIMIT $top\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top $top indexes by estimated bloat %", + "transformations": [ + { + "id": "merge", + "options": { + "reducers": [] + } + } + ], + "type": "table" + }, { "datasource": { "type": "grafana-postgresql-datasource", @@ -308,7 +549,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 16 }, "id": 13, "options": { @@ -426,7 +667,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 16 }, "id": 14, "options": { @@ -545,7 +786,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 24 }, "id": 15, "options": { @@ -664,7 +905,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 24 }, "id": 16, "options": { @@ -844,7 +1085,7 @@ "h": 8, "w": 24, "x": 0, - "y": 24 + "y": 32 }, "id": 12, "options": { @@ -1024,7 +1265,7 @@ "h": 7, "w": 12, "x": 0, - "y": 32 + "y": 40 }, "id": 11, "options": { @@ -1259,7 +1500,7 @@ "h": 7, "w": 12, "x": 12, - "y": 32 + "y": 40 }, "id": 10, "options": { @@ -1522,7 +1763,7 @@ "h": 6, "w": 24, "x": 0, - "y": 39 + "y": 47 }, "id": 8, "options": { @@ -1604,7 +1845,7 @@ "h": 7, "w": 24, "x": 0, - "y": 45 + "y": 53 }, "id": 6, "options": { @@ -1755,5 +1996,5 @@ "timezone": "browser", "title": "Index overview", "uid": "index-overview", - "version": 102 + "version": 4 } \ No newline at end of file From f848784b82d86019047550483bb0dd9d21b95be1 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 16 Feb 2026 19:50:39 +0200 Subject: [PATCH 4/9] Add `Top $n indexes by estimated bloat %` panel to prom index overview dashboard --- .../v12/index-overview-prometheus.json | 148 +++++++++++++++--- 1 file changed, 126 insertions(+), 22 deletions(-) diff --git a/grafana/prometheus/v12/index-overview-prometheus.json b/grafana/prometheus/v12/index-overview-prometheus.json index cc60f72558..4011c9c11e 100644 --- a/grafana/prometheus/v12/index-overview-prometheus.json +++ b/grafana/prometheus/v12/index-overview-prometheus.json @@ -19,7 +19,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 22, + "id": 34, "links": [], "panels": [ { @@ -70,6 +70,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": true, "stacking": { "group": "A", @@ -121,6 +122,7 @@ "sort": "none" } }, + "pluginVersion": "12.3.1", "targets": [ { "expr": "topk(10, sum by (index_name, dbname) (rate(pgwatch_index_stats_idx_scan{dbname=~\"$dbname\"}[$agg_interval])))", @@ -166,6 +168,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": true, "stacking": { "group": "A", @@ -217,6 +220,7 @@ "sort": "none" } }, + "pluginVersion": "12.3.1", "targets": [ { "expr": "topk(10, sum by (index_name, dbname) (rate(pgwatch_index_stats_idx_tup_fetch{dbname=~\"$dbname\"}[$agg_interval])))", @@ -275,6 +279,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": true, "stacking": { "group": "A", @@ -326,6 +331,7 @@ "sort": "none" } }, + "pluginVersion": "12.3.1", "targets": [ { "expr": "topk(10, sum by (index_name, dbname) (pgwatch_index_stats_idx_tup_read{dbname=~\"$dbname\"}) / sum by (index_name, dbname) (pgwatch_index_stats_idx_tup_fetch{dbname=~\"$dbname\"} > 0))", @@ -371,6 +377,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": true, "stacking": { "group": "A", @@ -422,6 +429,7 @@ "sort": "none" } }, + "pluginVersion": "12.3.1", "targets": [ { "expr": "topk(10, sum by (index_name, dbname) (pgwatch_index_stats_index_size_b{dbname=~\"$dbname\"}))", @@ -445,6 +453,107 @@ "title": "Index Health and Issues", "type": "row" }, + { + "datasource": { + "type": "prometheus", + "uid": "pgwatch-prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "expr": "topk(10, pgwatch_btree_bloat_bloat_pct{dbname=\"$dbname\"})", + "legendFormat": "{{dbname}}: {{schemaname}}.{{tblname}}.{{idxname}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 indexes by estimated bloat %", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -491,7 +600,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 28 }, "id": 8, "options": { @@ -511,7 +620,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "12.1.0", + "pluginVersion": "12.3.1", "targets": [ { "expr": "sum by (object_name, dbname) (metrics_recommendations_index_size_b{dbname=~\"$dbname\", issue_type=\"invalid\"})", @@ -557,6 +666,7 @@ "type": "linear" }, "showPoints": "never", + "showValues": false, "spanNulls": true, "stacking": { "group": "A", @@ -589,7 +699,7 @@ "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 28 }, "id": 9, "options": { @@ -607,6 +717,7 @@ "sort": "none" } }, + "pluginVersion": "12.3.1", "targets": [ { "expr": "topk(10, sum by (object_name, dbname) (metrics_recommendations_index_size_b{dbname=~\"$dbname\", issue_type=\"unused\"}))", @@ -623,7 +734,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 36 }, "id": 10, "panels": [], @@ -659,7 +770,7 @@ "h": 8, "w": 24, "x": 0, - "y": 28 + "y": 37 }, "id": 11, "options": { @@ -679,7 +790,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "12.1.0", + "pluginVersion": "12.3.1", "targets": [ { "expr": "count by (dbname) (pgwatch_index_stats_is_pk_int{dbname=~\"$dbname\"} == 1)", @@ -706,8 +817,9 @@ "type": "stat" } ], + "preload": false, "refresh": "30s", - "schemaVersion": 39, + "schemaVersion": 42, "tags": [ "pgwatch", "prometheus" @@ -724,7 +836,6 @@ "type": "prometheus" }, "definition": "label_values(pgwatch_instance_up, dbname)", - "hide": 0, "includeAll": true, "label": "Database", "multi": true, @@ -733,20 +844,16 @@ "query": "label_values(pgwatch_instance_up, dbname)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, - "text": "5m", - "value": "5m" + "text": "1h", + "value": "1h" }, - "hide": 0, "includeAll": false, "label": "Aggregation interval", - "multi": false, "name": "agg_interval", "options": [ { @@ -755,7 +862,7 @@ "value": "1m" }, { - "selected": true, + "selected": false, "text": "5m", "value": "5m" }, @@ -770,14 +877,12 @@ "value": "30m" }, { - "selected": false, + "selected": true, "text": "1h", "value": "1h" } ], "query": "1m,5m,10m,30m,1h", - "queryValue": "", - "skipUrlSync": false, "type": "custom" } ] @@ -790,6 +895,5 @@ "timezone": "", "title": "Index Overview (Prometheus)", "uid": "index-overview-prometheus", - "version": 1, - "weekStart": "" -} + "version": 4 +} \ No newline at end of file From 08283a718d62990e8a5b4b928962ff673f182b60 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 16 Feb 2026 19:55:26 +0200 Subject: [PATCH 5/9] Add `Top $n indexes by estimated bloat size` panel to prom index overview dashboard --- .../v12/index-overview-prometheus.json | 116 ++++++++++++++++-- 1 file changed, 109 insertions(+), 7 deletions(-) diff --git a/grafana/prometheus/v12/index-overview-prometheus.json b/grafana/prometheus/v12/index-overview-prometheus.json index 4011c9c11e..7e181be374 100644 --- a/grafana/prometheus/v12/index-overview-prometheus.json +++ b/grafana/prometheus/v12/index-overview-prometheus.json @@ -518,8 +518,8 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 24, + "h": 8, + "w": 12, "x": 0, "y": 19 }, @@ -554,6 +554,108 @@ "title": "Top 10 indexes by estimated bloat %", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "pgwatch-prometheus" + }, + "description": "The top 10 indexes by estimated bloat size among the top ~100 indexes by bloat % retrieved by the `btree_bloat` metric.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "editorMode": "code", + "expr": "topk(10, pgwatch_btree_bloat_bloat_size{dbname=\"$dbname\"})", + "legendFormat": "{{dbname}}: {{schemaname}}.{{tblname}}.{{idxname}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 indexes by estimated bloat size", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -600,7 +702,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 27 }, "id": 8, "options": { @@ -699,7 +801,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 27 }, "id": 9, "options": { @@ -734,7 +836,7 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 35 }, "id": 10, "panels": [], @@ -770,7 +872,7 @@ "h": 8, "w": 24, "x": 0, - "y": 37 + "y": 36 }, "id": 11, "options": { @@ -895,5 +997,5 @@ "timezone": "", "title": "Index Overview (Prometheus)", "uid": "index-overview-prometheus", - "version": 4 + "version": 5 } \ No newline at end of file From 5c07adb2fd44edab53ef4a09778778a4db35392b Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Mon, 16 Feb 2026 21:36:04 +0200 Subject: [PATCH 6/9] Add sizing note in metric description --- internal/metrics/metrics.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/metrics/metrics.yaml b/internal/metrics/metrics.yaml index 83965cbc88..02855edc6b 100644 --- a/internal/metrics/metrics.yaml +++ b/internal/metrics/metrics.yaml @@ -4192,6 +4192,7 @@ metrics: Estimates B-tree indexes bloat by comparing actual index size against expected size derived from pg_stats and pg_class catalog metadata. It reports bloat percentage, real size, extra bloat bytes, and fillfactor for each index, helping identify candidates for REINDEX or maintenance. Based on the bloat estimation queries from https://github.com/ioguix/pgsql-bloat-estimation. + **Note - The query filters to bloated indexes over 1 MiB; for large databases, you might only care about indexes over 100 MiB or so** sqls: 14: |- SELECT /* pgwatch_generated */ @@ -4297,7 +4298,7 @@ metrics: (bs * relpages::float / (1024 * 1024)) > 1 /* exclude indexes below 1 MiB */ AND relpages > est_pages_ff ORDER BY is_na, bloat_pct desc - LIMIT 100 + LIMIT 100; gauges: - real_size - extra_size From ee05b576a03670d100b1f0339e696b93a6299571 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Tue, 17 Feb 2026 15:16:09 +0200 Subject: [PATCH 7/9] Add note about `btree_bloat` metric in panels description --- grafana/postgres/v12/index-overview.json | 1 + grafana/prometheus/v12/index-overview-prometheus.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/grafana/postgres/v12/index-overview.json b/grafana/postgres/v12/index-overview.json index 5fc2adc03e..a475fa1ee4 100644 --- a/grafana/postgres/v12/index-overview.json +++ b/grafana/postgres/v12/index-overview.json @@ -250,6 +250,7 @@ "type": "grafana-postgresql-datasource", "uid": "pgwatch-metrics" }, + "description": "Note: The `btree_bloat` metric query filters to bloated indexes over 1 MiB by default; for large databases, you might only care about indexes over 100 MiB or so, so modify the metric accordingly in your metric configs.", "fieldConfig": { "defaults": { "color": { diff --git a/grafana/prometheus/v12/index-overview-prometheus.json b/grafana/prometheus/v12/index-overview-prometheus.json index 7e181be374..34bf35df51 100644 --- a/grafana/prometheus/v12/index-overview-prometheus.json +++ b/grafana/prometheus/v12/index-overview-prometheus.json @@ -458,7 +458,7 @@ "type": "prometheus", "uid": "pgwatch-prometheus" }, - "description": "", + "description": "Note: The `btree_bloat` metric query filters to bloated indexes over 1 MiB by default; for large databases, you might only care about indexes over 100 MiB or so, so modify the metric accordingly in your metric configs.", "fieldConfig": { "defaults": { "color": { From 6eb2ee8a0ecdb535afb04ff0881de56b27ddf369 Mon Sep 17 00:00:00 2001 From: Ahmed Gouda Date: Tue, 17 Feb 2026 15:38:39 +0200 Subject: [PATCH 8/9] Fix panels' queries Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- grafana/postgres/v12/index-overview.json | 2 +- grafana/prometheus/v12/index-overview-prometheus.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/grafana/postgres/v12/index-overview.json b/grafana/postgres/v12/index-overview.json index a475fa1ee4..3c9bdd34ec 100644 --- a/grafana/postgres/v12/index-overview.json +++ b/grafana/postgres/v12/index-overview.json @@ -431,7 +431,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "WITH last_fetch_time(time) AS (\n SELECT time\n FROM btree_bloat\n WHERE time <= $__timeTo()\n ORDER BY time DESC\n LIMIT 1\n)\n\nSELECT\n dbname AS \"Source Name\",\n tag_data->>'tblname' AS \"Table Name\",\n tag_data->>'idxname' AS \"Index Name\",\n data->>'real_size' AS \"Index Size\",\n data->>'bloat_pct' AS \"Estimated Bloat %\",\n data->>'bloat_size' AS \"Estimated Bloat size\"\nFROM\n btree_bloat\nWHERE time = (SELECT time FROM last_fetch_time)\nORDER BY (data->>'bloat_pct')::float8 DESC\nLIMIT $top\n", + "rawSql": "WITH last_fetch_time(time) AS (\n SELECT time\n FROM btree_bloat\n WHERE time <= $__timeTo()\n ORDER BY time DESC\n LIMIT 1\n)\n\nSELECT\n dbname AS \"Source Name\",\n tag_data->>'tblname' AS \"Table Name\",\n tag_data->>'idxname' AS \"Index Name\",\n data->>'real_size' AS \"Index Size\",\n data->>'bloat_pct' AS \"Estimated Bloat %\",\n data->>'bloat_size' AS \"Estimated Bloat size\"\nFROM\n btree_bloat\nWHERE time = (SELECT time FROM last_fetch_time)\n AND dbname IN ($dbname)\nORDER BY (data->>'bloat_pct')::float8 DESC\nLIMIT $top\n", "refId": "A", "resultFormat": "table", "select": [ diff --git a/grafana/prometheus/v12/index-overview-prometheus.json b/grafana/prometheus/v12/index-overview-prometheus.json index 34bf35df51..09477f67c3 100644 --- a/grafana/prometheus/v12/index-overview-prometheus.json +++ b/grafana/prometheus/v12/index-overview-prometheus.json @@ -545,7 +545,7 @@ "pluginVersion": "12.3.1", "targets": [ { - "expr": "topk(10, pgwatch_btree_bloat_bloat_pct{dbname=\"$dbname\"})", + "expr": "topk(10, pgwatch_btree_bloat_bloat_pct{dbname=~\"$dbname\"})", "legendFormat": "{{dbname}}: {{schemaname}}.{{tblname}}.{{idxname}}", "range": true, "refId": "A" @@ -647,7 +647,7 @@ "targets": [ { "editorMode": "code", - "expr": "topk(10, pgwatch_btree_bloat_bloat_size{dbname=\"$dbname\"})", + "expr": "topk(10, pgwatch_btree_bloat_bloat_size{dbname=~\"$dbname\"})", "legendFormat": "{{dbname}}: {{schemaname}}.{{tblname}}.{{idxname}}", "range": true, "refId": "A" From b44b1f733015e17665df10d03970eadb394aa714 Mon Sep 17 00:00:00 2001 From: 0xgouda Date: Tue, 17 Feb 2026 16:13:54 +0200 Subject: [PATCH 9/9] Fix 'Table Name' column link --- grafana/postgres/v12/index-overview.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grafana/postgres/v12/index-overview.json b/grafana/postgres/v12/index-overview.json index 3c9bdd34ec..3de6556d0d 100644 --- a/grafana/postgres/v12/index-overview.json +++ b/grafana/postgres/v12/index-overview.json @@ -71,7 +71,7 @@ { "targetBlank": true, "title": "Opens 'Table details' dashboard", - "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" + "url": "/d/table-details?var-dbname=${__data.fields['Source Name']}&var-table_full_name=${__data.fields['Table Name']}" } ] }, @@ -296,7 +296,7 @@ { "targetBlank": true, "title": "Opens 'Table details' dashboard", - "url": "/d/table-details?var-dbname=${__data.fields[\"dbname\"]}&var-table_full_name=${__data.fields[\"table_full_name\"]}" + "url": "/d/table-details?var-dbname=${__data.fields['Source Name']}&var-table_full_name=${__data.fields['Table Name']}" } ] }, @@ -431,7 +431,7 @@ "orderByTime": "ASC", "policy": "default", "rawQuery": true, - "rawSql": "WITH last_fetch_time(time) AS (\n SELECT time\n FROM btree_bloat\n WHERE time <= $__timeTo()\n ORDER BY time DESC\n LIMIT 1\n)\n\nSELECT\n dbname AS \"Source Name\",\n tag_data->>'tblname' AS \"Table Name\",\n tag_data->>'idxname' AS \"Index Name\",\n data->>'real_size' AS \"Index Size\",\n data->>'bloat_pct' AS \"Estimated Bloat %\",\n data->>'bloat_size' AS \"Estimated Bloat size\"\nFROM\n btree_bloat\nWHERE time = (SELECT time FROM last_fetch_time)\n AND dbname IN ($dbname)\nORDER BY (data->>'bloat_pct')::float8 DESC\nLIMIT $top\n", + "rawSql": "WITH last_fetch_time(time) AS (\n SELECT time\n FROM btree_bloat\n WHERE time <= $__timeTo()\n ORDER BY time DESC\n LIMIT 1\n)\n\nSELECT\n dbname AS \"Source Name\",\n (tag_data->>'schemaname')::text || '.' || (tag_data->>'tblname')::text AS \"Table Name\",\n tag_data->>'idxname' AS \"Index Name\",\n data->>'real_size' AS \"Index Size\",\n data->>'bloat_pct' AS \"Estimated Bloat %\",\n data->>'bloat_size' AS \"Estimated Bloat size\"\nFROM\n btree_bloat\nWHERE time = (SELECT time FROM last_fetch_time)\n AND dbname IN ($dbname)\nORDER BY (data->>'bloat_pct')::float8 DESC\nLIMIT $top\n", "refId": "A", "resultFormat": "table", "select": [