Skip to content

Commit ef70909

Browse files
l-wangreshke
l-wang
authored and
reshke
committed
Collect vacuum stats for append-optimized tables (#15262)
The following fields of the pg_stat_all_tables view are now available for append-optimized tables: n_live_tup n_dead_tup last_vacuum vacuum_count Note that n_dead_tup only accounts for dead tuples in non-awaiting-drop segment files. Reviewed-by: Hansong Fu <fuhuansong@gmail.com>
1 parent 7ffe7ea commit ef70909

7 files changed

Lines changed: 159 additions & 5 deletions

File tree

‎src/backend/commands/vacuum_ao.c‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ static bool appendonly_tid_reaped(ItemPointer itemptr, void *state);
157157

158158
static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel,
159159
BlockNumber *rel_pages, double *rel_tuples,
160-
bool *relhasindex, BlockNumber *total_file_segs);
160+
double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs);
161161
static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs,
162162
BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats);
163163
static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params,
@@ -236,6 +236,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt
236236
{
237237
BlockNumber relpages;
238238
double reltuples;
239+
double deadtuples;
239240
bool relhasindex;
240241
/* AO/AOCO total file segment number, use type BlockNumber to
241242
* represent same type with num_all_visible_pages in libpq.
@@ -282,6 +283,7 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt
282283
elevel,
283284
&relpages,
284285
&reltuples,
286+
&deadtuples,
285287
&relhasindex,
286288
&total_file_segs);
287289

@@ -306,6 +308,12 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt
306308
false,
307309
true /* isvacuum */);
308310

311+
/* report results to the stats collector, too */
312+
pgstat_report_vacuum(RelationGetRelid(onerel),
313+
onerel->rd_rel->relisshared,
314+
reltuples,
315+
deadtuples);
316+
309317
SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end");
310318
}
311319

@@ -684,7 +692,7 @@ appendonly_tid_reaped(ItemPointer itemptr, void *state)
684692
static void
685693
vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel,
686694
BlockNumber *rel_pages, double *rel_tuples,
687-
bool *relhasindex, BlockNumber *total_file_segs)
695+
double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs)
688696
{
689697
FileSegTotals *fstotal;
690698
BlockNumber nblocks;
@@ -737,6 +745,7 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel,
737745

738746
*rel_pages = nblocks;
739747
*rel_tuples = num_tuples;
748+
*dead_tuples = hidden_tupcount;
740749
*relhasindex = aorel->rd_rel->relhasindex;
741750
*total_file_segs = fstotal->totalfilesegs;
742751

‎src/test/isolation2/expected/setup.out‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,6 @@ CREATE
133133
CREATE OR REPLACE FUNCTION wait_till_master_shutsdown() RETURNS void AS $$ DECLARE i int; /* in func */ BEGIN i := 0; /* in func */ while i < 120 loop i := i + 1; /* in func */ PERFORM pg_sleep(.5); /* in func */ end loop; /* in func */ END; /* in func */ $$ LANGUAGE plpgsql;
134134
CREATE
135135

136+
-- Helper function that ensures stats collector receives stat from the latest operation.
137+
create or replace function wait_until_dead_tup_change_to(relid oid, stat_val_expected bigint) returns text as $$ declare stat_val int; /* in func */ i int; /* in func */ begin i := 0; /* in func */ while i < 1200 loop select pg_stat_get_dead_tuples(relid) into stat_val; /* in func */ if stat_val = stat_val_expected then /* in func */ return 'OK'; /* in func */ end if; /* in func */ perform pg_sleep(0.1); /* in func */ perform pg_stat_clear_snapshot(); /* in func */ i := i + 1; /* in func */ end loop; /* in func */ return 'Fail'; /* in func */ end; /* in func */ $$ language plpgsql;
138+
CREATE

‎src/test/isolation2/expected/vacuum_progress_column.out‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,18 @@ ABORT
4545
DELETE FROM vacuum_progress_ao_column where j % 2 = 0;
4646
DELETE 50000
4747

48+
-- Lookup pg_class and collected stats view before VACUUM
49+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column';
50+
relpages | reltuples | relallvisible
51+
----------+-----------+---------------
52+
0 | 0 | 0
53+
(1 row)
54+
SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column';
55+
n_live_tup | n_dead_tup | last_vacuum | vacuum_count
56+
------------+------------+-------------+--------------
57+
100000 | 200000 | | 0
58+
(1 row)
59+
4860
-- Perform VACUUM and observe the progress
4961

5062
-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF.
@@ -145,6 +157,24 @@ SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segm
145157
1<: <... completed>
146158
VACUUM
147159

160+
-- pg_class and collected stats view should be updated after the 1st VACUUM
161+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0);
162+
wait_until_dead_tup_change_to
163+
-------------------------------
164+
OK
165+
(1 row)
166+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column';
167+
relpages | reltuples | relallvisible
168+
----------+-----------+---------------
169+
37 | 50000 | 0
170+
(1 row)
171+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column';
172+
n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count
173+
------------+------------+-----------------+--------------
174+
50000 | 0 | t | 1
175+
(1 row)
176+
177+
148178
-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run.
149179
SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
150180
gp_inject_fault
@@ -207,6 +237,23 @@ VACUUM
207237
---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+-----------------
208238
(0 rows)
209239

240+
-- pg_class and collected stats view should be updated after the 2nd VACUUM
241+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0);
242+
wait_until_dead_tup_change_to
243+
-------------------------------
244+
OK
245+
(1 row)
246+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column';
247+
relpages | reltuples | relallvisible
248+
----------+-----------+---------------
249+
13 | 50000 | 0
250+
(1 row)
251+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column';
252+
n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count
253+
------------+------------+-----------------+--------------
254+
50000 | 0 | t | 2
255+
(1 row)
256+
210257
-- Cleanup
211258
SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration;
212259
gp_inject_fault_infinite

‎src/test/isolation2/expected/vacuum_progress_row.out‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,18 @@ ABORT
4545
DELETE FROM vacuum_progress_ao_row where j % 2 = 0;
4646
DELETE 50000
4747

48+
-- Lookup pg_class and collected stats view before VACUUM
49+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row';
50+
relpages | reltuples | relallvisible
51+
----------+-----------+---------------
52+
0 | 0 | 0
53+
(1 row)
54+
SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row';
55+
n_live_tup | n_dead_tup | last_vacuum | vacuum_count
56+
------------+------------+-------------+--------------
57+
100000 | 200000 | | 0
58+
(1 row)
59+
4860
-- Perform VACUUM and observe the progress
4961

5062
-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF.
@@ -145,6 +157,23 @@ SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segm
145157
1<: <... completed>
146158
VACUUM
147159

160+
-- pg_class and collected stats view should be updated after the 1st VACUUM
161+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0);
162+
wait_until_dead_tup_change_to
163+
-------------------------------
164+
OK
165+
(1 row)
166+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row';
167+
relpages | reltuples | relallvisible
168+
----------+-----------+---------------
169+
83 | 50000 | 0
170+
(1 row)
171+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row';
172+
n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count
173+
------------+------------+-----------------+--------------
174+
50000 | 0 | t | 1
175+
(1 row)
176+
148177
-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run.
149178
SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
150179
gp_inject_fault
@@ -201,12 +230,29 @@ SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid)
201230
1<: <... completed>
202231
VACUUM
203232

204-
-- Vacuum has finished, nothing should show up in the view.
233+
-- Vacuum has finished, nothing should show up in the progress view.
205234
1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum;
206235
relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples
207236
---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+-----------------
208237
(0 rows)
209238

239+
-- pg_class and collected stats view should be updated after the 2nd VACUUM
240+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0);
241+
wait_until_dead_tup_change_to
242+
-------------------------------
243+
OK
244+
(1 row)
245+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row';
246+
relpages | reltuples | relallvisible
247+
----------+-----------+---------------
248+
28 | 50000 | 0
249+
(1 row)
250+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row';
251+
n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count
252+
------------+------------+-----------------+--------------
253+
50000 | 0 | t | 2
254+
(1 row)
255+
210256
-- Cleanup
211257
SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration;
212258
gp_inject_fault_infinite

‎src/test/isolation2/sql/setup.sql‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,4 +390,24 @@ $$ LANGUAGE plpgsql;
390390

391391
-- start_ignore
392392
set statement_timeout='180s';
393-
-- end_ignore
393+
-- end_ignore
394+
-- Helper function that ensures stats collector receives stat from the latest operation.
395+
create or replace function wait_until_dead_tup_change_to(relid oid, stat_val_expected bigint)
396+
returns text as $$
397+
declare
398+
stat_val int; /* in func */
399+
i int; /* in func */
400+
begin
401+
i := 0; /* in func */
402+
while i < 1200 loop
403+
select pg_stat_get_dead_tuples(relid) into stat_val; /* in func */
404+
if stat_val = stat_val_expected then /* in func */
405+
return 'OK'; /* in func */
406+
end if; /* in func */
407+
perform pg_sleep(0.1); /* in func */
408+
perform pg_stat_clear_snapshot(); /* in func */
409+
i := i + 1; /* in func */
410+
end loop; /* in func */
411+
return 'Fail'; /* in func */
412+
end; /* in func */
413+
$$ language plpgsql;

‎src/test/isolation2/sql/vacuum_progress_column.sql‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ CREATE INDEX on vacuum_progress_ao_column(j);
3030
-- Also delete half of the tuples evenly before the EOF of segno 2.
3131
DELETE FROM vacuum_progress_ao_column where j % 2 = 0;
3232

33+
-- Lookup pg_class and collected stats view before VACUUM
34+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column';
35+
SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column';
36+
3337
-- Perform VACUUM and observe the progress
3438

3539
-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF.
@@ -64,6 +68,12 @@ SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM
6468
SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
6569
1<:
6670

71+
-- pg_class and collected stats view should be updated after the 1st VACUUM
72+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0);
73+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column';
74+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column';
75+
76+
6777
-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run.
6878
SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
6979
1&: VACUUM vacuum_progress_ao_column;
@@ -86,6 +96,11 @@ SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid)
8696
-- Vacuum has finished, nothing should show up in the view.
8797
1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum;
8898

99+
-- pg_class and collected stats view should be updated after the 2nd VACUUM
100+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0);
101+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column';
102+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column';
103+
89104
-- Cleanup
90105
SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration;
91106
reset Debug_appendonly_print_compaction;

‎src/test/isolation2/sql/vacuum_progress_row.sql‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ CREATE INDEX on vacuum_progress_ao_row(j);
3030
-- Also delete half of the tuples evenly before the EOF of segno 2.
3131
DELETE FROM vacuum_progress_ao_row where j % 2 = 0;
3232

33+
-- Lookup pg_class and collected stats view before VACUUM
34+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row';
35+
SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row';
36+
3337
-- Perform VACUUM and observe the progress
3438

3539
-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF.
@@ -64,6 +68,11 @@ SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM
6468
SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
6569
1<:
6670

71+
-- pg_class and collected stats view should be updated after the 1st VACUUM
72+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0);
73+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row';
74+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row';
75+
6776
-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run.
6877
SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
6978
1&: VACUUM vacuum_progress_ao_row;
@@ -83,9 +92,14 @@ SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1
8392
SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p';
8493
1<:
8594

86-
-- Vacuum has finished, nothing should show up in the view.
95+
-- Vacuum has finished, nothing should show up in the progress view.
8796
1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum;
8897

98+
-- pg_class and collected stats view should be updated after the 2nd VACUUM
99+
1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0);
100+
SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row';
101+
SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row';
102+
89103
-- Cleanup
90104
SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration;
91105
reset Debug_appendonly_print_compaction;

0 commit comments

Comments
 (0)