diff --git a/src/backend/access/aocs/aocs_compaction.c b/src/backend/access/aocs/aocs_compaction.c index e80887af9ba..9f86d590639 100644 --- a/src/backend/access/aocs/aocs_compaction.c +++ b/src/backend/access/aocs/aocs_compaction.c @@ -39,6 +39,8 @@ #include "utils/snapmgr.h" #include "utils/guc.h" #include "miscadmin.h" +#include "commands/progress.h" +#include "pgstat.h" /* * Hook for plugins to get control after move or throw away tuple in @@ -60,7 +62,7 @@ aocs_compaction_delete_hook_type aocs_compaction_delete_hook = NULL; * segments, including any empty ones we've left behind. */ void -AOCSCompaction_DropSegmentFile(Relation aorel, int segno) +AOCSCompaction_DropSegmentFile(Relation aorel, int segno, AOVacuumRelStats *vacrelstats) { int col; @@ -83,7 +85,7 @@ AOCSCompaction_DropSegmentFile(Relation aorel, int segno) fd = OpenAOSegmentFile(aorel, filenamepath, 0); if (fd >= 0) { - TruncateAOSegmentFile(fd, aorel, pseudoSegNo, 0); + TruncateAOSegmentFile(fd, aorel, pseudoSegNo, 0, vacrelstats); CloseAOSegmentFile(fd); } else @@ -109,7 +111,7 @@ AOCSCompaction_DropSegmentFile(Relation aorel, int segno) * transactions. */ void -AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, AOCSVPInfo *vpinfo) +AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, AOCSVPInfo *vpinfo, AOVacuumRelStats *vacrelstats) { const char *relname = RelationGetRelationName(aorel); int j; @@ -146,7 +148,7 @@ AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, AOCSVPInfo *vpinfo) fd = OpenAOSegmentFile(aorel, filenamepath, segeof); if (fd >= 0) { - TruncateAOSegmentFile(fd, aorel, fileSegNo, segeof); + TruncateAOSegmentFile(fd, aorel, fileSegNo, segeof, vacrelstats); CloseAOSegmentFile(fd); elogif(Debug_appendonly_print_compaction, LOG, @@ -221,7 +223,8 @@ static bool AOCSSegmentFileFullCompaction(Relation aorel, AOCSInsertDesc insertDesc, AOCSFileSegInfo *fsinfo, - Snapshot snapshot) + Snapshot snapshot, + AOVacuumRelStats *vacrelstats) { const char *relname; AppendOnlyVisimap visiMap; @@ -229,14 +232,15 @@ AOCSSegmentFileFullCompaction(Relation aorel, TupleDesc tupDesc; TupleTableSlot *slot; int compact_segno; - int64 movedTupleCount = 0; ResultRelInfo *resultRelInfo; MemTupleBinding *mt_bind; EState *estate; - AOTupleId *aoTupleId; - ItemPointerData otid; int64 tupleCount = 0; int64 tuplePerPage = INT_MAX; + int64 curr_num_dead_tuples = 0; + int64 prev_num_dead_tuples = 0; + int64 curr_heap_blks_scanned = 0; + int64 prev_heap_blks_scanned = 0; Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY); Assert(RelationIsAoCols(aorel)); @@ -294,21 +298,15 @@ AOCSSegmentFileFullCompaction(Relation aorel, { CHECK_FOR_INTERRUPTS(); - aoTupleId = (AOTupleId *) &slot->tts_tid; - otid = slot->tts_tid; - if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId)) - { - AOCSMoveTuple(slot, - insertDesc, - resultRelInfo, - estate); - movedTupleCount++; - } - else - { - /* Tuple is invisible and needs to be dropped */ - AppendOnlyThrowAwayTuple(aorel, slot, mt_bind); - } + /* + * AppendOnlyVisimap_IsVisible() has already been called in aocs_getnext(). + */ + Assert(AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, + (AOTupleId *) &slot->tts_tid)); + AOCSMoveTuple(slot, + insertDesc, + resultRelInfo, + estate); if (aocs_compaction_delete_hook) (*aocs_compaction_delete_hook) (aorel, &otid); @@ -316,12 +314,33 @@ AOCSSegmentFileFullCompaction(Relation aorel, /* * Check for vacuum delay point after approximatly a var block */ - tupleCount++; - if (VacuumCostActive && tupleCount % tuplePerPage == 0) + moved_tupleCount++; + if (VacuumCostActive && moved_tupleCount % tuplePerPage == 0) { vacuum_delay_point(); } + + /* + * Report that we are now scanning and compacting segment files. + */ + curr_num_dead_tuples = scanDesc->cur_seg_row + 1 - moved_tupleCount; + if (curr_num_dead_tuples > prev_num_dead_tuples) + { + pgstat_progress_update_param(PROGRESS_VACUUM_NUM_DEAD_TUPLES, + vacrelstats->num_dead_tuples + curr_num_dead_tuples); + prev_num_dead_tuples = curr_num_dead_tuples; + } + + curr_heap_blks_scanned = RelationGuessNumberOfBlocksFromSize(scanDesc->totalBytesRead); + if (curr_heap_blks_scanned > prev_heap_blks_scanned) + { + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, + curr_heap_blks_scanned); + prev_heap_blks_scanned = curr_heap_blks_scanned; + } } + /* Accumulate total number dead tuples */ + vacrelstats->num_dead_tuples += scanDesc->cur_seg_row - moved_tupleCount; MarkAOCSFileSegInfoAwaitingDrop(aorel, compact_segno); @@ -340,7 +359,7 @@ AOCSSegmentFileFullCompaction(Relation aorel, elogif(Debug_appendonly_print_compaction, LOG, "Finished compaction: " "AO segfile %d, relation %s, moved tuple count " INT64_FORMAT, - compact_segno, relname, movedTupleCount); + compact_segno, relname, tupleCount); AppendOnlyVisimap_Finish(&visiMap, NoLock); @@ -361,7 +380,7 @@ AOCSSegmentFileFullCompaction(Relation aorel, * The compaction segment file should be locked for this transaction in * the appendonlywriter.c code. * - * On exit, *insert_segno will be set to the the segment that was used as the + * On exit, *insert_segno will be set to the segment that was used as the * insertion target. The segfiles listed in 'avoid_segnos' will not be used * for insertion. * @@ -373,7 +392,8 @@ AOCSCompact(Relation aorel, int compaction_segno, int *insert_segno, bool isFull, - List *avoid_segnos) + List *avoid_segnos, + AOVacuumRelStats *vacrelstats) { const char *relname; AOCSInsertDesc insertDesc = NULL; @@ -407,7 +427,8 @@ AOCSCompact(Relation aorel, AOCSSegmentFileFullCompaction(aorel, insertDesc, fsinfo, - appendOnlyMetaDataSnapshot); + appendOnlyMetaDataSnapshot, + vacrelstats); insertDesc->skipModCountIncrement = true; aocs_insert_finish(insertDesc, NULL); diff --git a/src/backend/access/aocs/aocsam.c b/src/backend/access/aocs/aocsam.c index 9f87ae55b11..2c788d53290 100644 --- a/src/backend/access/aocs/aocsam.c +++ b/src/backend/access/aocs/aocsam.c @@ -592,8 +592,7 @@ aocs_beginscan_internal(Relation relation, &scan->checksum, NULL); - GetAppendOnlyEntryAuxOids(RelationGetRelid(relation), - scan->appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(relation, NULL, NULL, NULL, &visimaprelid, &visimapidxid); @@ -1008,8 +1007,7 @@ aocs_insert_init(Relation rel, int segno) &nd); desc->compType = NameStr(nd); - GetAppendOnlyEntryAuxOids(rel->rd_id, - desc->appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(rel, &desc->segrelid, &desc->blkdirrelid, NULL, &desc->visimaprelid, &desc->visimapidxid); @@ -1481,8 +1479,7 @@ aocs_fetch_init(Relation relation, bool checksum; Oid visimaprelid; Oid visimapidxid; - GetAppendOnlyEntryAuxOids(relation->rd_id, - appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(relation, &aocsFetchDesc->segrelid, NULL, NULL, &visimaprelid, &visimapidxid); @@ -1854,8 +1851,7 @@ aocs_delete_init(Relation rel) Snapshot snapshot = GetCatalogSnapshot(InvalidOid); - GetAppendOnlyEntryAuxOids(rel->rd_id, - snapshot, + GetAppendOnlyEntryAuxOids(rel, NULL, NULL, NULL, &visimaprelid, &visimapidxid); diff --git a/src/backend/access/aocs/aocsam_handler.c b/src/backend/access/aocs/aocsam_handler.c index 508473b70bf..28a37dcde60 100644 --- a/src/backend/access/aocs/aocsam_handler.c +++ b/src/backend/access/aocs/aocsam_handler.c @@ -1326,7 +1326,6 @@ heap_truncate_one_relid(Oid relid) static void aoco_relation_nontransactional_truncate(Relation rel) { - Oid ao_base_relid = RelationGetRelid(rel); Oid aoseg_relid = InvalidOid; Oid aoblkdir_relid = InvalidOid; Oid aovisimap_relid = InvalidOid; @@ -1334,7 +1333,7 @@ aoco_relation_nontransactional_truncate(Relation rel) ao_truncate_one_rel(rel); /* Also truncate the aux tables */ - GetAppendOnlyEntryAuxOids(ao_base_relid, NULL, + GetAppendOnlyEntryAuxOids(rel, &aoseg_relid, &aoblkdir_relid, NULL, &aovisimap_relid, NULL); @@ -1742,8 +1741,8 @@ aoco_index_build_range_scan(Relation heapRelation, Oid blkdirrelid; Oid blkidxrelid; - GetAppendOnlyEntryAuxOids(RelationGetRelid(aocoscan->rs_base.rs_rd), NULL, NULL, - &blkdirrelid, &blkidxrelid, NULL, NULL); + GetAppendOnlyEntryAuxOids(heapRelation, NULL, + &blkdirrelid, &blkidxrelid, NULL, NULL); /* * Note that block directory is created during creation of the first * index. If it is found empty, it means the block directory was created diff --git a/src/backend/access/aocs/aocssegfiles.c b/src/backend/access/aocs/aocssegfiles.c index 585f5d4e914..e227d875c1f 100644 --- a/src/backend/access/aocs/aocssegfiles.c +++ b/src/backend/access/aocs/aocssegfiles.c @@ -159,8 +159,7 @@ GetAOCSFileSegInfo(Relation prel, bool isNull; Oid segrelid; - GetAppendOnlyEntryAuxOids(prel->rd_id, - appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(prel, &segrelid, NULL, NULL, NULL, NULL); @@ -273,8 +272,7 @@ GetAllAOCSFileSegInfo(Relation prel, Assert(RelationIsAoCols(prel)); - GetAppendOnlyEntryAuxOids(prel->rd_id, - appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(prel, &segrelid, NULL, NULL, NULL, NULL); @@ -521,8 +519,7 @@ MarkAOCSFileSegInfoAwaitingDrop(Relation prel, int segno) Assert(RelationIsAoCols(prel)); appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid)); - GetAppendOnlyEntryAuxOids(prel->rd_id, - appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(prel, &segrelid, NULL, NULL, NULL, NULL); UnregisterSnapshot(appendOnlyMetaDataSnapshot); @@ -614,8 +611,7 @@ ClearAOCSFileSegInfo(Relation prel, int segno) RelationGetRelationName(prel)); appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid)); - GetAppendOnlyEntryAuxOids(prel->rd_id, - appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(prel, &segrelid, NULL, NULL, NULL, NULL); UnregisterSnapshot(appendOnlyMetaDataSnapshot); @@ -905,8 +901,7 @@ AOCSFileSegInfoAddVpe(Relation prel, int32 segno, } Oid segrelid; - GetAppendOnlyEntryAuxOids(prel->rd_id, - NULL, + GetAppendOnlyEntryAuxOids(prel, &segrelid, NULL, NULL, NULL, NULL); segrel = heap_open(segrelid, RowExclusiveLock); @@ -1024,8 +1019,7 @@ AOCSFileSegInfoAddCount(Relation prel, int32 segno, TupleDesc tupdesc; Oid segrelid; - GetAppendOnlyEntryAuxOids(prel->rd_id, - NULL, + GetAppendOnlyEntryAuxOids(prel, &segrelid, NULL, NULL, NULL, NULL); @@ -1212,8 +1206,7 @@ gp_aocsseg_internal(PG_FUNCTION_ARGS, Oid aocsRelOid) context->relnatts = aocsRel->rd_rel->relnatts; Oid segrelid; - GetAppendOnlyEntryAuxOids(aocsRel->rd_id, - appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(aocsRel, &segrelid, NULL, NULL, NULL, NULL); pg_aocsseg_rel = heap_open(segrelid, AccessShareLock); @@ -1425,8 +1418,7 @@ gp_aocsseg_history(PG_FUNCTION_ARGS) context->relnatts = aocsRel->rd_rel->relnatts; Oid segrelid; - GetAppendOnlyEntryAuxOids(aocsRel->rd_id, - NULL, + GetAppendOnlyEntryAuxOids(aocsRel, &segrelid, NULL, NULL, NULL, NULL); @@ -1552,7 +1544,7 @@ aocol_compression_ratio_internal(Relation parentrel) Assert(Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_UTILITY); - GetAppendOnlyEntryAuxOids(RelationGetRelid(parentrel), NULL, + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); Assert(OidIsValid(segrelid)); diff --git a/src/backend/access/appendonly/aomd.c b/src/backend/access/appendonly/aomd.c index c147562f7f9..dc1c1f6cdbb 100644 --- a/src/backend/access/appendonly/aomd.c +++ b/src/backend/access/appendonly/aomd.c @@ -28,16 +28,22 @@ #include #include "access/aomd.h" +#include "access/aocssegfiles.h" #include "access/appendonlytid.h" #include "access/appendonlywriter.h" +#include "access/appendonly_compaction.h" +#include "access/table.h" #include "catalog/catalog.h" #include "catalog/pg_appendonly.h" #include "cdb/cdbappendonlystorage.h" #include "cdb/cdbappendonlyxlog.h" #include "crypto/bufenc.h" +#include "commands/progress.h" #include "common/relpath.h" #include "pgstat.h" +#include "storage/bufmgr.h" #include "storage/sync.h" +#include "utils/faultinjector.h" #include "utils/guc.h" #define SEGNO_SUFFIX_LENGTH 12 @@ -46,6 +52,7 @@ static void mdunlink_ao_base_relfile(void *ctx); static bool mdunlink_ao_perFile(const int segno, void *ctx); static bool copy_append_only_data_perFile(const int segno, void *ctx); static bool truncate_ao_perFile(const int segno, void *ctx); +static uint64 ao_segfile_get_physical_size(Relation aorel, int segno, int col); int AOSegmentFilePathNameLen(Relation rel) @@ -177,13 +184,20 @@ CloseAOSegmentFile(File fd) * Truncate all bytes from offset to end of file. */ void -TruncateAOSegmentFile(File fd, Relation rel, int32 segFileNum, int64 offset) +TruncateAOSegmentFile(File fd, Relation rel, int32 segFileNum, int64 offset, AOVacuumRelStats *vacrelstats) { char *relname = RelationGetRelationName(rel); + int64 filesize_before; Assert(fd > 0); Assert(offset >= 0); + filesize_before = FileSize(fd); + if (filesize_before < offset) + ereport(ERROR, + (errmsg("\"%s\": file size smaller than logical eof: %m", + relname))); + /* * Call the 'fd' module with a 64-bit length since AO segment files * can be multi-gigabyte to the terabytes... @@ -192,9 +206,19 @@ TruncateAOSegmentFile(File fd, Relation rel, int32 segFileNum, int64 offset) ereport(ERROR, (errmsg("\"%s\": failed to truncate data after eof: %m", relname))); + if (vacrelstats) + { + /* report heap-equivalent blocks vacuumed */ + vacrelstats->nbytes_truncated += filesize_before - offset; + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_VACUUMED, + RelationGuessNumberOfBlocksFromSize(vacrelstats->nbytes_truncated)); + } + if (XLogIsNeeded() && RelationNeedsWAL(rel)) xlog_ao_truncate(rel->rd_node, segFileNum, offset); + SIMPLE_FAULT_INJECTOR("appendonly_after_truncate_segment_file"); + if (file_truncate_hook) { RelFileNodeBackend rnode; @@ -570,7 +594,7 @@ truncate_ao_perFile(const int segno, void *ctx) if (fd >= 0) { - TruncateAOSegmentFile(fd, aorel, segno, 0); + TruncateAOSegmentFile(fd, aorel, segno, 0, NULL); CloseAOSegmentFile(fd); } else @@ -584,3 +608,104 @@ truncate_ao_perFile(const int segno, void *ctx) return true; } + +/* + * Returns the total of segment files' on-disk size for an AO/AOCO relation. + * This is only used by AO vaccum progress reporting. + */ +uint64 +ao_rel_get_physical_size(Relation aorel) +{ + Relation pg_aoseg_rel; + TupleDesc pg_aoseg_dsc; + SysScanDesc aoscan; + HeapTuple tuple; + Snapshot appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid)); + Oid segrelid; + uint64 total_physical_size = 0; + + Assert(RelationIsAppendOptimized(aorel)); + + GetAppendOnlyEntryAuxOids(aorel, + &segrelid, NULL, NULL); + + pg_aoseg_rel = heap_open(segrelid, AccessShareLock); + pg_aoseg_dsc = RelationGetDescr(pg_aoseg_rel); + + aoscan = systable_beginscan(pg_aoseg_rel, InvalidOid, false, appendOnlyMetaDataSnapshot, 0, NULL); + while ((tuple = systable_getnext(aoscan)) != NULL) + { + int segno; + bool isNull; + + if (RelationIsAoRows(aorel)) + { + segno = DatumGetInt32(fastgetattr(tuple, + Anum_pg_aoseg_segno, + pg_aoseg_dsc, &isNull)); + total_physical_size += ao_segfile_get_physical_size(aorel, segno, -1); + } + else + { + Datum d; + AOCSVPInfo *vpinfo; + int col; + + Assert(RelationIsAoCols(aorel)); + + segno = DatumGetInt32(fastgetattr(tuple, + Anum_pg_aocs_segno, + pg_aoseg_dsc, &isNull)); + d = fastgetattr(tuple, + Anum_pg_aocs_vpinfo, + pg_aoseg_dsc, &isNull); + vpinfo = (AOCSVPInfo *) PG_DETOAST_DATUM(d); + + for (col = 0; col < vpinfo->nEntry; ++col) + total_physical_size += ao_segfile_get_physical_size(aorel, segno, col); + + if (DatumGetPointer(d) != (Pointer) vpinfo) + pfree(vpinfo); + } + } + systable_endscan(aoscan); + + heap_close(pg_aoseg_rel, AccessShareLock); + UnregisterSnapshot(appendOnlyMetaDataSnapshot); + return total_physical_size; +} + +static uint64 +ao_segfile_get_physical_size(Relation aorel, int segno, int col) +{ + const char *relname; + File fd; + int32 fileSegNo; + char filenamepath[MAXPGPATH]; + uint64 physical_size = 0; + + relname = RelationGetRelationName(aorel); + + MakeAOSegmentFileName(aorel, segno, col, &fileSegNo, filenamepath); + elogif(Debug_appendonly_print_compaction, LOG, + "Opening append-optimized relation \"%s\", relation id %u, relfilenode %u column #%d, logical segment #%d (physical segment file #%d)", + RelationGetRelationName(aorel), + aorel->rd_id, + aorel->rd_node.relNode, + col, + segno, + fileSegNo); + fd = PathNameOpenFile(filenamepath, O_RDONLY | PG_BINARY); + if (fd >= 0) + physical_size = FileDiskSize(fd); + else + elogif(Debug_appendonly_print_compaction, LOG, + "No gp_relation_node entry for append-optimized relation \"%s\", relation id %u, relfilenode %u column #%d, logical segment #%d (physical segment file #%d)", + relname, + aorel->rd_id, + aorel->rd_node.relNode, + col, + segno, + fileSegNo); + return physical_size; +} diff --git a/src/backend/access/appendonly/aosegfiles.c b/src/backend/access/appendonly/aosegfiles.c index f905f924c67..0e4e32a5167 100644 --- a/src/backend/access/appendonly/aosegfiles.c +++ b/src/backend/access/appendonly/aosegfiles.c @@ -108,7 +108,7 @@ InsertInitialSegnoEntry(Relation parentrel, int segno) /* New segments are always created in the latest format */ formatVersion = AOSegfileFormatVersion_GetLatest(); - GetAppendOnlyEntryAuxOids(parentrel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); pg_aoseg_rel = heap_open(segrelid, RowExclusiveLock); @@ -185,7 +185,7 @@ GetFileSegInfo(Relation parentrel, Snapshot appendOnlyMetaDataSnapshot, int segn FileSegInfo *fsinfo; Oid segrelid; - GetAppendOnlyEntryAuxOids(parentrel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); /* * Check the pg_aoseg relation to be certain the ao table segment file is @@ -351,7 +351,7 @@ GetAllFileSegInfo(Relation parentrel, FileSegInfo **result; Oid segrelid; - GetAppendOnlyEntryAuxOids(parentrel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); if (segrelid == InvalidOid) elog(ERROR, "could not find pg_aoseg aux table for AO table \"%s\"", @@ -615,7 +615,7 @@ ClearFileSegInfo(Relation parentrel, int segno) bool isNull; Oid segrelid; - GetAppendOnlyEntryAuxOids(parentrel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); Assert(RelationIsAoRows(parentrel)); @@ -753,7 +753,7 @@ UpdateFileSegInfo_internal(Relation parentrel, bool isNull; Oid segrelid; - GetAppendOnlyEntryAuxOids(parentrel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); Assert(RelationIsAoRows(parentrel)); Assert(newState >= AOSEG_STATE_USECURRENT && newState <= AOSEG_STATE_AWAITING_DROP); @@ -960,7 +960,7 @@ GetSegFilesTotals(Relation parentrel, Snapshot appendOnlyMetaDataSnapshot) result = (FileSegTotals *) palloc0(sizeof(FileSegTotals)); - GetAppendOnlyEntryAuxOids(parentrel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); if (segrelid == InvalidOid) elog(ERROR, "could not find pg_aoseg aux table for AO table \"%s\"", @@ -1079,7 +1079,7 @@ gp_aoseg_history(PG_FUNCTION_ARGS) errmsg("'%s' is not an append-only row relation", RelationGetRelationName(aocsRel)))); - GetAppendOnlyEntryAuxOids(aocsRel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(aocsRel, &segrelid, NULL, NULL, NULL, NULL); pg_aoseg_rel = table_open(segrelid, AccessShareLock); @@ -1231,7 +1231,7 @@ gp_aoseg(PG_FUNCTION_ARGS) errmsg("'%s' is not an append-only row relation", RelationGetRelationName(aocsRel)))); - GetAppendOnlyEntryAuxOids(aocsRel->rd_id, NULL, &segrelid, NULL, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(aocsRel, &segrelid, NULL, NULL, NULL, NULL); pg_aoseg_rel = table_open(segrelid, AccessShareLock); @@ -1387,7 +1387,7 @@ get_ao_distribution(PG_FUNCTION_ARGS) errmsg("'%s' is not an append-only relation", RelationGetRelationName(parentrel)))); - GetAppendOnlyEntryAuxOids(RelationGetRelid(parentrel), NULL, + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); Assert(OidIsValid(segrelid)); @@ -1564,7 +1564,7 @@ aorow_compression_ratio_internal(Relation parentrel) Assert(Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_UTILITY); - GetAppendOnlyEntryAuxOids(RelationGetRelid(parentrel), NULL, + GetAppendOnlyEntryAuxOids(parentrel, &segrelid, NULL, NULL, NULL, NULL); Assert(OidIsValid(segrelid)); diff --git a/src/backend/access/appendonly/appendonly_blkdir_udf.c b/src/backend/access/appendonly/appendonly_blkdir_udf.c index 2c807b58fb0..5ceb6ef00b3 100644 --- a/src/backend/access/appendonly/appendonly_blkdir_udf.c +++ b/src/backend/access/appendonly/appendonly_blkdir_udf.c @@ -97,7 +97,7 @@ gp_aoblkdir(PG_FUNCTION_ARGS) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function not supported on non append-optimized relation"))); sst = GetLatestSnapshot(); - GetAppendOnlyEntryAuxOids(aoRelOid, sst, + GetAppendOnlyEntryAuxOids(context->aorel, NULL, &blkdirrelid, NULL, NULL, NULL); sst = gp_select_invisible ? SnapshotAny : GetLatestSnapshot(); diff --git a/src/backend/access/appendonly/appendonly_compaction.c b/src/backend/access/appendonly/appendonly_compaction.c index 8fe266a38ea..07904e77dab 100644 --- a/src/backend/access/appendonly/appendonly_compaction.c +++ b/src/backend/access/appendonly/appendonly_compaction.c @@ -41,11 +41,14 @@ #include "catalog/pg_appendonly.h" #include "cdb/cdbappendonlyam.h" #include "cdb/cdbvars.h" +#include "commands/progress.h" #include "commands/vacuum.h" #include "executor/executor.h" #include "nodes/execnodes.h" +#include "pgstat.h" #include "storage/procarray.h" #include "storage/lmgr.h" +#include "utils/faultinjector.h" #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/relcache.h" @@ -73,7 +76,7 @@ appendonly_compaction_delete_hook_type appendonly_compaction_delete_hook = NULL; * segments, including any empty ones we've left behind. */ static void -AppendOnlyCompaction_DropSegmentFile(Relation aorel, int segno) +AppendOnlyCompaction_DropSegmentFile(Relation aorel, int segno, AOVacuumRelStats *vacrelstats) { char filenamepath[MAXPGPATH]; int32 fileSegNo; @@ -90,7 +93,7 @@ AppendOnlyCompaction_DropSegmentFile(Relation aorel, int segno) fd = OpenAOSegmentFile(aorel, filenamepath, 0); if (fd >= 0) { - TruncateAOSegmentFile(fd, aorel, fileSegNo, 0); + TruncateAOSegmentFile(fd, aorel, fileSegNo, 0, vacrelstats); CloseAOSegmentFile(fd); } else @@ -140,7 +143,7 @@ AppendOnlyCompaction_ShouldCompact(Relation aoRelation, Oid visimapidxid; Assert(RelationIsAppendOptimized(aoRelation)); - GetAppendOnlyEntryAuxOids(aoRelation->rd_id, appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(aoRelation, NULL, NULL, NULL, &visimaprelid, &visimapidxid); @@ -229,7 +232,7 @@ AppendOnlyCompaction_ShouldCompact(Relation aoRelation, * For the segment file is truncates to the eof. */ static void -AppendOnlySegmentFileTruncateToEOF(Relation aorel, int segno, int64 segeof) +AppendOnlySegmentFileTruncateToEOF(Relation aorel, int segno, int64 segeof, AOVacuumRelStats *vacrelstats) { const char *relname = RelationGetRelationName(aorel); File fd; @@ -255,7 +258,7 @@ AppendOnlySegmentFileTruncateToEOF(Relation aorel, int segno, int64 segeof) fd = OpenAOSegmentFile(aorel, filenamepath, segeof); if (fd >= 0) { - TruncateAOSegmentFile(fd, aorel, fileSegNo, segeof); + TruncateAOSegmentFile(fd, aorel, fileSegNo, segeof, vacrelstats); CloseAOSegmentFile(fd); elogif(Debug_appendonly_print_compaction, LOG, @@ -381,7 +384,8 @@ static void AppendOnlySegmentFileFullCompaction(Relation aorel, AppendOnlyInsertDesc insertDesc, FileSegInfo *fsinfo, - Snapshot appendOnlyMetaDataSnapshot) + Snapshot appendOnlyMetaDataSnapshot, + AOVacuumRelStats *vacrelstats) { const char *relname; AppendOnlyVisimap visiMap; @@ -400,6 +404,7 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, Oid visimaprelid; Oid visimapidxid; Oid blkdirrelid; + int64 heap_blks_scanned = 0; Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY); Assert(RelationIsAoRows(aorel)); @@ -412,7 +417,7 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, } relname = RelationGetRelationName(aorel); - GetAppendOnlyEntryAuxOids(aorel->rd_id, appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(aorel, NULL, &blkdirrelid, NULL, &visimaprelid, &visimapidxid); @@ -485,6 +490,10 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, { /* Tuple is invisible and needs to be dropped */ AppendOnlyThrowAwayTuple(aorel, slot, mt_bind); + vacrelstats->num_dead_tuples++; + // TODO: need to evaluate performance impact of reporting with such granularity + pgstat_progress_update_param(PROGRESS_VACUUM_NUM_DEAD_TUPLES, + vacrelstats->num_dead_tuples); } if (appendonly_compaction_delete_hook) @@ -500,6 +509,11 @@ AppendOnlySegmentFileFullCompaction(Relation aorel, } } + /* Report progress after compacting a segment file. */ + heap_blks_scanned += RelationGuessNumberOfBlocksFromSize(fsinfo->eof); + pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED, + heap_blks_scanned); + MarkFileSegInfoAwaitingDrop(aorel, compact_segno); AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno); @@ -548,7 +562,7 @@ AppendOptimizedCollectDeadSegments(Relation aorel) Assert(RelationIsAppendOptimized(aorel)); - GetAppendOnlyEntryAuxOids(aorel->rd_id, appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(aorel, &segrelid, NULL, NULL, NULL, NULL); pg_aoseg_rel = heap_open(segrelid, AccessShareLock); @@ -652,22 +666,22 @@ AppendOptimizedCollectDeadSegments(Relation aorel) * row, though. */ static inline void -AppendOptimizedDropDeadSegment(Relation aorel, int segno) +AppendOptimizedDropDeadSegment(Relation aorel, int segno, AOVacuumRelStats *vacrelstats) { if (RelationIsAoRows(aorel)) { - AppendOnlyCompaction_DropSegmentFile(aorel, segno); + AppendOnlyCompaction_DropSegmentFile(aorel, segno, vacrelstats); ClearFileSegInfo(aorel, segno); } else { - AOCSCompaction_DropSegmentFile(aorel, segno); + AOCSCompaction_DropSegmentFile(aorel, segno, vacrelstats); ClearAOCSFileSegInfo(aorel, segno); } } void -AppendOptimizedDropDeadSegments(Relation aorel, Bitmapset *segnos) +AppendOptimizedDropDeadSegments(Relation aorel, Bitmapset *segnos, AOVacuumRelStats *vacrelstats) { int segno; @@ -678,7 +692,7 @@ AppendOptimizedDropDeadSegments(Relation aorel, Bitmapset *segnos) segno = -1; while ((segno = bms_next_member(segnos, segno)) >= 0) - AppendOptimizedDropDeadSegment(aorel, segno); + AppendOptimizedDropDeadSegment(aorel, segno, vacrelstats); UnlockRelationForExtension(aorel, ExclusiveLock); } @@ -689,7 +703,7 @@ AppendOptimizedDropDeadSegments(Relation aorel, Bitmapset *segnos) * the segment file is skipped. */ void -AppendOptimizedTruncateToEOF(Relation aorel) +AppendOptimizedTruncateToEOF(Relation aorel, AOVacuumRelStats *vacrelstats) { const char *relname; Relation pg_aoseg_rel; @@ -709,7 +723,7 @@ AppendOptimizedTruncateToEOF(Relation aorel) */ LockDatabaseObject(aorel->rd_node.dbNode, (Oid)aorel->rd_node.relNode, 0, ExclusiveLock); - GetAppendOnlyEntryAuxOids(aorel->rd_id, appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(aorel, &segrelid, NULL, NULL, NULL, NULL); pg_aoseg_rel = heap_open(segrelid, AccessShareLock); @@ -751,7 +765,7 @@ AppendOptimizedTruncateToEOF(Relation aorel) Anum_pg_aoseg_eof, pg_aoseg_dsc, &isNull)); Assert(!isNull); - AppendOnlySegmentFileTruncateToEOF(aorel, segno, segeof); + AppendOnlySegmentFileTruncateToEOF(aorel, segno, segeof, vacrelstats); } else { @@ -760,7 +774,7 @@ AppendOptimizedTruncateToEOF(Relation aorel) pg_aoseg_dsc, &isNull); AOCSVPInfo *vpinfo = (AOCSVPInfo *) PG_DETOAST_DATUM(d); - AOCSSegmentFileTruncateToEOF(aorel, segno, vpinfo); + AOCSSegmentFileTruncateToEOF(aorel, segno, vpinfo, vacrelstats); if (DatumGetPointer(d) != (Pointer) vpinfo) pfree(vpinfo); @@ -790,7 +804,8 @@ AppendOnlyCompact(Relation aorel, int compaction_segno, int *insert_segno, bool isFull, - List *avoid_segnos) + List *avoid_segnos, + AOVacuumRelStats *vacrelstats) { const char *relname; AppendOnlyInsertDesc insertDesc = NULL; @@ -820,7 +835,8 @@ AppendOnlyCompact(Relation aorel, AppendOnlySegmentFileFullCompaction(aorel, insertDesc, fsinfo, - appendOnlyMetaDataSnapshot); + appendOnlyMetaDataSnapshot, + vacrelstats); insertDesc->skipModCountIncrement = true; appendonly_insert_finish(insertDesc, NULL); diff --git a/src/backend/access/appendonly/appendonly_visimap.c b/src/backend/access/appendonly/appendonly_visimap.c index 738f2b3da47..a4599e88f31 100644 --- a/src/backend/access/appendonly/appendonly_visimap.c +++ b/src/backend/access/appendonly/appendonly_visimap.c @@ -934,8 +934,7 @@ void AppendOnlyVisimap_Init_forUniqueCheck( Assert(snapshot->snapshot_type == SNAPSHOT_DIRTY || snapshot->snapshot_type == SNAPSHOT_SELF); - GetAppendOnlyEntryAuxOids(aoRel->rd_id, - InvalidSnapshot, /* catalog snapshot is enough */ + GetAppendOnlyEntryAuxOids(aoRel, NULL, NULL, NULL, &visimaprelid, &visimapidxid); if (!OidIsValid(visimaprelid) || !OidIsValid(visimapidxid)) elog(ERROR, "Could not find block directory for relation: %u", aoRel->rd_id); diff --git a/src/backend/access/appendonly/appendonly_visimap_udf.c b/src/backend/access/appendonly/appendonly_visimap_udf.c index b864cceeefa..0b4210ee938 100644 --- a/src/backend/access/appendonly/appendonly_visimap_udf.c +++ b/src/backend/access/appendonly/appendonly_visimap_udf.c @@ -89,7 +89,7 @@ gp_aovisimap(PG_FUNCTION_ARGS) Snapshot sst = GetLatestSnapshot(); - GetAppendOnlyEntryAuxOids(context->aorel->rd_id, sst, + GetAppendOnlyEntryAuxOids(context->aorel, NULL, NULL, NULL, &visimaprelid, &visimapidxid); @@ -204,7 +204,7 @@ gp_aovisimap_hidden_info(PG_FUNCTION_ARGS) Oid segrelid; snapshot = GetLatestSnapshot(); - GetAppendOnlyEntryAuxOids(context->parentRelation->rd_id, snapshot, + GetAppendOnlyEntryAuxOids(context->parentRelation, &segrelid, NULL, NULL, &visimaprelid, &visimapidxid); @@ -384,7 +384,7 @@ gp_aovisimap_entry(PG_FUNCTION_ARGS) Snapshot sst = GetLatestSnapshot(); - GetAppendOnlyEntryAuxOids(context->parentRelation->rd_id, sst, + GetAppendOnlyEntryAuxOids(context->parentRelation, NULL, NULL, NULL, &visimaprelid, &visimapidxid); diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c index 32a306f3ae0..f81d798c06a 100755 --- a/src/backend/access/appendonly/appendonlyam.c +++ b/src/backend/access/appendonly/appendonlyam.c @@ -1571,7 +1571,7 @@ appendonly_beginrangescan_internal(Relation relation, Oid visimaprelid; Oid visimapidxid; - GetAppendOnlyEntryAuxOids(relation->rd_id, NULL, + GetAppendOnlyEntryAuxOids(relation, NULL, NULL, NULL, &visimaprelid, &visimapidxid); AppendOnlyVisimap_Init(&scan->visibilityMap, @@ -2139,7 +2139,7 @@ appendonly_fetch_init(Relation relation, FormData_pg_appendonly aoFormData; int segno; - GetAppendOnlyEntry(relation->rd_id, &aoFormData); + GetAppendOnlyEntry(relation, &aoFormData); /* * increment relation ref count while scanning relation @@ -2543,7 +2543,7 @@ appendonly_delete_init(Relation rel) Oid visimaprelid; Oid visimapidxid; - GetAppendOnlyEntryAuxOids(rel->rd_id, NULL, NULL, NULL, NULL, &visimaprelid, &visimapidxid); + GetAppendOnlyEntryAuxOids(rel, NULL, NULL, NULL, &visimaprelid, &visimapidxid); AppendOnlyDeleteDesc aoDeleteDesc = palloc0(sizeof(AppendOnlyDeleteDescData)); @@ -2818,7 +2818,7 @@ appendonly_insert_init(Relation rel, int segno) */ Assert(aoInsertDesc->fsInfo->segno == segno); - GetAppendOnlyEntryAuxOids(aoInsertDesc->aoi_rel->rd_id, NULL, &segrelid, + GetAppendOnlyEntryAuxOids(aoInsertDesc->aoi_rel, &aoInsertDesc->segrelid, NULL, NULL, NULL, NULL); firstSequence = diff --git a/src/backend/access/appendonly/appendonlyam_handler.c b/src/backend/access/appendonly/appendonlyam_handler.c index 4478e2958c7..cb9d39f2faf 100644 --- a/src/backend/access/appendonly/appendonlyam_handler.c +++ b/src/backend/access/appendonly/appendonlyam_handler.c @@ -1200,7 +1200,7 @@ appendonly_relation_nontransactional_truncate(Relation rel) ao_truncate_one_rel(rel); /* Also truncate the aux tables */ - GetAppendOnlyEntryAuxOids(ao_base_relid, NULL, + GetAppendOnlyEntryAuxOids(rel, &aoseg_relid, &aoblkdir_relid, NULL, &aovisimap_relid, NULL); @@ -1628,7 +1628,7 @@ appendonly_index_build_range_scan(Relation heapRelation, Oid blkdirrelid; Oid blkidxrelid; - GetAppendOnlyEntryAuxOids(RelationGetRelid(aoscan->aos_rd), NULL, NULL, + GetAppendOnlyEntryAuxOids(aoscan->aos_rd, NULL, &blkdirrelid, &blkidxrelid, NULL, NULL); /* * Note that block directory is created during creation of the first @@ -2055,7 +2055,7 @@ appendonly_relation_size(Relation rel, ForkNumber forkNumber) result = 0; - GetAppendOnlyEntryAuxOids(rel->rd_id, NULL, &segrelid, NULL, + GetAppendOnlyEntryAuxOids(rel, &segrelid, NULL, NULL, NULL, NULL); if (segrelid == InvalidOid) diff --git a/src/backend/access/appendonly/appendonlyblockdirectory.c b/src/backend/access/appendonly/appendonlyblockdirectory.c index 5b20a82e228..b0d48760862 100644 --- a/src/backend/access/appendonly/appendonlyblockdirectory.c +++ b/src/backend/access/appendonly/appendonlyblockdirectory.c @@ -181,7 +181,7 @@ AppendOnlyBlockDirectory_Init_forSearch( Oid blkdiridxid; blockDirectory->aoRel = aoRel; - GetAppendOnlyEntryAuxOids(aoRel->rd_id, NULL, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); + GetAppendOnlyEntryAuxOids(aoRel, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); if (!OidIsValid(blkdirrelid)) { @@ -251,8 +251,7 @@ AppendOnlyBlockDirectory_Init_forUniqueChecks( Assert(snapshot->snapshot_type == SNAPSHOT_DIRTY || snapshot->snapshot_type == SNAPSHOT_SELF); - GetAppendOnlyEntryAuxOids(aoRel->rd_id, - InvalidSnapshot, /* catalog snapshot is enough */ + GetAppendOnlyEntryAuxOids(aoRel, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); if (!OidIsValid(blkdirrelid) || !OidIsValid(blkdiridxid)) @@ -310,7 +309,7 @@ AppendOnlyBlockDirectory_Init_forInsert( blockDirectory->aoRel = aoRel; blockDirectory->appendOnlyMetaDataSnapshot = appendOnlyMetaDataSnapshot; - GetAppendOnlyEntryAuxOids(aoRel->rd_id, NULL, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); + GetAppendOnlyEntryAuxOids(aoRel, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); if (!OidIsValid(blkdirrelid)) { @@ -379,7 +378,7 @@ AppendOnlyBlockDirectory_Init_addCol( blockDirectory->aoRel = aoRel; blockDirectory->appendOnlyMetaDataSnapshot = appendOnlyMetaDataSnapshot; - GetAppendOnlyEntryAuxOids(aoRel->rd_id, NULL, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); + GetAppendOnlyEntryAuxOids(aoRel, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); if (!OidIsValid(blkdirrelid)) { @@ -1035,7 +1034,7 @@ AppendOnlyBlockDirectory_DeleteSegmentFile(Relation aoRel, Oid blkdirrelid; Oid blkdiridxid; - GetAppendOnlyEntryAuxOids(aoRel->rd_id, NULL, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); + GetAppendOnlyEntryAuxOids(aoRel, NULL, &blkdirrelid, &blkdiridxid, NULL, NULL); Assert(OidIsValid(blkdirrelid)); Assert(OidIsValid(blkdiridxid)); diff --git a/src/backend/access/appendonly/appendonlywriter.c b/src/backend/access/appendonly/appendonlywriter.c index e010e12db37..7fceaed2388 100644 --- a/src/backend/access/appendonly/appendonlywriter.c +++ b/src/backend/access/appendonly/appendonlywriter.c @@ -141,7 +141,7 @@ LockSegnoForWrite(Relation rel, int segno) LockDatabaseObject(rel->rd_node.dbNode, (Oid)rel->rd_node.relNode, 0, ExclusiveLock); appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid)); - GetAppendOnlyEntryAuxOids(rel->rd_id, appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(rel, &segrelid, NULL, NULL, NULL, NULL); /* * Now pick a segment that is not in use, and is not over the allowed @@ -432,7 +432,7 @@ choose_segno_internal(Relation rel, List *avoid_segnos, choose_segno_mode mode) LogDistributedSnapshotInfo(snapshot, "Used snapshot: "); } - GetAppendOnlyEntryAuxOids(rel->rd_id, NULL, + GetAppendOnlyEntryAuxOids(rel, &segrelid, NULL, NULL, NULL, NULL); /* @@ -635,7 +635,7 @@ choose_new_segfile(Relation rel, bool *used, List *avoid_segnos) Snapshot appendOnlyMetaDataSnapshot; appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid)); - GetAppendOnlyEntryAuxOids(rel->rd_id, appendOnlyMetaDataSnapshot, + GetAppendOnlyEntryAuxOids(rel, &segrelid, NULL, NULL, NULL, NULL); UnregisterSnapshot(appendOnlyMetaDataSnapshot); diff --git a/src/backend/catalog/aocatalog.c b/src/backend/catalog/aocatalog.c index a59e1eae097..78ea25b7efd 100644 --- a/src/backend/catalog/aocatalog.c +++ b/src/backend/catalog/aocatalog.c @@ -85,15 +85,15 @@ CreateAOAuxiliaryTable( switch(relkind) { case RELKIND_AOVISIMAP: - GetAppendOnlyEntryAuxOids(relOid, NULL, NULL, + GetAppendOnlyEntryAuxOids(rel, NULL, NULL, NULL, &aoauxiliary_relid, &aoauxiliary_idxid); break; case RELKIND_AOBLOCKDIR: - GetAppendOnlyEntryAuxOids(relOid, NULL, NULL, + GetAppendOnlyEntryAuxOids(rel, NULL, &aoauxiliary_relid, &aoauxiliary_idxid, NULL, NULL); break; case RELKIND_AOSEGMENTS: - GetAppendOnlyEntryAuxOids(relOid, NULL, + GetAppendOnlyEntryAuxOids(rel, &aoauxiliary_relid, NULL, NULL, NULL, NULL); break; diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index cb5ff640b52..35089deef1f 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -4201,6 +4201,16 @@ reindex_relation(Oid relid, int flags, ReindexParams *params) i++; } + /* + * While we have the relation opened, obtain the aoseg_relid and + * aoblkdir_relid if it is an AO table. + */ + if ((flags & REINDEX_REL_PROCESS_TOAST) && relIsAO) + GetAppendOnlyEntryAuxOids(rel, + &aoseg_relid, + &aoblkdir_relid, NULL, + &aovisimap_relid, NULL); + /* * Close rel, but continue to hold the lock. */ @@ -4229,13 +4239,6 @@ reindex_relation(Oid relid, int flags, ReindexParams *params) result |= reindex_relation(toast_relid, flags, &newparams); } - /* Obtain the aoseg_relid and aoblkdir_relid if the relation is an AO table. */ - if ((flags & REINDEX_REL_PROCESS_TOAST) && relIsAO) - GetAppendOnlyEntryAuxOids(relid, NULL, - &aoseg_relid, - &aoblkdir_relid, NULL, - &aovisimap_relid, NULL); - /* * If an AO rel has a secondary segment list rel, reindex that too while we * still hold the lock on the master table. diff --git a/src/backend/catalog/pg_appendonly.c b/src/backend/catalog/pg_appendonly.c index a051bd9288d..512043e996b 100644 --- a/src/backend/catalog/pg_appendonly.c +++ b/src/backend/catalog/pg_appendonly.c @@ -180,48 +180,18 @@ GetAppendOnlyEntryAttributes(Oid relid, * * The OIDs will be retrieved only when the corresponding output variable is * not NULL. - * - * 'appendOnlyMetaDataSnapshot' can be passed as NULL, which means use the - * latest snapshot, like in systable_beginscan. */ void -GetAppendOnlyEntryAuxOids(Oid relid, - Snapshot appendOnlyMetaDataSnapshot, +GetAppendOnlyEntryAuxOids(Relation rel, Oid *segrelid, Oid *blkdirrelid, Oid *blkdiridxid, Oid *visimaprelid, Oid *visimapidxid) { - Relation pg_appendonly; - TupleDesc tupDesc; - ScanKeyData key[1]; - SysScanDesc scan; - HeapTuple tuple; Form_pg_appendonly aoForm; - /* - * Check the pg_appendonly relation to be certain the ao table - * is there. - */ - pg_appendonly = table_open(AppendOnlyRelationId, AccessShareLock); - tupDesc = RelationGetDescr(pg_appendonly); - - ScanKeyInit(&key[0], - Anum_pg_appendonly_relid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(relid)); - - scan = systable_beginscan(pg_appendonly, AppendOnlyRelidIndexId, true, - appendOnlyMetaDataSnapshot, 1, key); - tuple = systable_getnext(scan); - if (!HeapTupleIsValid(tuple)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("missing pg_appendonly entry for relation \"%s\"", - get_rel_name(relid)))); - - aoForm = (Form_pg_appendonly) GETSTRUCT(tuple); + aoForm = rel->rd_appendonly; if (segrelid != NULL) *segrelid = aoForm->segrelid; @@ -237,49 +207,15 @@ GetAppendOnlyEntryAuxOids(Oid relid, if (visimapidxid != NULL) *visimapidxid = aoForm->visimapidxid; - - /* Finish up scan and close pg_appendonly catalog. */ - systable_endscan(scan); - table_close(pg_appendonly, AccessShareLock); } void -GetAppendOnlyEntry(Oid relid, Form_pg_appendonly aoEntry) +GetAppendOnlyEntry(Relation rel, Form_pg_appendonly aoEntry) { - Relation pg_appendonly; - TupleDesc tupDesc; - ScanKeyData key[1]; - SysScanDesc scan; - HeapTuple tuple; Form_pg_appendonly aoForm; - /* - * Check the pg_appendonly relation to be certain the ao table - * is there. - */ - pg_appendonly = table_open(AppendOnlyRelationId, AccessShareLock); - tupDesc = RelationGetDescr(pg_appendonly); - - ScanKeyInit(&key[0], - Anum_pg_appendonly_relid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(relid)); - - scan = systable_beginscan(pg_appendonly, AppendOnlyRelidIndexId, true, - NULL, 1, key); - tuple = systable_getnext(scan); - if (!HeapTupleIsValid(tuple)) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("missing pg_appendonly entry for relation \"%s\"", - get_rel_name(relid)))); - - aoForm = (Form_pg_appendonly) GETSTRUCT(tuple); + aoForm = rel->rd_appendonly; memcpy(aoEntry, aoForm, APPENDONLY_TUPLE_SIZE); - - /* Finish up scan and close pg_appendonly catalog. */ - systable_endscan(scan); - table_close(pg_appendonly, AccessShareLock); } /* diff --git a/src/backend/catalog/pg_attribute_encoding.c b/src/backend/catalog/pg_attribute_encoding.c index 00c46e56b12..15ed72de8e1 100644 --- a/src/backend/catalog/pg_attribute_encoding.c +++ b/src/backend/catalog/pg_attribute_encoding.c @@ -195,7 +195,6 @@ cloneAttributeEncoding(Oid oldrelid, Oid newrelid, AttrNumber max_attno) n + 1, attoptions[n]); } - CommandCounterIncrement(); } List ** diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 090f1020dad..eb7ad87994b 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1620,6 +1620,9 @@ CREATE VIEW pg_stat_progress_vacuum AS WHEN 4 THEN 'cleaning up indexes' WHEN 5 THEN 'truncating heap' WHEN 6 THEN 'performing final cleanup' + WHEN 7 THEN 'append-optimized pre-cleanup' + WHEN 8 THEN 'append-optimized compact' + WHEN 9 THEN 'append-optimized post-cleanup' END AS phase, S.param2 AS heap_blks_total, S.param3 AS heap_blks_scanned, S.param4 AS heap_blks_vacuumed, S.param5 AS index_vacuum_count, diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index 6cdbc4888b3..61c54ab5015 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -823,7 +823,7 @@ DefineIndex(Oid relationId, rel = table_open(relationId, NoLock); if (RelationIsAppendOptimized(rel)) { - GetAppendOnlyEntryAuxOids(relationId, NULL, NULL, &blkdirrelid, NULL, NULL, NULL); + GetAppendOnlyEntryAuxOids(rel, NULL, &blkdirrelid, NULL, NULL, NULL); if (!OidIsValid(blkdirrelid)) lockmode = ShareRowExclusiveLock; /* Relation is AO, and has no block directory */ diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index d4851f819db..7453104849e 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -1891,7 +1891,7 @@ ao_aux_tables_safe_truncate(Relation rel) Oid aoblkdir_relid = InvalidOid; Oid aovisimap_relid = InvalidOid; - GetAppendOnlyEntryAuxOids(relid, NULL, &aoseg_relid, + GetAppendOnlyEntryAuxOids(rel, &aoseg_relid, &aoblkdir_relid, NULL, &aovisimap_relid, NULL); @@ -15540,7 +15540,7 @@ ATExecChangeOwner(Oid relationOid, Oid newOwnerId, bool recursing, LOCKMODE lock { Oid segrelid, blkdirrelid; Oid visimap_relid; - GetAppendOnlyEntryAuxOids(relationOid, NULL, + GetAppendOnlyEntryAuxOids(target_rel, &segrelid, &blkdirrelid, NULL, &visimap_relid, NULL); @@ -16133,7 +16133,7 @@ ATExecSetTableSpace(Oid tableOid, Oid newTableSpace, LOCKMODE lockmode) /* Get the ao sub objects */ if (RelationIsAppendOptimized(rel)) - GetAppendOnlyEntryAuxOids(tableOid, NULL, + GetAppendOnlyEntryAuxOids(rel, &relaosegrelid, &relaoblkdirrelid, &relaoblkdiridxid, &relaovisimaprelid, &relaovisimapidxid); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 7c588a872ce..8d66aa19514 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -44,6 +44,7 @@ #include "catalog/pg_namespace.h" #include "commands/cluster.h" #include "commands/defrem.h" +#include "commands/tablecmds.h" #include "commands/vacuum.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -55,8 +56,11 @@ #include "storage/proc.h" #include "storage/procarray.h" #include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/elog.h" #include "utils/fmgroids.h" #include "utils/memutils.h" +#include "utils/palloc.h" #include "utils/snapmgr.h" #include "utils/syscache.h" @@ -141,6 +145,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_s bool skip_locked = false; bool analyze = false; bool freeze = false; + bool ao_aux_only = false; bool full = false; bool disable_page_skipping = false; bool rootonly = false; @@ -148,7 +153,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_s int ao_phase = 0; bool process_toast = true; bool update_datfrozenxid = false; - ListCell *lc; + ListCell *lc; /* index_cleanup and truncate values unspecified for now */ params.index_cleanup = VACOPTVALUE_UNSPECIFIED; @@ -184,6 +189,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_s freeze = defGetBoolean(opt); else if (strcmp(opt->defname, "full") == 0) full = defGetBoolean(opt); + else if (strcmp(opt->defname, "ao_aux_only") == 0) + ao_aux_only = defGetBoolean(opt); else if (strcmp(opt->defname, "disable_page_skipping") == 0) disable_page_skipping = defGetBoolean(opt); else if (strcmp(opt->defname, "index_cleanup") == 0) @@ -260,6 +267,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel, bool auto_s (analyze ? VACOPT_ANALYZE : 0) | (freeze ? VACOPT_FREEZE : 0) | (full ? VACOPT_FULL : 0) | + (ao_aux_only ? VACOPT_AO_AUX_ONLY : 0) | (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0) | (process_toast ? VACOPT_PROCESS_TOAST : 0) | (update_datfrozenxid ? VACOPT_UPDATE_DATFROZENXID : 0); @@ -457,8 +465,8 @@ vacuum(List *relations, VacuumParams *params, */ if (relations != NIL) { - List *newrels = NIL; - ListCell *lc; + List *newrels = NIL; + ListCell *lc; foreach(lc, relations) { @@ -848,6 +856,12 @@ vacuum_open_relation(Oid relid, RangeVar *relation, bits32 options, * new VacuumRelation(s) to return. (But note that they will reference * unmodified parts of the input, eg column lists.) New data structures * are made in vac_context. + * + * GPDB: In addition to expanding a partitioned table to include its + * partitions, we also use this function to expand appendoptimized tables to + * their auxiliary tables if the AO_AUX_ONLY option is passed. This is + * something of a misnomer because we do not return the originally input AO + * table, so it's a replacement instead of strictly an expansion. */ static List * expand_vacuum_rel(VacuumRelation *vrel, int options) @@ -1102,8 +1116,60 @@ expand_vacuum_rel(VacuumRelation *vrel, int options) child_relid = parent_relid; } } - } + /* + * If AO_AUX_ONLY option is passed, replace list of relations with the + * auxiliary tables for each AO table in the list + */ + if ((options & VACOPT_AO_AUX_ONLY) != 0) + { + ListCell *lc; + List *ao_aux_vacrels = NIL; + Oid aoseg_relid = InvalidOid; + Oid aoblkdir_relid = InvalidOid; + Oid aovisimap_relid = InvalidOid; + + foreach (lc, vacrels) + { + VacuumRelation *part_vrel = lfirst_node(VacuumRelation, lc); + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(part_vrel->oid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", relid); + classForm = (Form_pg_class) GETSTRUCT(tuple); + if (IsAccessMethodAO(classForm->relam)) + { + Relation aorel = table_open(classForm->oid, AccessShareLock); + oldcontext = MemoryContextSwitchTo(vac_context); + + GetAppendOnlyEntryAuxOids(aorel, + &aoseg_relid, + &aoblkdir_relid, NULL, + &aovisimap_relid, NULL); + + /* make new VacuumRelations for each valid member of the 3 auxiliary tables */ + if (OidIsValid(aoseg_relid)) + ao_aux_vacrels = lappend(ao_aux_vacrels, makeVacuumRelation(NULL, aoseg_relid, part_vrel->va_cols)); + if (OidIsValid(aoblkdir_relid)) + ao_aux_vacrels = lappend(ao_aux_vacrels, makeVacuumRelation(NULL, aoblkdir_relid, part_vrel->va_cols)); + if (OidIsValid(aovisimap_relid)) + ao_aux_vacrels = lappend(ao_aux_vacrels, makeVacuumRelation(NULL, aovisimap_relid, part_vrel->va_cols)); + + MemoryContextSwitchTo(oldcontext); + table_close(aorel, AccessShareLock); + } + else + { + ereport(classForm->relispartition ? LOG : WARNING, + (errmsg("skipping \"%s\" for VACUUM AO_AUX_ONLY --- it is not an append-optimized table", + NameStr(classForm->relname))), + errdetail("it will not have auxiliary tables to vacuum")); + } + ReleaseSysCache(tuple); + } + /* return only the AO AUX vacrels, dropping all heap and AO tables from the to-vacuum list */ + vacrels = ao_aux_vacrels; + } + } return vacrels; } @@ -1137,12 +1203,23 @@ get_all_vacuum_rels(int options) * to be performed, caller will decide whether to process or ignore * them. */ - if (classForm->relkind != RELKIND_RELATION && + if ((options & VACOPT_AO_AUX_ONLY) == 0 && + classForm->relkind != RELKIND_RELATION && classForm->relkind != RELKIND_DIRECTORY_TABLE && classForm->relkind != RELKIND_MATVIEW && classForm->relkind != RELKIND_PARTITIONED_TABLE) continue; + /* + * If ao_aux_only option is given without a tablename, we only want all + * AO AUX tables + */ + if ((options & VACOPT_AO_AUX_ONLY) != 0 && + classForm->relkind != RELKIND_AOSEGMENTS && + classForm->relkind != RELKIND_AOBLOCKDIR && + classForm->relkind != RELKIND_AOVISIMAP) + continue; + /* skip mid-level partition tables if we have disabled collecting statistics for them */ if (!optimizer_analyze_midlevel_partition && classForm->relkind == RELKIND_PARTITIONED_TABLE && @@ -2183,23 +2260,27 @@ static bool vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, bool recursing) { - LOCKMODE lmode; - Relation rel; - LockRelId lockrelid; + LOCKMODE lmode; + Relation onerel; + LockRelId onerelid; Oid toast_relid; Oid aoseg_relid = InvalidOid; - Oid aoblkdir_relid = InvalidOid; - Oid aovisimap_relid = InvalidOid; + Oid aoblkdir_relid = InvalidOid; + Oid aovisimap_relid = InvalidOid; Oid save_userid; - RangeVar *this_rangevar = NULL; + RangeVar *this_rangevar = NULL; int ao_vacuum_phase; int save_sec_context; int save_nestlevel; - bool is_appendoptimized; - bool is_toast; + bool is_appendoptimized; + bool is_toast; Assert(params != NULL); +#ifdef FAULT_INJECTOR + char onerelname[NAMEDATALEN]; +#endif + ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); /* Begin a transaction for vacuuming this relation */ @@ -2286,6 +2367,11 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, return false; } +#ifdef FAULT_INJECTOR + // preserve relation name for us in fault tests + StrNCpy(onerelname, NameStr(onerel->rd_rel->relname), NAMEDATALEN); +#endif + /* * Check if relation needs to be skipped based on ownership. This check * happens also when building the relation list to vacuum for a manual @@ -2421,7 +2507,12 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, if (RelationIsAppendOptimized(rel)) { - GetAppendOnlyEntryAuxOids(RelationGetRelid(rel), NULL, + /* + * GPDB: AO tables should never be passed into vacuum_rel if the + * AO_AUX_ONLY option is specified + */ + Assert(!(params->options & VACOPT_AO_AUX_ONLY)); + GetAppendOnlyEntryAuxOids(onerel, &aoseg_relid, &aoblkdir_relid, NULL, &aovisimap_relid, NULL); @@ -2605,7 +2696,7 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, /* Restore userid and security context */ SetUserIdAndSecContext(save_userid, save_sec_context); - + /* all done with this class, but hold lock until commit */ if (rel) relation_close(rel, NoLock); @@ -2666,18 +2757,12 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, * "analyze" will not get done on the toast table. This is good, because * the toaster always uses hardcoded index access and statistics are * totally unimportant for toast relations. + * + * Note, for GPDB, set recursing to true for auxilary tables to avoid + * being dispatched vacuum separately. */ if (toast_relid != InvalidOid) - vacuum_rel(toast_relid, NULL, params, false); - - /* - * If an AO/CO table is empty on a segment, - * - * Similar to toast, a VacuumStmt object for each AO auxiliary relation is - * constructed and dispatched separately by the QD, when vacuuming the - * base AO relation. A backend executing dispatched VacuumStmt - * (GP_ROLE_EXECUTE), therefore, should not execute this block of code. - */ + vacuum_rel(toast_relid, NULL, params, true); /* do the same for an AO segments table, if any */ if (aoseg_relid != InvalidOid) @@ -2757,6 +2842,14 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params, */ UnlockRelationIdForSession(&lockrelid, lmode); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "vacuum_rel_finished_one_relation", + DDLNotSpecified, + "", /* databaseName */ + onerelname); /* tableName */ +#endif + /* Report that we really did it. */ return true; } @@ -3030,6 +3123,11 @@ vacuum_params_to_options_list(VacuumParams *params) options = lappend(options, makeDefElem("full", (Node *) makeInteger(1), -1)); optmask &= ~VACOPT_FULL; } + if (optmask & VACOPT_AO_AUX_ONLY) + { + options = lappend(options, makeDefElem("ao_aux_only", (Node *) makeInteger(1), -1)); + optmask &= ~VACOPT_AO_AUX_ONLY; + } if (optmask & VACOPT_SKIP_LOCKED) { options = lappend(options, makeDefElem("skip_locked", (Node *) makeInteger(1), -1)); diff --git a/src/backend/commands/vacuum_ao.c b/src/backend/commands/vacuum_ao.c index 531d0eade4a..37706c03370 100644 --- a/src/backend/commands/vacuum_ao.c +++ b/src/backend/commands/vacuum_ao.c @@ -117,6 +117,7 @@ #include "access/table.h" #include "access/aocs_compaction.h" +#include "access/aomd.h" #include "access/appendonlywriter.h" #include "access/appendonly_compaction.h" #include "access/genam.h" @@ -127,6 +128,7 @@ #include "cdb/cdbappendonlyam.h" #include "cdb/cdbtm.h" #include "cdb/cdbvars.h" +#include "commands/progress.h" #include "commands/vacuum.h" #include "pgstat.h" #include "postmaster/autovacuum.h" @@ -148,24 +150,34 @@ static void vacuum_appendonly_index(Relation indexRelation, Relation aoRelation, Bitmapset *dead_segs, int elevel, - BufferAccessStrategy bstrategy); + BufferAccessStrategy bstrategy, + AOVacuumRelStats *vacrelstats); static bool appendonly_tid_reaped(ItemPointer itemptr, void *state); static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - bool *relhasindex, BlockNumber *total_file_segs); + double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs); static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs, - BufferAccessStrategy bstrategy); + BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); static void ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, - BufferAccessStrategy bstrategy); + BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats); +static AOVacuumRelStats *init_vacrelstats(void); +static void cleanup_vacrelstats(AOVacuumRelStats **vacrelstatsp); static void -ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy) +ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) { char *relname; int elevel; int options = params->options; + FileSegTotals *fstotal; + const int initprog_index[] = { + PROGRESS_VACUUM_PHASE, + PROGRESS_VACUUM_TOTAL_HEAP_BLKS, + PROGRESS_VACUUM_MAX_DEAD_TUPLES + }; + int64 initprog_val[3]; if (options & VACOPT_VERBOSE) elevel = INFO; @@ -181,11 +193,27 @@ ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStr get_namespace_name(RelationGetNamespace(onerel)), relname))); + /* Get statistics from the pg_aoseg table for progress reporting */ + if (RelationIsAoRows(onerel)) + fstotal = GetSegFilesTotals(onerel, GetActiveSnapshot()); + else + { + Assert(RelationIsAoCols(onerel)); + fstotal = GetAOCSSSegFilesTotals(onerel, GetActiveSnapshot()); + } + + /* + * Report that we are now in pre-cleanup phase, advertising total # of + * heap-equivalent blocks + */ + initprog_val[0] = PROGRESS_VACUUM_PHASE_AO_PRE_CLEANUP; + initprog_val[1] = RelationGuessNumberOfBlocksFromSize( + ao_rel_get_physical_size(onerel)); + initprog_val[2] = fstotal->totaltuples; + pgstat_progress_update_multi_param(3, initprog_index, initprog_val); + /* - * Truncate AWAITING_DROP segments that are no longer visible to anyone - * to 0 bytes. We cannot actually remove them yet, because there might - * still be index entries pointing to them. We cannot recycle the segments - * until the indexes have been vacuumed. + * Recycle AWAITING_DROP segments that are no longer visible to anyone. * * This is optional. We'll drop old AWAITING_DROP segments in the * post-cleanup phase, too, but doing this first helps to reclaim some @@ -193,21 +221,22 @@ ao_vacuum_rel_pre_cleanup(Relation onerel, VacuumParams *params, BufferAccessStr * * This could run in a local transaction. */ - ao_vacuum_rel_recycle_dead_segments(onerel, params, bstrategy); + ao_vacuum_rel_recycle_dead_segments(onerel, params, bstrategy, vacrelstats); /* * Also truncate all live segments to the EOF values stored in pg_aoseg. * This releases space left behind by aborted inserts. */ - AppendOptimizedTruncateToEOF(onerel); + AppendOptimizedTruncateToEOF(onerel, vacrelstats); } static void -ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy) +ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) { BlockNumber relpages; double reltuples; + double deadtuples; bool relhasindex; /* AO/AOCO total file segment number, use type BlockNumber to * represent same type with num_all_visible_pages in libpq. @@ -242,14 +271,19 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt * 4. Update statistics. */ Assert(RelationIsAoRows(onerel) || RelationIsAoCols(onerel)); + Assert(vacrelstats != NULL); + + pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, + PROGRESS_VACUUM_PHASE_AO_POST_CLEANUP); - ao_vacuum_rel_recycle_dead_segments(onerel, params, bstrategy); + ao_vacuum_rel_recycle_dead_segments(onerel, params, bstrategy, vacrelstats); /* Update statistics in pg_class */ vacuum_appendonly_fill_stats(onerel, GetActiveSnapshot(), elevel, &relpages, &reltuples, + &deadtuples, &relhasindex, &total_file_segs); @@ -273,10 +307,18 @@ ao_vacuum_rel_post_cleanup(Relation onerel, VacuumParams *params, BufferAccessSt MultiXactCutoff, false, true /* isvacuum */); + + /* report results to the stats collector, too */ + pgstat_report_vacuum(RelationGetRelid(onerel), + onerel->rd_rel->relisshared, + reltuples, + deadtuples); + + SIMPLE_FAULT_INJECTOR("vacuum_ao_post_cleanup_end"); } static void -ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy) +ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) { int compaction_segno; int insert_segno; @@ -296,6 +338,7 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg DistributedTransactionContext == DTX_CONTEXT_QE_TWO_PHASE_IMPLICIT_WRITER || DistributedTransactionContext == DTX_CONTEXT_QE_TWO_PHASE_EXPLICIT_WRITER); Assert(RelationIsAoRows(onerel) || RelationIsAoCols(onerel)); + Assert(vacrelstats != NULL); if (options & VACOPT_VERBOSE) elevel = INFO; @@ -311,6 +354,8 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg get_namespace_name(RelationGetNamespace(onerel)), relname))); + pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, + PROGRESS_VACUUM_PHASE_AO_COMPACT); /* * Compact all the segfiles. Repeat as many times as required. * @@ -340,7 +385,8 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg compaction_segno, &insert_segno, (options & VACOPT_FULL) != 0, - compacted_segments); + compacted_segments, + vacrelstats); else { Assert(RelationIsAoCols(onerel)); @@ -348,7 +394,8 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg compaction_segno, &insert_segno, (options & VACOPT_FULL) != 0, - compacted_segments); + compacted_segments, + vacrelstats); } if (insert_segno != -1) @@ -361,6 +408,8 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg */ CommandCounterIncrement(); } + + SIMPLE_FAULT_INJECTOR("vacuum_ao_after_compact"); } /* @@ -371,20 +420,33 @@ ao_vacuum_rel_compact(Relation onerel, VacuumParams *params, BufferAccessStrateg void ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy) { + static AOVacuumRelStats *vacrelstats = NULL; Assert(RelationIsAppendOptimized(rel)); Assert(params != NULL); int ao_vacuum_phase = (params->options & VACUUM_AO_PHASE_MASK); + if (!vacrelstats) + { + Assert(ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE || Gp_role != GP_ROLE_EXECUTE); + + pgstat_progress_start_command(PROGRESS_COMMAND_VACUUM, RelationGetRelid(rel)); + vacrelstats = init_vacrelstats(); + } + /* * Do the actual work --- either FULL or "lazy" vacuum */ if (ao_vacuum_phase == VACOPT_AO_PRE_CLEANUP_PHASE) - ao_vacuum_rel_pre_cleanup(rel, params, bstrategy); + ao_vacuum_rel_pre_cleanup(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_COMPACT_PHASE) - ao_vacuum_rel_compact(rel, params, bstrategy); + ao_vacuum_rel_compact(rel, params, bstrategy, vacrelstats); else if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) - ao_vacuum_rel_post_cleanup(rel, params, bstrategy); + { + ao_vacuum_rel_post_cleanup(rel, params, bstrategy, vacrelstats); + pgstat_progress_end_command(); + cleanup_vacrelstats(&vacrelstats); + } else /* Do nothing here, we will launch the stages later */ Assert(ao_vacuum_phase == 0); @@ -394,7 +456,8 @@ ao_vacuum_rel(Relation rel, VacuumParams *params, BufferAccessStrategy bstrategy * Recycling AWAITING_DROP segments. */ static void -ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, BufferAccessStrategy bstrategy) +ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, + BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) { Bitmapset *dead_segs; int options = params->options; @@ -424,7 +487,7 @@ ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, Buffe * (AWAITING_DROP state in pg_aoseg) to cleanup dead index tuples * effectively. */ - vacuum_appendonly_indexes(onerel, options, dead_segs, bstrategy); + vacuum_appendonly_indexes(onerel, options, dead_segs, bstrategy, vacrelstats); /* * Truncate above collected AWAITING_DROP segments to 0 byte. * AppendOptimizedCollectDeadSegments() should guarantee that @@ -433,7 +496,7 @@ ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, Buffe * ExclusiveLock will be held in case of concurrent VACUUM being * on the same file. */ - AppendOptimizedDropDeadSegments(onerel, dead_segs); + AppendOptimizedDropDeadSegments(onerel, dead_segs, vacrelstats); } else { @@ -443,7 +506,7 @@ ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, Buffe * for updating statistics. */ if ((options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE) - vacuum_appendonly_indexes(onerel, options, dead_segs, bstrategy); + vacuum_appendonly_indexes(onerel, options, dead_segs, bstrategy, vacrelstats); } bms_free(dead_segs); @@ -458,7 +521,7 @@ ao_vacuum_rel_recycle_dead_segments(Relation onerel, VacuumParams *params, Buffe */ static int vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs, - BufferAccessStrategy bstrategy) + BufferAccessStrategy bstrategy, AOVacuumRelStats *vacrelstats) { int i; Relation *Irel; @@ -469,6 +532,8 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs if (Debug_appendonly_print_compaction) elog(LOG, "Vacuum indexes for append-only relation %s", RelationGetRelationName(aoRelation)); + pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, + PROGRESS_VACUUM_PHASE_VACUUM_INDEX); /* Now open all indexes of the relation */ if ((options & VACOPT_FULL)) @@ -500,17 +565,21 @@ vacuum_appendonly_indexes(Relation aoRelation, int options, Bitmapset *dead_segs else { for (i = 0; i < nindexes; i++) - { + { vacuum_appendonly_index(Irel[i], aoRelation, dead_segs, elevel, - bstrategy); + bstrategy, + vacrelstats); } } } vac_close_indexes(nindexes, Irel, NoLock); + pgstat_progress_update_param(PROGRESS_VACUUM_PHASE, + ((options & VACUUM_AO_PHASE_MASK) == VACOPT_AO_POST_CLEANUP_PHASE) ? + PROGRESS_VACUUM_PHASE_AO_POST_CLEANUP : PROGRESS_VACUUM_PHASE_AO_PRE_CLEANUP); return nindexes; } @@ -525,10 +594,11 @@ vacuum_appendonly_index(Relation indexRelation, Relation aoRelation, Bitmapset *dead_segs, int elevel, - BufferAccessStrategy bstrategy) + BufferAccessStrategy bstrategy, + AOVacuumRelStats *vacrelstats) { IndexBulkDeleteResult *stats; - IndexVacuumInfo ivinfo; + IndexVacuumInfo ivinfo = {0}; PGRUsage ru0; Assert(RelationIsValid(indexRelation)); @@ -548,7 +618,10 @@ vacuum_appendonly_index(Relation indexRelation, /* Do bulk deletion */ stats = index_bulk_delete(&ivinfo, NULL, appendonly_tid_reaped, - (void *) dead_segs); + (void *) dead_segs); + vacrelstats->num_index_vacuumed++; + pgstat_progress_update_param(PROGRESS_VACUUM_NUM_INDEX_VACUUMS, + vacrelstats->num_index_vacuumed); SIMPLE_FAULT_INJECTOR("vacuum_ao_after_index_delete"); @@ -619,7 +692,7 @@ appendonly_tid_reaped(ItemPointer itemptr, void *state) static void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, BlockNumber *rel_pages, double *rel_tuples, - bool *relhasindex, BlockNumber *total_file_segs) + double *dead_tuples, bool *relhasindex, BlockNumber *total_file_segs) { FileSegTotals *fstotal; BlockNumber nblocks; @@ -649,8 +722,7 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, num_tuples = (double)fstotal->totaltuples; nblocks = (uint32)RelationGetNumberOfBlocks(aorel); - GetAppendOnlyEntryAuxOids(aorel->rd_id, - snapshot, + GetAppendOnlyEntryAuxOids(aorel, NULL, NULL, NULL, &visimaprelid, &visimapidxid); @@ -673,6 +745,7 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, *rel_pages = nblocks; *rel_tuples = num_tuples; + *dead_tuples = hidden_tupcount; *relhasindex = aorel->rd_rel->relhasindex; *total_file_segs = fstotal->totalfilesegs; @@ -682,16 +755,42 @@ vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, int elevel, pfree(fstotal); } +static AOVacuumRelStats * +init_vacrelstats() +{ + AOVacuumRelStats *vacrelstats; + MemoryContext old_context; + + old_context = MemoryContextSwitchTo(TopMemoryContext); + vacrelstats = (AOVacuumRelStats *) palloc0(sizeof(AOVacuumRelStats)); + MemoryContextSwitchTo(old_context); + + return vacrelstats; +} + +static void +cleanup_vacrelstats(AOVacuumRelStats **vacrelstats) +{ + pfree(*vacrelstats); + *vacrelstats = NULL; +} + /* * scan_index() -- scan one index relation to update pg_class statistics. * * We use this when we have no deletions to do. + * + * We used to pass an argument num_tuples with value of table->reltuples to + * ivinfo.num_heap_tuples, now with the new VACUUM strategy, we cannot get + * exact table->reltuples in the calling context. We able to use existing + * value from pg_class for table. Therefore, ivinfo.num_heap_tuples is not an + * accurate value, so we need to set estimated_count to true. */ void scan_index(Relation indrel, Relation aorel, int elevel, BufferAccessStrategy vac_strategy) { IndexBulkDeleteResult *stats; - IndexVacuumInfo ivinfo; + IndexVacuumInfo ivinfo = {0}; PGRUsage ru0; pg_rusage_init(&ru0); diff --git a/src/backend/gpopt/translate/CPartPruneStepsBuilder.cpp b/src/backend/gpopt/translate/CPartPruneStepsBuilder.cpp index 0264a7302f3..23a5a47e270 100644 --- a/src/backend/gpopt/translate/CPartPruneStepsBuilder.cpp +++ b/src/backend/gpopt/translate/CPartPruneStepsBuilder.cpp @@ -82,7 +82,7 @@ CPartPruneStepsBuilder::CreatePartPruneInfoForOneLevel(CDXLNode *filterNode) // partitions that survived static partition pruning; iterate over this list // to populate pinfo->subplan_map, pinfo->relid_map & pinfo->present_parts ULONG part_ptr = 0; - for (int i = 0; i < pinfo->nparts; ++i) + for (ULONG i = 0; (int) i < pinfo->nparts; ++i) { pinfo->subpart_map[i] = -1; if (part_ptr < m_part_indexes->Size() && diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index e8cf0795b68..a4bca224b5c 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -528,7 +528,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %type opt_instead %type opt_unique opt_concurrently opt_verbose opt_full -%type opt_freeze opt_analyze opt_default opt_recheck +%type opt_freeze opt_analyze opt_ao_aux_only opt_default opt_recheck %type opt_dxl %type opt_binary copy_delimiter @@ -842,7 +842,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ /* GPDB-added keywords, in alphabetical order */ %token - ACCOUNT ACTIVE ALLOWED_VALUES + ACCOUNT ACTIVE ALLOWED_VALUES AO_AUX_ONLY CONTAINS COORDINATOR CPUSET CPU_MAX_PERCENT CPU_WEIGHT @@ -983,6 +983,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %nonassoc AGGREGATE %nonassoc ALSO %nonassoc ALTER + %nonassoc AO_AUX_ONLY %nonassoc ASSERTION %nonassoc ASSIGNMENT %nonassoc BACKWARD @@ -14193,7 +14194,7 @@ cluster_index_specification: * *****************************************************************************/ -VacuumStmt: VACUUM opt_full opt_freeze opt_verbose opt_analyze opt_vacuum_relation_list +VacuumStmt: VACUUM opt_full opt_freeze opt_verbose opt_analyze opt_ao_aux_only opt_vacuum_relation_list { VacuumStmt *n = makeNode(VacuumStmt); n->options = NIL; @@ -14209,7 +14210,10 @@ VacuumStmt: VACUUM opt_full opt_freeze opt_verbose opt_analyze opt_vacuum_relati if ($5) n->options = lappend(n->options, makeDefElem("analyze", NULL, @5)); - n->rels = $6; + if ($6) + n->options = lappend(n->options, + makeDefElem("ao_aux_only", NULL, @6)); + n->rels = $7; n->is_vacuumcmd = true; $$ = (Node *)n; } @@ -14373,6 +14377,10 @@ opt_vacuum_relation_list: | /*EMPTY*/ { $$ = NIL; } ; +/*GPDB: only vacuum supporting heap tables of given AO table*/ +opt_ao_aux_only: AO_AUX_ONLY { $$ = true; } + | /*EMPTY*/ { $$ = false; } + ; /***************************************************************************** * @@ -20049,7 +20057,8 @@ col_name_keyword: * - thomas 2000-11-28 */ type_func_name_keyword: - AUTHORIZATION + AO_AUX_ONLY + | AUTHORIZATION | BINARY | COLLATION | CONCURRENTLY diff --git a/src/backend/utils/adt/dbsize.c b/src/backend/utils/adt/dbsize.c index 6210966e1e7..2716b653d5f 100644 --- a/src/backend/utils/adt/dbsize.c +++ b/src/backend/utils/adt/dbsize.c @@ -645,7 +645,7 @@ calculate_table_size(Relation rel) if (RelationIsAppendOptimized(rel)) { Oid auxRelIds[3]; - GetAppendOnlyEntryAuxOids(rel->rd_id, NULL, &auxRelIds[0], + GetAppendOnlyEntryAuxOids(rel, &auxRelIds[0], &auxRelIds[1], NULL, &auxRelIds[2], NULL); diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index ebec2de5f07..77b08937276 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -315,6 +315,7 @@ static void RelationParseRelOptions(Relation relation, HeapTuple tuple); static void RelationBuildTupleDesc(Relation relation); static Relation RelationBuildDesc(Oid targetRelId, bool insertIt); static void RelationInitPhysicalAddr(Relation relation); +static void RelationInitAppendOnlyInfo(Relation relation); static void load_critical_index(Oid indexoid, Oid heapoid); static TupleDesc GetPgClassDescriptor(void); static TupleDesc GetPgIndexDescriptor(void); @@ -1260,6 +1261,12 @@ RelationBuildDesc(Oid targetRelId, bool insertIt) break; } + /* + * If it's an append-only table, get information from pg_appendonly. + */ + if (RelationIsAppendOptimized(relation)) + RelationInitAppendOnlyInfo(relation); + /* extract reloptions if any */ RelationParseRelOptions(relation, pg_class_tuple); @@ -2081,6 +2088,43 @@ formrdesc(const char *relationName, Oid relationReltype, relation->rd_isvalid = true; } +static void +RelationInitAppendOnlyInfo(Relation relation) +{ + Relation pg_appendonly_rel; + HeapTuple tuple; + MemoryContext oldcontext; + SysScanDesc scan; + ScanKeyData skey; + + /* + * Check the pg_appendonly relation to be certain the ao table + * is there. + */ + pg_appendonly_rel = heap_open(AppendOnlyRelationId, AccessShareLock); + + ScanKeyInit(&skey, + Anum_pg_appendonly_relid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationGetRelid(relation))); + scan = systable_beginscan(pg_appendonly_rel, AppendOnlyRelidIndexId, true, + NULL, 1, &skey); + + tuple = systable_getnext(scan); + if (!tuple) + elog(ERROR, "could not find pg_appendonly tuple for relation \"%s\"", + RelationGetRelationName(relation)); + + /* + * Make a copy of the pg_appendonly entry for the table. + */ + oldcontext = MemoryContextSwitchTo(CacheMemoryContext); + relation->rd_aotuple = heap_copytuple(tuple); + relation->rd_appendonly = (Form_pg_appendonly) GETSTRUCT(relation->rd_aotuple); + MemoryContextSwitchTo(oldcontext); + systable_endscan(scan); + heap_close(pg_appendonly_rel, AccessShareLock); +} /* ---------------------------------------------------------------- * Relation Descriptor Lookup Interface @@ -2516,6 +2560,8 @@ RelationDestroyRelation(Relation relation, bool remember_tupdesc) pfree(relation->rd_options); if (relation->rd_indextuple) pfree(relation->rd_indextuple); + if (relation->rd_aotuple) + pfree(relation->rd_aotuple); if (relation->rd_amcache) pfree(relation->rd_amcache); if (relation->rd_fdwroutine) diff --git a/src/include/access/aocs_compaction.h b/src/include/access/aocs_compaction.h index ac5f387ff6b..04ccda2cb56 100644 --- a/src/include/access/aocs_compaction.h +++ b/src/include/access/aocs_compaction.h @@ -17,12 +17,14 @@ #include "utils/rel.h" struct AOCSVPInfo; -extern void AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, struct AOCSVPInfo *vpinfo); -extern void AOCSCompaction_DropSegmentFile(Relation aorel, int segno); +struct AOVacuumRelStats; +extern void AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, struct AOCSVPInfo *vpinfo, struct AOVacuumRelStats *vacrelstats); +extern void AOCSCompaction_DropSegmentFile(Relation aorel, int segno, struct AOVacuumRelStats *vacrelstats); extern void AOCSCompact(Relation aorel, int compaction_segno, int *insert_segno, bool isFull, - List *avoid_segnos); + List *avoid_segnos, + struct AOVacuumRelStats *vacrelstats); #endif diff --git a/src/include/access/aomd.h b/src/include/access/aomd.h index 717cb70f79a..ed3e207adf0 100644 --- a/src/include/access/aomd.h +++ b/src/include/access/aomd.h @@ -19,6 +19,7 @@ #include "storage/fd.h" #include "utils/rel.h" +struct AOVacuumRelStats; extern int AOSegmentFilePathNameLen(Relation rel); extern void FormatAOSegmentFileName( @@ -45,7 +46,8 @@ extern void TruncateAOSegmentFile(File fd, Relation rel, int32 segmentFileNum, - int64 offset); + int64 offset, + struct AOVacuumRelStats *vacrelstats); extern void ao_truncate_one_rel(Relation rel); @@ -67,4 +69,5 @@ extern void ao_foreach_extent_file(ao_extent_callback callback, void *ctx); extern void register_dirty_segment_ao(RelFileNode rnode, int segno, File vfd); +extern uint64 ao_rel_get_physical_size(Relation aorel); #endif /* AOMD_H */ diff --git a/src/include/access/appendonly_compaction.h b/src/include/access/appendonly_compaction.h index ce44d2a4979..44aa78a39fb 100644 --- a/src/include/access/appendonly_compaction.h +++ b/src/include/access/appendonly_compaction.h @@ -21,13 +21,26 @@ #define APPENDONLY_COMPACTION_SEGNO_INVALID (-1) +/* + * Stats for progress reporting. + * This is AO/AOCO counterpart of LVRelStats for Heap. It lives throughout + * all phases of AO VACUUM. + */ +typedef struct AOVacuumRelStats +{ + int nbytes_truncated; /* current # of bytes truncated from segment file */ + int num_dead_tuples; /* current # of dead tuples */ + int num_index_vacuumed; /* current # of indexes been vacuumed */ +} AOVacuumRelStats; + extern Bitmapset *AppendOptimizedCollectDeadSegments(Relation aorel); -extern void AppendOptimizedDropDeadSegments(Relation aorel, Bitmapset *segnos); +extern void AppendOptimizedDropDeadSegments(Relation aorel, Bitmapset *segnos, AOVacuumRelStats *vacrelstats); extern void AppendOnlyCompact(Relation aorel, int compaction_segno, int *insert_segno, bool isFull, - List *avoid_segnos); + List *avoid_segnos, + AOVacuumRelStats *vacrelstats); extern bool AppendOnlyCompaction_ShouldCompact( Relation aoRelation, int segno, @@ -35,6 +48,6 @@ extern bool AppendOnlyCompaction_ShouldCompact( bool isFull, Snapshot appendOnlyMetaDataSnapshot); extern void AppendOnlyThrowAwayTuple(Relation rel, TupleTableSlot *slot, MemTupleBinding *mt_bind); -extern void AppendOptimizedTruncateToEOF(Relation aorel); +extern void AppendOptimizedTruncateToEOF(Relation aorel, AOVacuumRelStats *vacrelstats); #endif diff --git a/src/include/catalog/pg_appendonly.h b/src/include/catalog/pg_appendonly.h index fe3a17d0f91..a03fd6542d6 100644 --- a/src/include/catalog/pg_appendonly.h +++ b/src/include/catalog/pg_appendonly.h @@ -159,8 +159,7 @@ GetAppendOnlyEntryAttributes(Oid relid, * not NULL. */ void -GetAppendOnlyEntryAuxOids(Oid relid, - Snapshot appendOnlyMetaDataSnapshot, +GetAppendOnlyEntryAuxOids(Relation rel, Oid *segrelid, Oid *blkdirrelid, Oid *blkdiridxid, @@ -168,7 +167,7 @@ GetAppendOnlyEntryAuxOids(Oid relid, Oid *visimapidxid); void -GetAppendOnlyEntry(Oid relid, Form_pg_appendonly aoEntry); +GetAppendOnlyEntry(Relation rel, Form_pg_appendonly aoEntry); /* * Update the segrelid and/or blkdirrelid if the input new values * are valid OIDs. diff --git a/src/include/commands/progress.h b/src/include/commands/progress.h index d7bf16368bd..990f8b8e6f4 100644 --- a/src/include/commands/progress.h +++ b/src/include/commands/progress.h @@ -33,6 +33,10 @@ #define PROGRESS_VACUUM_PHASE_INDEX_CLEANUP 4 #define PROGRESS_VACUUM_PHASE_TRUNCATE 5 #define PROGRESS_VACUUM_PHASE_FINAL_CLEANUP 6 +/* Phases of vacumm for Greenplum AO/AOCO tables */ +#define PROGRESS_VACUUM_PHASE_AO_PRE_CLEANUP 7 +#define PROGRESS_VACUUM_PHASE_AO_COMPACT 8 +#define PROGRESS_VACUUM_PHASE_AO_POST_CLEANUP 9 /* Progress parameters for analyze */ #define PROGRESS_ANALYZE_PHASE 0 diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 30d34ccfe5d..048b727e685 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -188,25 +188,28 @@ typedef struct VacAttrStats float4 corrval; /* correlation gathered from segments */ } VacAttrStats; - -/* flag bits for VacuumParams->options */ -#define VACOPT_VACUUM 0x01 /* do VACUUM */ -#define VACOPT_ANALYZE 0x02 /* do ANALYZE */ -#define VACOPT_VERBOSE 0x04 /* print progress info */ -#define VACOPT_FREEZE 0x08 /* FREEZE option */ -#define VACOPT_FULL 0x10 /* FULL (non-concurrent) vacuum */ -#define VACOPT_SKIP_LOCKED 0x20 /* skip if cannot get lock */ -#define VACOPT_PROCESS_TOAST 0x40 /* process the TOAST table, if any */ -#define VACOPT_DISABLE_PAGE_SKIPPING 0x80 /* don't skip any pages */ -/* GPDB_14_MERGE_FIXME: is flags bits suitable with upstream? */ -/* Extra GPDB options */ -#define VACOPT_ROOTONLY 0x400 -#define VACOPT_FULLSCAN 0x800 -/* AO vacuum phases. Mutually exclusive */ -#define VACOPT_AO_PRE_CLEANUP_PHASE 0x1000 -#define VACOPT_AO_COMPACT_PHASE 0x2000 -#define VACOPT_AO_POST_CLEANUP_PHASE 0x4000 -#define VACOPT_UPDATE_DATFROZENXID 0x8000 +typedef enum VacuumOption +{ + VACOPT_VACUUM = 1 << 0, /* do VACUUM */ + VACOPT_ANALYZE = 1 << 1, /* do ANALYZE */ + VACOPT_VERBOSE = 1 << 2, /* print progress info */ + VACOPT_FREEZE = 1 << 3, /* FREEZE option */ + VACOPT_FULL = 1 << 4, /* FULL (non-concurrent) vacuum */ + VACOPT_SKIP_LOCKED = 1 << 5, /* skip if cannot get lock */ + VACOPT_PROCESS_TOAST = 1 << 6, /* process the TOAST table, if any */ + VACOPT_DISABLE_PAGE_SKIPPING = 1 << 7, /* don't skip any pages */ + + /* Extra GPDB options */ + VACOPT_AO_AUX_ONLY = 1 << 8, + VACOPT_ROOTONLY = 1 << 10, + VACOPT_FULLSCAN = 1 << 11, + + /* AO vacuum phases. Mutually exclusive */ + VACOPT_AO_PRE_CLEANUP_PHASE = 1 << 12, + VACOPT_AO_COMPACT_PHASE = 1 << 13, + VACOPT_AO_POST_CLEANUP_PHASE = 1 << 14, + VACOPT_UPDATE_DATFROZENXID = 1 << 15 +} VacuumOption; #define VACUUM_AO_PHASE_MASK (VACOPT_AO_PRE_CLEANUP_PHASE | \ VACOPT_AO_COMPACT_PHASE | \ diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index a8514b189d7..355ec9511c2 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -45,6 +45,7 @@ PG_KEYWORD("analyse", ANALYSE, RESERVED_KEYWORD, BARE_LABEL) /* British spellin PG_KEYWORD("analyze", ANALYZE, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("and", AND, RESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("any", ANY, RESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("ao_aux_only", AO_AUX_ONLY, TYPE_FUNC_NAME_KEYWORD) /* GPDB */ PG_KEYWORD("array", ARRAY, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("as", AS, RESERVED_KEYWORD, AS_LABEL) PG_KEYWORD("asc", ASC, RESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 3feed516a14..4697ec1fb4e 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -233,6 +233,12 @@ typedef struct RelationData */ void *rd_amcache; /* available for use by index/table AM */ + /* + * AO table support info (used only for AO and AOCS relations) + */ + Form_pg_appendonly rd_appendonly; + struct HeapTupleData *rd_aotuple; /* all of pg_appendonly tuple */ + /* * foreign-table support * diff --git a/src/test/isolation/expected/vacuum-ao-concurrent-drop.out b/src/test/isolation/expected/vacuum-ao-concurrent-drop.out new file mode 100644 index 00000000000..9657938f58e --- /dev/null +++ b/src/test/isolation/expected/vacuum-ao-concurrent-drop.out @@ -0,0 +1,65 @@ +Parsed test spec with 2 sessions + +starting permutation: begin_and_drop vac_specified commit +step begin_and_drop: + BEGIN; + DROP TABLE ao_vac_conc; + +step vac_specified: VACUUM AO_AUX_ONLY ao_vac_conc; +step commit: + COMMIT; + +step vac_specified: <... completed> +ERROR: relation "ao_vac_conc" does not exist + +starting permutation: begin_and_drop vac_all_aux_tables commit +step begin_and_drop: + BEGIN; + DROP TABLE ao_vac_conc; + +step vac_all_aux_tables: VACUUM AO_AUX_ONLY; +step commit: + COMMIT; + +step vac_all_aux_tables: <... completed> + +starting permutation: begin_and_drop vac_all_tables commit +step begin_and_drop: + BEGIN; + DROP TABLE ao_vac_conc; + +step vac_all_tables: VACUUM; +step commit: + COMMIT; + +step vac_all_tables: <... completed> + +starting permutation: begin vac_specified drop_and_commit +step begin: + BEGIN; + +step vac_specified: VACUUM AO_AUX_ONLY ao_vac_conc; +step drop_and_commit: + DROP TABLE ao_vac_conc; + COMMIT; + + +starting permutation: begin vac_all_aux_tables drop_and_commit +step begin: + BEGIN; + +step vac_all_aux_tables: VACUUM AO_AUX_ONLY; +step drop_and_commit: + DROP TABLE ao_vac_conc; + COMMIT; + + +starting permutation: begin vac_all_tables drop_and_commit +step begin: + BEGIN; + +step vac_all_tables: VACUUM; +step drop_and_commit: + DROP TABLE ao_vac_conc; + COMMIT; + diff --git a/src/test/isolation/isolation_schedule b/src/test/isolation/isolation_schedule index 41dbce94000..03241eede81 100644 --- a/src/test/isolation/isolation_schedule +++ b/src/test/isolation/isolation_schedule @@ -178,3 +178,4 @@ test: heap-repeatable-read-vacuum test: ao-repeatable-read-vacuum test: ao-insert-eof create_index_hot udf-insert-deadlock heap-repeatable-read ao-repeatable-read +test: vacuum-ao-concurrent-drop diff --git a/src/test/isolation/specs/vacuum-ao-concurrent-drop.spec b/src/test/isolation/specs/vacuum-ao-concurrent-drop.spec new file mode 100644 index 00000000000..b97c213e905 --- /dev/null +++ b/src/test/isolation/specs/vacuum-ao-concurrent-drop.spec @@ -0,0 +1,45 @@ +# Test for lack of unexpected errors when an AO table is dropped while its +# auxiliary is being vacuumed. +# +setup +{ + CREATE TABLE ao_vac_conc (a INT) WITH (appendonly=true); + INSERT INTO ao_vac_conc SELECT j FROM generate_series(1, 100)j; +} + +teardown +{ + DROP TABLE IF EXISTS ao_vac_conc; +} + +session s1 +step begin +{ + BEGIN; +} +step drop_and_commit +{ + DROP TABLE ao_vac_conc; + COMMIT; +} +step begin_and_drop +{ + BEGIN; + DROP TABLE ao_vac_conc; +} +step commit +{ + COMMIT; +} + +session s2 +step vac_specified { VACUUM AO_AUX_ONLY ao_vac_conc; } +step vac_all_aux_tables { VACUUM AO_AUX_ONLY; } +step vac_all_tables { VACUUM; } + +permutation begin_and_drop vac_specified commit +permutation begin_and_drop vac_all_aux_tables commit +permutation begin_and_drop vac_all_tables commit +permutation begin vac_specified drop_and_commit +permutation begin vac_all_aux_tables drop_and_commit +permutation begin vac_all_tables drop_and_commit diff --git a/src/test/isolation2/expected/setup.out b/src/test/isolation2/expected/setup.out index 193a5d76361..5184e6d7cbd 100644 --- a/src/test/isolation2/expected/setup.out +++ b/src/test/isolation2/expected/setup.out @@ -133,3 +133,6 @@ CREATE CREATE OR REPLACE FUNCTION wait_till_master_shutsdown() RETURNS void AS $$ DECLARE i int; /* in func */ BEGIN i := 0; /* in func */ while i < 120 loop i := i + 1; /* in func */ PERFORM pg_sleep(.5); /* in func */ end loop; /* in func */ END; /* in func */ $$ LANGUAGE plpgsql; CREATE +-- Helper function that ensures stats collector receives stat from the latest operation. +create or replace function wait_until_dead_tup_change_to(relid oid, stat_val_expected bigint) returns text as $$ declare stat_val int; /* in func */ i int; /* in func */ begin i := 0; /* in func */ while i < 1200 loop select pg_stat_get_dead_tuples(relid) into stat_val; /* in func */ if stat_val = stat_val_expected then /* in func */ return 'OK'; /* in func */ end if; /* in func */ perform pg_sleep(0.1); /* in func */ perform pg_stat_clear_snapshot(); /* in func */ i := i + 1; /* in func */ end loop; /* in func */ return 'Fail'; /* in func */ end; /* in func */ $$ language plpgsql; +CREATE diff --git a/src/test/isolation2/expected/vacuum_progress_column.out b/src/test/isolation2/expected/vacuum_progress_column.out new file mode 100644 index 00000000000..662a5e05a89 --- /dev/null +++ b/src/test/isolation2/expected/vacuum_progress_column.out @@ -0,0 +1,273 @@ +-- @Description Test to ensure we correctly report progress in +-- pg_stat_progress_create_vacuum for append-optimized tables. + +set default_table_access_method=ao_column; +SET + +-- Setup the append-optimized table to be vacuumed +DROP TABLE IF EXISTS vacuum_progress_ao_column; +DROP +CREATE TABLE vacuum_progress_ao_column(i int, j int); +CREATE + +-- Add two indexes to be vacuumed as well +CREATE INDEX on vacuum_progress_ao_column(i); +CREATE +CREATE INDEX on vacuum_progress_ao_column(j); +CREATE + +-- Insert all tuples to seg1 from two current sessions so that data are stored +-- in two segment files. +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +1: INSERT INTO vacuum_progress_ao_column SELECT 0, i FROM generate_series(1, 100000) i; +INSERT 100000 +2: INSERT INTO vacuum_progress_ao_column SELECT 0, i FROM generate_series(1, 100000) i; +INSERT 100000 +-- Commit so that the logical EOF of segno 2 is non-zero. +2: COMMIT; +COMMIT +2: BEGIN; +BEGIN +2: INSERT INTO vacuum_progress_ao_column SELECT 0, i FROM generate_series(1, 100000) i; +INSERT 100000 +-- Abort so that segno 2 has dead tuples after its logical EOF +2: ABORT; +ABORT +2q: ... +-- Abort so that segno 1 has logical EOF = 0. +1: ABORT; +ABORT + +-- Also delete half of the tuples evenly before the EOF of segno 2. +DELETE FROM vacuum_progress_ao_column where j % 2 = 0; +DELETE 50000 + +-- Lookup pg_class and collected stats view before VACUUM +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column'; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 0 | 0 | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column'; + n_live_tup | n_dead_tup | last_vacuum | vacuum_count +------------+------------+-------------+-------------- + 100000 | 200000 | | 0 +(1 row) + +-- Perform VACUUM and observe the progress + +-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) + +1: set Debug_appendonly_print_compaction to on; +SET +1&: VACUUM vacuum_progress_ao_column; +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in pre_cleanup phase and some blocks should've been vacuumed by now +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_column | append-optimized pre-cleanup | 74 | 0 | 25 | 0 | 100000 | 0 +(1 row) + +-- Resume execution and suspend again in the middle of compact phase +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_column | append-optimized compact | 74 | 2 | 49 | 0 | 100000 | 199 +(1 row) + +-- Resume execution and suspend again after compacting all segfiles +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- After compacting all segfiles we expect 50000 dead tuples +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_column | append-optimized compact | 74 | 25 | 49 | 0 | 100000 | 50000 +(1 row) + +-- Resume execution and entering post_cleaup phase, suspend at the end of it. +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_column | append-optimized post-cleanup | 74 | 25 | 49 | 0 | 100000 | 50000 +(1 row) +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +1<: <... completed> +VACUUM + +-- pg_class and collected stats view should be updated after the 1st VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0); + wait_until_dead_tup_change_to +------------------------------- + OK +(1 row) +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column'; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 37 | 50000 | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column'; + n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count +------------+------------+-----------------+-------------- + 50000 | 0 | t | 1 +(1 row) + + +-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +1&: VACUUM vacuum_progress_ao_column; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------------------------+-------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_column | vacuuming indexes | 37 | 0 | 0 | 1 | 50000 | 0 +(1 row) + +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_column | append-optimized pre-cleanup | 37 | 0 | 13 | 2 | 50000 | 0 +(1 row) + +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +1<: <... completed> +VACUUM + +-- Vacuum has finished, nothing should show up in the view. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- +(0 rows) + +-- pg_class and collected stats view should be updated after the 2nd VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0); + wait_until_dead_tup_change_to +------------------------------- + OK +(1 row) +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column'; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 13 | 50000 | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column'; + n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count +------------+------------+-----------------+-------------- + 50000 | 0 | t | 2 +(1 row) + +-- Cleanup +SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: + Success: + Success: + Success: + Success: + Success: +(8 rows) +reset Debug_appendonly_print_compaction; +RESET +reset default_table_access_method; +RESET diff --git a/src/test/isolation2/expected/vacuum_progress_row.out b/src/test/isolation2/expected/vacuum_progress_row.out new file mode 100644 index 00000000000..dee1dc6dcdc --- /dev/null +++ b/src/test/isolation2/expected/vacuum_progress_row.out @@ -0,0 +1,272 @@ +-- @Description Test to ensure we correctly report progress in +-- pg_stat_progress_create_vacuum for append-optimized tables. + +set default_table_access_method=ao_row; +SET + +-- Setup the append-optimized table to be vacuumed +DROP TABLE IF EXISTS vacuum_progress_ao_row; +DROP +CREATE TABLE vacuum_progress_ao_row(i int, j int); +CREATE + +-- Add two indexes to be vacuumed as well +CREATE INDEX on vacuum_progress_ao_row(i); +CREATE +CREATE INDEX on vacuum_progress_ao_row(j); +CREATE + +-- Insert all tuples to seg1 from two current sessions so that data are stored +-- in two segment files. +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +1: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +INSERT 100000 +2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +INSERT 100000 +-- Commit so that the logical EOF of segno 2 is non-zero. +2: COMMIT; +COMMIT +2: BEGIN; +BEGIN +2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +INSERT 100000 +-- Abort so that segno 2 has dead tuples after its logical EOF +2: ABORT; +ABORT +2q: ... +-- Abort so that segno 1 has logical EOF = 0. +1: ABORT; +ABORT + +-- Also delete half of the tuples evenly before the EOF of segno 2. +DELETE FROM vacuum_progress_ao_row where j % 2 = 0; +DELETE 50000 + +-- Lookup pg_class and collected stats view before VACUUM +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 0 | 0 | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | last_vacuum | vacuum_count +------------+------------+-------------+-------------- + 100000 | 200000 | | 0 +(1 row) + +-- Perform VACUUM and observe the progress + +-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) + +1: set Debug_appendonly_print_compaction to on; +SET +1&: VACUUM vacuum_progress_ao_row; +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in pre_cleanup phase and some blocks should've been vacuumed by now +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 165 | 0 | 110 | 0 | 100000 | 0 +(1 row) + +-- Resume execution and suspend again in the middle of compact phase +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized compact | 165 | 0 | 110 | 0 | 100000 | 199 +(1 row) + +-- Resume execution and suspend again after compacting all segfiles +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- After compacting all segfiles we expect 50000 dead tuples +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+--------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized compact | 165 | 55 | 110 | 0 | 100000 | 50000 +(1 row) + +-- Resume execution and entering post_cleaup phase, suspend at the end of it. +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized post-cleanup | 165 | 55 | 110 | 0 | 100000 | 50000 +(1 row) +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +1<: <... completed> +VACUUM + +-- pg_class and collected stats view should be updated after the 1st VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); + wait_until_dead_tup_change_to +------------------------------- + OK +(1 row) +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 83 | 50000 | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count +------------+------------+-----------------+-------------- + 50000 | 0 | t | 1 +(1 row) + +-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +1&: VACUUM vacuum_progress_ao_row; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+-------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | vacuuming indexes | 83 | 0 | 0 | 1 | 50000 | 0 +(1 row) + +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) +-- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +------------------------+------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- + vacuum_progress_ao_row | append-optimized pre-cleanup | 83 | 0 | 55 | 2 | 50000 | 0 +(1 row) + +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + gp_inject_fault +----------------- + Success: +(1 row) +1<: <... completed> +VACUUM + +-- Vacuum has finished, nothing should show up in the progress view. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples +---------+-------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- +(0 rows) + +-- pg_class and collected stats view should be updated after the 2nd VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); + wait_until_dead_tup_change_to +------------------------------- + OK +(1 row) +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; + relpages | reltuples | relallvisible +----------+-----------+--------------- + 28 | 50000 | 0 +(1 row) +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; + n_live_tup | n_dead_tup | has_last_vacuum | vacuum_count +------------+------------+-----------------+-------------- + 50000 | 0 | t | 2 +(1 row) + +-- Cleanup +SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; + gp_inject_fault_infinite +-------------------------- + Success: + Success: + Success: + Success: + Success: + Success: + Success: + Success: +(8 rows) +reset Debug_appendonly_print_compaction; +RESET +reset default_table_access_method; +RESET diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 15f678bfe9d..842edfccfdd 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -228,6 +228,9 @@ test: vacuum_after_vacuum_skip_drop_column test: workfile_mgr_test test: pg_basebackup test: pg_basebackup_with_tablespaces +test: pg_basebackup_large_database_oid +test: vacuum_progress_row +test: vacuum_progress_column test: enable_autovacuum test: segwalrep/die_commit_pending_replication diff --git a/src/test/isolation2/sql/setup.sql b/src/test/isolation2/sql/setup.sql index a3afc9487ea..fa81e3685cf 100644 --- a/src/test/isolation2/sql/setup.sql +++ b/src/test/isolation2/sql/setup.sql @@ -390,4 +390,24 @@ $$ LANGUAGE plpgsql; -- start_ignore set statement_timeout='180s'; --- end_ignore \ No newline at end of file +-- end_ignore +-- Helper function that ensures stats collector receives stat from the latest operation. +create or replace function wait_until_dead_tup_change_to(relid oid, stat_val_expected bigint) + returns text as $$ +declare + stat_val int; /* in func */ + i int; /* in func */ +begin + i := 0; /* in func */ + while i < 1200 loop + select pg_stat_get_dead_tuples(relid) into stat_val; /* in func */ + if stat_val = stat_val_expected then /* in func */ + return 'OK'; /* in func */ + end if; /* in func */ + perform pg_sleep(0.1); /* in func */ + perform pg_stat_clear_snapshot(); /* in func */ + i := i + 1; /* in func */ + end loop; /* in func */ + return 'Fail'; /* in func */ +end; /* in func */ +$$ language plpgsql; diff --git a/src/test/isolation2/sql/vacuum_progress_column.sql b/src/test/isolation2/sql/vacuum_progress_column.sql new file mode 100644 index 00000000000..f13f75dc427 --- /dev/null +++ b/src/test/isolation2/sql/vacuum_progress_column.sql @@ -0,0 +1,107 @@ +-- @Description Test to ensure we correctly report progress in +-- pg_stat_progress_create_vacuum for append-optimized tables. + +set default_table_access_method=ao_column; + +-- Setup the append-optimized table to be vacuumed +DROP TABLE IF EXISTS vacuum_progress_ao_column; +CREATE TABLE vacuum_progress_ao_column(i int, j int); + +-- Add two indexes to be vacuumed as well +CREATE INDEX on vacuum_progress_ao_column(i); +CREATE INDEX on vacuum_progress_ao_column(j); + +-- Insert all tuples to seg1 from two current sessions so that data are stored +-- in two segment files. +1: BEGIN; +2: BEGIN; +1: INSERT INTO vacuum_progress_ao_column SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_column SELECT 0, i FROM generate_series(1, 100000) i; +-- Commit so that the logical EOF of segno 2 is non-zero. +2: COMMIT; +2: BEGIN; +2: INSERT INTO vacuum_progress_ao_column SELECT 0, i FROM generate_series(1, 100000) i; +-- Abort so that segno 2 has dead tuples after its logical EOF +2: ABORT; +2q: +-- Abort so that segno 1 has logical EOF = 0. +1: ABORT; + +-- Also delete half of the tuples evenly before the EOF of segno 2. +DELETE FROM vacuum_progress_ao_column where j % 2 = 0; + +-- Lookup pg_class and collected stats view before VACUUM +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column'; +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column'; + +-- Perform VACUUM and observe the progress + +-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + +1: set Debug_appendonly_print_compaction to on; +1&: VACUUM vacuum_progress_ao_column; +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in pre_cleanup phase and some blocks should've been vacuumed by now +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and suspend again in the middle of compact phase +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and suspend again after compacting all segfiles +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- After compacting all segfiles we expect 50000 dead tuples +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and entering post_cleaup phase, suspend at the end of it. +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +1<: + +-- pg_class and collected stats view should be updated after the 1st VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0); +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column'; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column'; + + +-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +1&: VACUUM vacuum_progress_ao_column; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +1<: + +-- Vacuum has finished, nothing should show up in the view. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- pg_class and collected stats view should be updated after the 2nd VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_column'::regclass::oid, 0); +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_column'; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_column'; + +-- Cleanup +SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; +reset Debug_appendonly_print_compaction; +reset default_table_access_method; diff --git a/src/test/isolation2/sql/vacuum_progress_row.sql b/src/test/isolation2/sql/vacuum_progress_row.sql new file mode 100644 index 00000000000..fef4afe78fc --- /dev/null +++ b/src/test/isolation2/sql/vacuum_progress_row.sql @@ -0,0 +1,106 @@ +-- @Description Test to ensure we correctly report progress in +-- pg_stat_progress_create_vacuum for append-optimized tables. + +set default_table_access_method=ao_row; + +-- Setup the append-optimized table to be vacuumed +DROP TABLE IF EXISTS vacuum_progress_ao_row; +CREATE TABLE vacuum_progress_ao_row(i int, j int); + +-- Add two indexes to be vacuumed as well +CREATE INDEX on vacuum_progress_ao_row(i); +CREATE INDEX on vacuum_progress_ao_row(j); + +-- Insert all tuples to seg1 from two current sessions so that data are stored +-- in two segment files. +1: BEGIN; +2: BEGIN; +1: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +-- Commit so that the logical EOF of segno 2 is non-zero. +2: COMMIT; +2: BEGIN; +2: INSERT INTO vacuum_progress_ao_row SELECT 0, i FROM generate_series(1, 100000) i; +-- Abort so that segno 2 has dead tuples after its logical EOF +2: ABORT; +2q: +-- Abort so that segno 1 has logical EOF = 0. +1: ABORT; + +-- Also delete half of the tuples evenly before the EOF of segno 2. +DELETE FROM vacuum_progress_ao_row where j % 2 = 0; + +-- Lookup pg_class and collected stats view before VACUUM +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; + +-- Perform VACUUM and observe the progress + +-- Suspend execution at pre-cleanup phase after truncating both segfiles to their logical EOF. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', '', '', '', 2, 2, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; + +1: set Debug_appendonly_print_compaction to on; +1&: VACUUM vacuum_progress_ao_row; +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 2, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in pre_cleanup phase and some blocks should've been vacuumed by now +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and suspend again in the middle of compact phase +SELECT gp_inject_fault('appendonly_insert', 'suspend', '', '', '', 200, 200, 0, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('appendonly_insert', 200, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in compact phase. num_dead_tuples should increase as we move and count tuples, one by one. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and suspend again after compacting all segfiles +SELECT gp_inject_fault('vacuum_ao_after_compact', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('appendonly_insert', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_compact', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- After compacting all segfiles we expect 50000 dead tuples +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and entering post_cleaup phase, suspend at the end of it. +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('vacuum_ao_post_cleanup_end', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We should have skipped recycling the awaiting drop segment because the segment was still visible to the SELECT gp_wait_until_triggered_fault query. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; +SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +1<: + +-- pg_class and collected stats view should be updated after the 1st VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; + +-- Perform VACUUM again to recycle the remaining awaiting drop segment marked by the previous run. +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +1&: VACUUM vacuum_progress_ao_row; +-- Resume execution and entering pre_cleanup phase, suspend at vacuuming indexes. +SELECT gp_inject_fault('vacuum_ao_after_compact', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('vacuum_ao_after_index_delete', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in vacuuming indexes phase (part of ao pre_cleanup phase), index_vacuum_count should increase to 1. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- Resume execution and moving on to truncate segments that were marked as AWAITING_DROP, there should be only 1. +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'suspend', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_inject_fault('vacuum_ao_after_index_delete', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +SELECT gp_wait_until_triggered_fault('appendonly_after_truncate_segment_file', 1, dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +-- We are in post_cleanup phase and should have truncated the old segfile. Both indexes should be vacuumed by now, and heap_blks_vacuumed should also increased +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +SELECT gp_inject_fault('appendonly_after_truncate_segment_file', 'reset', dbid) FROM gp_segment_configuration WHERE content = 1 AND role = 'p'; +1<: + +-- Vacuum has finished, nothing should show up in the progress view. +1U: select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from pg_stat_progress_vacuum; + +-- pg_class and collected stats view should be updated after the 2nd VACUUM +1U: SELECT wait_until_dead_tup_change_to('vacuum_progress_ao_row'::regclass::oid, 0); +SELECT relpages, reltuples, relallvisible FROM pg_class where relname = 'vacuum_progress_ao_row'; +SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuum_count FROM pg_stat_all_tables WHERE relname = 'vacuum_progress_ao_row'; + +-- Cleanup +SELECT gp_inject_fault_infinite('all', 'reset', dbid) FROM gp_segment_configuration; +reset Debug_appendonly_print_compaction; +reset default_table_access_method; diff --git a/src/test/regress/expected/brin_ao.out b/src/test/regress/expected/brin_ao.out index b0d4b234a64..7f4b6e592bb 100644 --- a/src/test/regress/expected/brin_ao.out +++ b/src/test/regress/expected/brin_ao.out @@ -466,15 +466,9 @@ CONTEXT: SQL function "brin_summarize_new_values" statement 1 SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index ERROR: "tenk1_unique1" is not a BRIN index CONTEXT: SQL function "brin_summarize_new_values" statement 1 --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- brin_summarize_new_values() will not always be accurate. So ignore the check to --- coordinate with the new behavior. --- start_ignore SELECT brin_summarize_new_values('brinaoidx'); -- ok, no change expected brin_summarize_new_values --------------------------- 0 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/brin_ao_optimizer.out b/src/test/regress/expected/brin_ao_optimizer.out index 7038d21493b..02caa9f2c83 100644 --- a/src/test/regress/expected/brin_ao_optimizer.out +++ b/src/test/regress/expected/brin_ao_optimizer.out @@ -489,15 +489,9 @@ CONTEXT: SQL function "brin_summarize_new_values" statement 1 SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index ERROR: "tenk1_unique1" is not a BRIN index CONTEXT: SQL function "brin_summarize_new_values" statement 1 --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- brin_summarize_new_values() will not always be accurate. So ignore the check to --- coordinate with the new behavior. --- start_ignore SELECT brin_summarize_new_values('brinaoidx'); -- ok, no change expected brin_summarize_new_values --------------------------- 0 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/brin_aocs.out b/src/test/regress/expected/brin_aocs.out index 69555f8297c..0fa29bb2581 100644 --- a/src/test/regress/expected/brin_aocs.out +++ b/src/test/regress/expected/brin_aocs.out @@ -466,15 +466,9 @@ CONTEXT: SQL function "brin_summarize_new_values" statement 1 SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index ERROR: "tenk1_unique1" is not a BRIN index CONTEXT: SQL function "brin_summarize_new_values" statement 1 --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- brin_summarize_new_values() will not always be accurate. So ignore the check to --- coordinate with the new behavior. --- start_ignore SELECT brin_summarize_new_values('brinaocsidx'); -- ok, no change expected brin_summarize_new_values --------------------------- 0 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/brin_aocs_optimizer.out b/src/test/regress/expected/brin_aocs_optimizer.out index 3ba2a58bb2a..5a83c375758 100644 --- a/src/test/regress/expected/brin_aocs_optimizer.out +++ b/src/test/regress/expected/brin_aocs_optimizer.out @@ -489,15 +489,9 @@ CONTEXT: SQL function "brin_summarize_new_values" statement 1 SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index ERROR: "tenk1_unique1" is not a BRIN index CONTEXT: SQL function "brin_summarize_new_values" statement 1 --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- brin_summarize_new_values() will not always be accurate. So ignore the check to --- coordinate with the new behavior. --- start_ignore SELECT brin_summarize_new_values('brinaocsidx'); -- ok, no change expected brin_summarize_new_values --------------------------- 0 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/uao_compaction/drop_column.out b/src/test/regress/expected/uao_compaction/drop_column.out index 898b82b172f..6ece6b370d5 100644 --- a/src/test/regress/expected/uao_compaction/drop_column.out +++ b/src/test/regress/expected/uao_compaction/drop_column.out @@ -30,17 +30,17 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_drop_col'; (1 row) -- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold. In this case, the ratio of dead tuples is about 30% +-- which is greater than default value (10%) of the compaction threshold, so compaction +-- should be triggered and reltuples of both table and index should be equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_drop_col_index'; relname | reltuples --------------------+----------- uao_drop_col_index | 7 (1 row) --- end_ignore ALTER TABLE uao_drop_col DROP COLUMN c; SELECT * FROM uao_drop_col; a | b diff --git a/src/test/regress/expected/uao_compaction/full_stats.out b/src/test/regress/expected/uao_compaction/full_stats.out index 1f9d473aa7a..dc5940cc0de 100644 --- a/src/test/regress/expected/uao_compaction/full_stats.out +++ b/src/test/regress/expected/uao_compaction/full_stats.out @@ -26,15 +26,11 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_full_stats'; uao_full_stats | 85 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Compaction is always triggered in VACUUM FULL, expecting +-- reltuples of both table and index are equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_full_stats_index'; relname | reltuples ----------------------+----------- uao_full_stats_index | 85 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/uao_compaction/index_stats.out b/src/test/regress/expected/uao_compaction/index_stats.out index afdad152b1d..0eb52a03c8e 100644 --- a/src/test/regress/expected/uao_compaction/index_stats.out +++ b/src/test/regress/expected/uao_compaction/index_stats.out @@ -29,11 +29,6 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'mytab'; mytab | 2 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore SELECT relname, reltuples FROM pg_class WHERE relname = 'mytab_int_idx1'; relname | reltuples ----------------+----------- diff --git a/src/test/regress/expected/uao_compaction/stats.out b/src/test/regress/expected/uao_compaction/stats.out index a18834d43e3..d8b3c76db19 100644 --- a/src/test/regress/expected/uao_compaction/stats.out +++ b/src/test/regress/expected/uao_compaction/stats.out @@ -20,6 +20,12 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats_index'; (1 row) DELETE FROM uao_stats WHERE a < 16; +-- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold: +-- a) without changing gp_appendonly_compaction_threshold (default is 10), index->reltuples +-- might greater than table->reltuples if no compaction was triggered during VACUUM; VACUUM uao_stats; SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; relname | reltuples @@ -27,15 +33,33 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; uao_stats | 85 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- expect index->reltuples greater than table->reltuples SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats_index'; relname | reltuples -----------------+----------- uao_stats_index | 88 (1 row) --- end_ignore \ No newline at end of file +-- re-setup for next case +TRUNCATE uao_stats; +INSERT INTO uao_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(1, 50) AS i; +INSERT INTO uao_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(51, 100) AS i; +ANALYZE uao_stats; +DELETE FROM uao_stats WHERE a < 16; +-- b) changing gp_appendonly_compaction_threshold to make sure compaction could be triggered, +-- index->reltuples should be equal to table->reltuples +SET gp_appendonly_compaction_threshold = 8; +VACUUM uao_stats; +SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; + relname | reltuples +-----------+----------- + uao_stats | 85 +(1 row) + +-- expect index->reltuples equals to table->reltuples +SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats_index'; + relname | reltuples +-----------------+----------- + uao_stats_index | 85 +(1 row) + diff --git a/src/test/regress/expected/uaocs_compaction/drop_column.out b/src/test/regress/expected/uaocs_compaction/drop_column.out index 5d9579e2e0b..7fcb23a451e 100644 --- a/src/test/regress/expected/uaocs_compaction/drop_column.out +++ b/src/test/regress/expected/uaocs_compaction/drop_column.out @@ -30,17 +30,17 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_drop'; (1 row) -- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold. In this case, the ratio of dead tuples is about 30% +-- which is greater than default value (10%) of the compaction threshold, so compaction +-- should be triggered and reltuples of both table and index should be equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_drop_index'; relname | reltuples ------------------+----------- uaocs_drop_index | 7 (1 row) --- end_ignore ALTER TABLE uaocs_drop DROP COLUMN c; SELECT * FROM uaocs_drop; a | b diff --git a/src/test/regress/expected/uaocs_compaction/full_stats.out b/src/test/regress/expected/uaocs_compaction/full_stats.out index e5d7825ecf1..b6859d1e622 100644 --- a/src/test/regress/expected/uaocs_compaction/full_stats.out +++ b/src/test/regress/expected/uaocs_compaction/full_stats.out @@ -44,15 +44,11 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_full_stats'; uaocs_full_stats | 85 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Compaction is always triggered in VACUUM FULL, expecting +-- reltuples of both table and index are equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_full_stats_index'; relname | reltuples ------------------------+----------- uaocs_full_stats_index | 85 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/uaocs_compaction/index_stats.out b/src/test/regress/expected/uaocs_compaction/index_stats.out index 7ada461449a..11ff27fa10c 100644 --- a/src/test/regress/expected/uaocs_compaction/index_stats.out +++ b/src/test/regress/expected/uaocs_compaction/index_stats.out @@ -34,11 +34,6 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_index_stats'; uaocs_index_stats | 2 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_index_stats_int_idx1'; relname | reltuples ----------------------------+----------- diff --git a/src/test/regress/expected/uaocs_compaction/stats.out b/src/test/regress/expected/uaocs_compaction/stats.out index f643fb4dcc7..62b681ab149 100644 --- a/src/test/regress/expected/uaocs_compaction/stats.out +++ b/src/test/regress/expected/uaocs_compaction/stats.out @@ -20,6 +20,12 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats_index'; (1 row) DELETE FROM uaocs_stats WHERE a < 16; +-- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold: +-- a) without changing gp_appendonly_compaction_threshold (default is 10), index->reltuples +-- might greater than table->reltuples if no compaction was triggered during VACUUM; VACUUM uaocs_stats; SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; relname | reltuples @@ -27,15 +33,33 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; uaocs_stats | 85 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- expect index->reltuples greater than table->reltuples +SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats_index'; + relname | reltuples +-------------------+----------- + uaocs_stats_index | 88 +(1 row) + +-- re-setup for next case +TRUNCATE uaocs_stats; +INSERT INTO uaocs_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(1, 50) AS i; +INSERT INTO uaocs_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(51, 100) AS i; +ANALYZE uaocs_stats; +DELETE FROM uaocs_stats WHERE a < 16; +-- b) changing gp_appendonly_compaction_threshold to make sure compaction could be triggered, +-- index->reltuples should be equal to table->reltuples +SET gp_appendonly_compaction_threshold = 8; +VACUUM uaocs_stats; +SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; + relname | reltuples +-------------+----------- + uaocs_stats | 85 +(1 row) + +-- expect index->reltuples equals to table->reltuples SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats_index'; relname | reltuples -------------------+----------- uaocs_stats_index | 85 (1 row) --- end_ignore \ No newline at end of file diff --git a/src/test/regress/expected/vacuum_ao_aux_only.out b/src/test/regress/expected/vacuum_ao_aux_only.out new file mode 100644 index 00000000000..41e13c3b4f6 --- /dev/null +++ b/src/test/regress/expected/vacuum_ao_aux_only.out @@ -0,0 +1,163 @@ +-- Test that the AO_AUX_ONLY option will vacuum ONLY auxiliary AO tables +-- create and switch to database +CREATE DATABASE vac_ao_aux; +\c vac_ao_aux +CREATE EXTENSION gp_inject_fault; +ALTER SYSTEM SET autovacuum = off;; +-- start_ignore +\! gpstop -u; +20230222:13:13:53:121318 gpstop:ajrdevbox:ajr-[INFO]:-Starting gpstop with args: -u +20230222:13:13:53:121318 gpstop:ajrdevbox:ajr-[INFO]:-Gathering information and validating the environment... +20230222:13:13:53:121318 gpstop:ajrdevbox:ajr-[INFO]:-Obtaining Greenplum Coordinator catalog information +20230222:13:13:53:121318 gpstop:ajrdevbox:ajr-[INFO]:-Obtaining Segment details from coordinator... +20230222:13:13:53:121318 gpstop:ajrdevbox:ajr-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 7.0.0-alpha.0+dev.18102.g5fee1f1ac9 build dev' +20230222:13:13:53:121318 gpstop:ajrdevbox:ajr-[INFO]:-Signalling all postmaster processes to reload +-- end_ignore +-- Test VACUUM AO_AUX_ONLY without providing a relation list +CREATE TABLE vac_example_heap(i int, j int) DISTRIBUTED BY (i); +INSERT INTO vac_example_heap SELECT j,j FROM generate_series(1, 1000000)j; +CREATE TABLE vac_example_ao(i int, j int) USING ao_row DISTRIBUTED BY (i); +INSERT INTO vac_example_ao SELECT j,j FROM generate_series(1, 1000000)j; +CREATE TABLE vac_example_ao2(i int, j int) USING ao_row DISTRIBUTED BY (i); +INSERT INTO vac_example_ao2 SELECT j,j FROM generate_series(1, 1000000)j; +-- set fault for ao visimap, triggered after vacuum completes on the relation +WITH tableNameCTE AS ( + SELECT c.relname + FROM pg_appendonly pa, pg_class c + WHERE pa.visimaprelid = c.oid AND pa.relid = 'vac_example_ao'::regclass +) +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'skip', '', '', relname, 1, 1, 0, dbid) +FROM tableNameCTE, gp_segment_configuration WHERE role = 'p' AND content != -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- generate bloat on ao aux table +BEGIN; DELETE FROM vac_example_ao WHERE j % 9 = 3; ABORT; +--generate bloat on main tables +DELETE FROM vac_example_heap; +DELETE FROM vac_example_ao2; +-- CALL VACUUM on all tables, but use option to target only AO auxiliary tables +VACUUM AO_AUX_ONLY; +--gp_select_invisible breaks injecting faults so be narrow with it +SET gp_select_invisible=true; +-- show that main tables are not vacuumed +SELECT count(*) FROM vac_example_heap; + count +--------- + 1000000 +(1 row) + +SELECT count(*) FROM vac_example_ao2; + count +--------- + 1000000 +(1 row) + +SET gp_select_invisible=false; +-- show that ao aux table has been vacuumed, causing fault to be triggered +SELECT gp_wait_until_triggered_fault('vacuum_rel_finished_one_relation', 1, dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- clean up fault +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'reset', dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- clean up tables +DROP TABLE vac_example_heap; +DROP TABLE vac_example_ao; +DROP TABLE vac_example_ao2; +-- Test VACUUM AO_AUX_ONLY with a provided a relation list +-- Include partitions in a provided table +CREATE TABLE vac_example_heap(i int, j int) PARTITION BY range (j) DISTRIBUTED by (i); +CREATE TABLE vac_example_0_to_500000 PARTITION OF vac_example_heap + FOR VALUES FROM (0) TO (500000) + USING ao_row; +NOTICE: table has parent, setting distribution columns to match parent table +CREATE TABLE vac_example_500000_to_1000001 PARTITION OF vac_example_heap + FOR VALUES FROM (500000) TO (1000001); +NOTICE: table has parent, setting distribution columns to match parent table +INSERT INTO vac_example_heap SELECT j,j FROM generate_series(1, 1000000)j; +CREATE TABLE vac_example_heap2 (i int, j int) DISTRIBUTED by (i); +INSERT INTO vac_example_heap2 SELECT j,j FROM generate_series(1, 1000000)j; +-- generate bloat on visimap for AO partition and example heap +BEGIN; DELETE FROM vac_example_heap WHERE j < 500000; ABORT; +DELETE FROM vac_example_heap2; +-- set fault for ao visimap, triggered after vacuum completes on the relation +WITH tableNameCTE AS ( + SELECT c.relname + FROM pg_appendonly pa, pg_class c + WHERE pa.visimaprelid = c.oid AND pa.relid = 'vac_example_0_to_500000'::regclass +) +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'skip', '', '', relname, 1, 1, 0, dbid) +FROM tableNameCTE, gp_segment_configuration WHERE role = 'p' AND content != -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +-- CALL VACUUM on list of tables, and use option to target only AO auxiliary tables +VACUUM AO_AUX_ONLY vac_example_heap, vac_example_heap2; +WARNING: skipping "vac_example_heap" for VACUUM AO_AUX_ONLY --- it is not an append-optimized table +DETAIL: it will not have auxiliary tables to vacuum +WARNING: skipping "vac_example_heap2" for VACUUM AO_AUX_ONLY --- it is not an append-optimized table +DETAIL: it will not have auxiliary tables to vacuum +--gp_select_invisible breaks injecting faults so be narrow with it +SET gp_select_invisible=true; +-- show that main table is not vacuumed +SELECT count(*) FROM vac_example_heap; + count +--------- + 1000000 +(1 row) + +SET gp_select_invisible=false; +-- show that ao aux table has been vacuumed, causing fault to be triggered +SELECT gp_wait_until_triggered_fault('vacuum_rel_finished_one_relation', 1, dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + gp_wait_until_triggered_fault +------------------------------- + Success: + Success: + Success: +(3 rows) + +-- clean up fault +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'reset', dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + gp_inject_fault +----------------- + Success: + Success: + Success: +(3 rows) + +ALTER SYSTEM RESET autovacuum; +-- start_ignore +\! gpstop -u; +20230222:13:14:00:121401 gpstop:ajrdevbox:ajr-[INFO]:-Starting gpstop with args: -u +20230222:13:14:00:121401 gpstop:ajrdevbox:ajr-[INFO]:-Gathering information and validating the environment... +20230222:13:14:00:121401 gpstop:ajrdevbox:ajr-[INFO]:-Obtaining Greenplum Coordinator catalog information +20230222:13:14:00:121401 gpstop:ajrdevbox:ajr-[INFO]:-Obtaining Segment details from coordinator... +20230222:13:14:00:121401 gpstop:ajrdevbox:ajr-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 7.0.0-alpha.0+dev.18102.g5fee1f1ac9 build dev' +20230222:13:14:00:121401 gpstop:ajrdevbox:ajr-[INFO]:-Signalling all postmaster processes to reload +-- end_ignore +\c regression +DROP DATABASE vac_ao_aux; diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index e794701fb30..74b55ea82aa 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -263,6 +263,7 @@ test: vacuum_full_ao test: vacuum_full_freeze_heap test: vacuum_full_heap test: vacuum_full_heap_bitmapindex +test: vacuum_ao_aux_only # Check for shmem leak for instrumentation slots test: instr_in_shmem_verify # check autostats diff --git a/src/test/regress/input/uao_dml/uao_dml.source b/src/test/regress/input/uao_dml/uao_dml.source index adb9f033b70..6e3916100b7 100644 --- a/src/test/regress/input/uao_dml/uao_dml.source +++ b/src/test/regress/input/uao_dml/uao_dml.source @@ -479,14 +479,7 @@ update mytab_@amname@ set col_text=' new value' where col_int = 1; select * from mytab_@amname@; vacuum mytab_@amname@; SELECT reltuples FROM pg_class WHERE relname = 'mytab_@amname@'; - --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore SELECT reltuples FROM pg_class WHERE relname = 'mytab_int_idx1_@amname@'; --- end_ignore -- @Description Checks that deleting works with many AO blocks. -- diff --git a/src/test/regress/output/uao_dml/uao_dml.source b/src/test/regress/output/uao_dml/uao_dml.source index f07b1b8de1e..9ecf578eb83 100644 --- a/src/test/regress/output/uao_dml/uao_dml.source +++ b/src/test/regress/output/uao_dml/uao_dml.source @@ -939,18 +939,12 @@ SELECT reltuples FROM pg_class WHERE relname = 'mytab_@amname@'; 2 (1 row) --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore SELECT reltuples FROM pg_class WHERE relname = 'mytab_int_idx1_@amname@'; reltuples ----------- 2 (1 row) --- end_ignore -- @Description Checks that deleting works with many AO blocks. -- DROP TABLE IF EXISTS foo; diff --git a/src/test/regress/sql/brin_ao.sql b/src/test/regress/sql/brin_ao.sql index 0c624a08276..68057a1fdd9 100644 --- a/src/test/regress/sql/brin_ao.sql +++ b/src/test/regress/sql/brin_ao.sql @@ -464,10 +464,4 @@ SELECT segment_id, segno, tupcount, state FROM gp_toolkit.__gp_aoseg('brintest_a -- Tests for brin_summarize_new_values SELECT brin_summarize_new_values('brintest_ao'); -- error, not an index SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- brin_summarize_new_values() will not always be accurate. So ignore the check to --- coordinate with the new behavior. --- start_ignore SELECT brin_summarize_new_values('brinaoidx'); -- ok, no change expected --- end_ignore diff --git a/src/test/regress/sql/brin_aocs.sql b/src/test/regress/sql/brin_aocs.sql index ff182c0aaf3..8e9d4c704b5 100644 --- a/src/test/regress/sql/brin_aocs.sql +++ b/src/test/regress/sql/brin_aocs.sql @@ -464,10 +464,4 @@ SELECT segment_id, segno, tupcount, state FROM gp_toolkit.__gp_aocsseg('brintest -- Tests for brin_summarize_new_values SELECT brin_summarize_new_values('brintest_aocs'); -- error, not an index SELECT brin_summarize_new_values('tenk1_unique1'); -- error, not a BRIN index --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- brin_summarize_new_values() will not always be accurate. So ignore the check to --- coordinate with the new behavior. --- start_ignore SELECT brin_summarize_new_values('brinaocsidx'); -- ok, no change expected --- end_ignore diff --git a/src/test/regress/sql/uao_compaction/drop_column.sql b/src/test/regress/sql/uao_compaction/drop_column.sql index 83e542be3c9..207a298ab5c 100644 --- a/src/test/regress/sql/uao_compaction/drop_column.sql +++ b/src/test/regress/sql/uao_compaction/drop_column.sql @@ -12,12 +12,12 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_drop_col_index'; VACUUM uao_drop_col; SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_drop_col'; -- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold. In this case, the ratio of dead tuples is about 30% +-- which is greater than default value (10%) of the compaction threshold, so compaction +-- should be triggered and reltuples of both table and index should be equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_drop_col_index'; --- end_ignore ALTER TABLE uao_drop_col DROP COLUMN c; SELECT * FROM uao_drop_col; INSERT INTO uao_drop_col VALUES (42, 42); diff --git a/src/test/regress/sql/uao_compaction/full_stats.sql b/src/test/regress/sql/uao_compaction/full_stats.sql index f8361ea0a94..b133adc4215 100644 --- a/src/test/regress/sql/uao_compaction/full_stats.sql +++ b/src/test/regress/sql/uao_compaction/full_stats.sql @@ -13,10 +13,6 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_full_stats_index'; DELETE FROM uao_full_stats WHERE a < 16; VACUUM FULL uao_full_stats; SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_full_stats'; --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Compaction is always triggered in VACUUM FULL, expecting +-- reltuples of both table and index are equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_full_stats_index'; --- end_ignore diff --git a/src/test/regress/sql/uao_compaction/index_stats.sql b/src/test/regress/sql/uao_compaction/index_stats.sql index f88e000e707..15550338d7a 100644 --- a/src/test/regress/sql/uao_compaction/index_stats.sql +++ b/src/test/regress/sql/uao_compaction/index_stats.sql @@ -16,12 +16,6 @@ update mytab set col_text=' new value' where col_int = 1; select * from mytab; vacuum mytab; SELECT relname, reltuples FROM pg_class WHERE relname = 'mytab'; - --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore SELECT relname, reltuples FROM pg_class WHERE relname = 'mytab_int_idx1'; -- Test to ensure that reltuples is updated for an index after lazy vacuum. diff --git a/src/test/regress/sql/uao_compaction/stats.sql b/src/test/regress/sql/uao_compaction/stats.sql index bd2c2ce8c0c..68ae82e8935 100644 --- a/src/test/regress/sql/uao_compaction/stats.sql +++ b/src/test/regress/sql/uao_compaction/stats.sql @@ -12,13 +12,30 @@ SET enable_seqscan=false; SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats_index'; DELETE FROM uao_stats WHERE a < 16; -VACUUM uao_stats; -SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; -- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold: + +-- a) without changing gp_appendonly_compaction_threshold (default is 10), index->reltuples +-- might greater than table->reltuples if no compaction was triggered during VACUUM; +VACUUM uao_stats; +SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; +-- expect index->reltuples greater than table->reltuples SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats_index'; --- end_ignore + +-- re-setup for next case +TRUNCATE uao_stats; +INSERT INTO uao_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(1, 50) AS i; +INSERT INTO uao_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(51, 100) AS i; +ANALYZE uao_stats; + +DELETE FROM uao_stats WHERE a < 16; +-- b) changing gp_appendonly_compaction_threshold to make sure compaction could be triggered, +-- index->reltuples should be equal to table->reltuples +SET gp_appendonly_compaction_threshold = 8; +VACUUM uao_stats; +SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats'; +-- expect index->reltuples equals to table->reltuples +SELECT relname, reltuples FROM pg_class WHERE relname = 'uao_stats_index'; \ No newline at end of file diff --git a/src/test/regress/sql/uaocs_compaction/drop_column.sql b/src/test/regress/sql/uaocs_compaction/drop_column.sql index 3e66ab489f4..d5fcb211074 100644 --- a/src/test/regress/sql/uaocs_compaction/drop_column.sql +++ b/src/test/regress/sql/uaocs_compaction/drop_column.sql @@ -11,12 +11,12 @@ SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_drop_index'; VACUUM uaocs_drop; SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_drop'; -- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold. In this case, the ratio of dead tuples is about 30% +-- which is greater than default value (10%) of the compaction threshold, so compaction +-- should be triggered and reltuples of both table and index should be equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_drop_index'; --- end_ignore ALTER TABLE uaocs_drop DROP COLUMN c; SELECT * FROM uaocs_drop; INSERT INTO uaocs_drop VALUES (42, 42); diff --git a/src/test/regress/sql/uaocs_compaction/full_stats.sql b/src/test/regress/sql/uaocs_compaction/full_stats.sql index 92780646eac..85cfa032c0c 100644 --- a/src/test/regress/sql/uaocs_compaction/full_stats.sql +++ b/src/test/regress/sql/uaocs_compaction/full_stats.sql @@ -15,10 +15,6 @@ SELECT COUNT(*) FROM uaocs_full_stats; VACUUM FULL uaocs_full_stats; SELECT COUNT(*) FROM uaocs_full_stats; SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_full_stats'; --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Compaction is always triggered in VACUUM FULL, expecting +-- reltuples of both table and index are equal. SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_full_stats_index'; --- end_ignore diff --git a/src/test/regress/sql/uaocs_compaction/index_stats.sql b/src/test/regress/sql/uaocs_compaction/index_stats.sql index d87b41a8b45..2901238e7ef 100644 --- a/src/test/regress/sql/uaocs_compaction/index_stats.sql +++ b/src/test/regress/sql/uaocs_compaction/index_stats.sql @@ -16,12 +16,6 @@ update uaocs_index_stats set col_text=' new value' where col_int = 1; select * from uaocs_index_stats; vacuum uaocs_index_stats; SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_index_stats'; - --- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_index_stats_int_idx1'; -- Test to ensure that reltuples is updated for an index after lazy vacuum. diff --git a/src/test/regress/sql/uaocs_compaction/stats.sql b/src/test/regress/sql/uaocs_compaction/stats.sql index 9e483641670..81e039ee4bc 100644 --- a/src/test/regress/sql/uaocs_compaction/stats.sql +++ b/src/test/regress/sql/uaocs_compaction/stats.sql @@ -12,13 +12,30 @@ SET enable_seqscan=false; SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats_index'; DELETE FROM uaocs_stats WHERE a < 16; -VACUUM uaocs_stats; -SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; -- New strategy of VACUUM AO/CO was introduced by PR #13255 for performance enhancement. --- Index dead tuples will not always be cleaned up completely after VACUUM, resulting --- index stats pg_class->reltuples will not always be accurate. So ignore the stats check --- for reltuples to coordinate with the new behavior. --- start_ignore +-- Index dead tuples will not always be cleaned up completely after VACUUM, leading to +-- index->reltuples not always equal to table->reltuples, it depends on the GUC +-- gp_appendonly_compaction_threshold: + +-- a) without changing gp_appendonly_compaction_threshold (default is 10), index->reltuples +-- might greater than table->reltuples if no compaction was triggered during VACUUM; +VACUUM uaocs_stats; +SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; +-- expect index->reltuples greater than table->reltuples SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats_index'; --- end_ignore + +-- re-setup for next case +TRUNCATE uaocs_stats; +INSERT INTO uaocs_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(1, 50) AS i; +INSERT INTO uaocs_stats SELECT i as a, i as b, 'hello world' as c FROM generate_series(51, 100) AS i; +ANALYZE uaocs_stats; + +DELETE FROM uaocs_stats WHERE a < 16; +-- b) changing gp_appendonly_compaction_threshold to make sure compaction could be triggered, +-- index->reltuples should be equal to table->reltuples +SET gp_appendonly_compaction_threshold = 8; +VACUUM uaocs_stats; +SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats'; +-- expect index->reltuples equals to table->reltuples +SELECT relname, reltuples FROM pg_class WHERE relname = 'uaocs_stats_index'; \ No newline at end of file diff --git a/src/test/regress/sql/vacuum_ao_aux_only.sql b/src/test/regress/sql/vacuum_ao_aux_only.sql new file mode 100644 index 00000000000..ec6990da61d --- /dev/null +++ b/src/test/regress/sql/vacuum_ao_aux_only.sql @@ -0,0 +1,113 @@ +-- Test that the AO_AUX_ONLY option will vacuum ONLY auxiliary AO tables +-- create and switch to database +CREATE DATABASE vac_ao_aux; +\c vac_ao_aux + +CREATE EXTENSION gp_inject_fault; + +ALTER SYSTEM SET autovacuum = off;; +-- start_ignore +\! gpstop -u; +-- end_ignore + +-- Test VACUUM AO_AUX_ONLY without providing a relation list +CREATE TABLE vac_example_heap(i int, j int) DISTRIBUTED BY (i); +INSERT INTO vac_example_heap SELECT j,j FROM generate_series(1, 1000000)j; + +CREATE TABLE vac_example_ao(i int, j int) USING ao_row DISTRIBUTED BY (i); +INSERT INTO vac_example_ao SELECT j,j FROM generate_series(1, 1000000)j; + +CREATE TABLE vac_example_ao2(i int, j int) USING ao_row DISTRIBUTED BY (i); +INSERT INTO vac_example_ao2 SELECT j,j FROM generate_series(1, 1000000)j; + +-- set fault for ao visimap, triggered after vacuum completes on the relation +WITH tableNameCTE AS ( + SELECT c.relname + FROM pg_appendonly pa, pg_class c + WHERE pa.visimaprelid = c.oid AND pa.relid = 'vac_example_ao'::regclass +) +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'skip', '', '', relname, 1, 1, 0, dbid) +FROM tableNameCTE, gp_segment_configuration WHERE role = 'p' AND content != -1; + +-- generate bloat on ao aux table +BEGIN; DELETE FROM vac_example_ao WHERE j % 9 = 3; ABORT; + +--generate bloat on main tables +DELETE FROM vac_example_heap; +DELETE FROM vac_example_ao2; + +-- CALL VACUUM on all tables, but use option to target only AO auxiliary tables +VACUUM AO_AUX_ONLY; + +--gp_select_invisible breaks injecting faults so be narrow with it +SET gp_select_invisible=true; +-- show that main tables are not vacuumed +SELECT count(*) FROM vac_example_heap; +SELECT count(*) FROM vac_example_ao2; +SET gp_select_invisible=false; + +-- show that ao aux table has been vacuumed, causing fault to be triggered +SELECT gp_wait_until_triggered_fault('vacuum_rel_finished_one_relation', 1, dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + +-- clean up fault +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'reset', dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + +-- clean up tables +DROP TABLE vac_example_heap; +DROP TABLE vac_example_ao; +DROP TABLE vac_example_ao2; + + +-- Test VACUUM AO_AUX_ONLY with a provided a relation list +-- Include partitions in a provided table +CREATE TABLE vac_example_heap(i int, j int) PARTITION BY range (j) DISTRIBUTED by (i); +CREATE TABLE vac_example_0_to_500000 PARTITION OF vac_example_heap + FOR VALUES FROM (0) TO (500000) + USING ao_row; +CREATE TABLE vac_example_500000_to_1000001 PARTITION OF vac_example_heap + FOR VALUES FROM (500000) TO (1000001); +INSERT INTO vac_example_heap SELECT j,j FROM generate_series(1, 1000000)j; + +CREATE TABLE vac_example_heap2 (i int, j int) DISTRIBUTED by (i); +INSERT INTO vac_example_heap2 SELECT j,j FROM generate_series(1, 1000000)j; + +-- generate bloat on visimap for AO partition and example heap +BEGIN; DELETE FROM vac_example_heap WHERE j < 500000; ABORT; +DELETE FROM vac_example_heap2; + +-- set fault for ao visimap, triggered after vacuum completes on the relation +WITH tableNameCTE AS ( + SELECT c.relname + FROM pg_appendonly pa, pg_class c + WHERE pa.visimaprelid = c.oid AND pa.relid = 'vac_example_0_to_500000'::regclass +) +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'skip', '', '', relname, 1, 1, 0, dbid) +FROM tableNameCTE, gp_segment_configuration WHERE role = 'p' AND content != -1; + +-- CALL VACUUM on list of tables, and use option to target only AO auxiliary tables +VACUUM AO_AUX_ONLY vac_example_heap, vac_example_heap2; + +--gp_select_invisible breaks injecting faults so be narrow with it +SET gp_select_invisible=true; +-- show that main table is not vacuumed +SELECT count(*) FROM vac_example_heap; +SET gp_select_invisible=false; + +-- show that ao aux table has been vacuumed, causing fault to be triggered +SELECT gp_wait_until_triggered_fault('vacuum_rel_finished_one_relation', 1, dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + +-- clean up fault +SELECT gp_inject_fault('vacuum_rel_finished_one_relation', 'reset', dbid) +FROM gp_segment_configuration WHERE role = 'p' AND content != -1; + + +ALTER SYSTEM RESET autovacuum; +-- start_ignore +\! gpstop -u; +-- end_ignore + +\c regression +DROP DATABASE vac_ao_aux;