Cherry-pick VACUUM stats & Vacuum progress related commits.

src/backend/access/aocs/aocs_compaction.c

@@ -39,6 +39,8 @@
 #include "utils/snapmgr.h"
 #include "utils/guc.h"
 #include "miscadmin.h"
+#include "commands/progress.h"
+#include "pgstat.h"
 
 /* 
  * Hook for plugins to get control after move or throw away tuple in 
@@ -60,7 +62,7 @@ aocs_compaction_delete_hook_type aocs_compaction_delete_hook = NULL;
  * segments, including any empty ones we've left behind.
  */
 void
-AOCSCompaction_DropSegmentFile(Relation aorel, int segno)
+AOCSCompaction_DropSegmentFile(Relation aorel, int segno, AOVacuumRelStats *vacrelstats)
 {
 	int			col;
 
@@ -83,7 +85,7 @@ AOCSCompaction_DropSegmentFile(Relation aorel, int segno)
 		fd = OpenAOSegmentFile(aorel, filenamepath, 0);
 		if (fd >= 0)
 		{
-			TruncateAOSegmentFile(fd, aorel, pseudoSegNo, 0);
+			TruncateAOSegmentFile(fd, aorel, pseudoSegNo, 0, vacrelstats);
 			CloseAOSegmentFile(fd);
 		}
 		else
@@ -109,7 +111,7 @@ AOCSCompaction_DropSegmentFile(Relation aorel, int segno)
  * transactions.
  */
 void
-AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, AOCSVPInfo *vpinfo)
+AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, AOCSVPInfo *vpinfo, AOVacuumRelStats *vacrelstats)
 {
 	const char *relname = RelationGetRelationName(aorel);
 	int			j;
@@ -146,7 +148,7 @@ AOCSSegmentFileTruncateToEOF(Relation aorel, int segno, AOCSVPInfo *vpinfo)
 		fd = OpenAOSegmentFile(aorel, filenamepath, segeof);
 		if (fd >= 0)
 		{
-			TruncateAOSegmentFile(fd, aorel, fileSegNo, segeof);
+			TruncateAOSegmentFile(fd, aorel, fileSegNo, segeof, vacrelstats);
 			CloseAOSegmentFile(fd);
 
 			elogif(Debug_appendonly_print_compaction, LOG,
@@ -221,22 +223,24 @@ static bool
 AOCSSegmentFileFullCompaction(Relation aorel,
 							  AOCSInsertDesc insertDesc,
 							  AOCSFileSegInfo *fsinfo,
-							  Snapshot snapshot)
+							  Snapshot snapshot,
+							  AOVacuumRelStats *vacrelstats)
 {
 	const char *relname;
 	AppendOnlyVisimap visiMap;
 	AOCSScanDesc scanDesc;
 	TupleDesc	tupDesc;
 	TupleTableSlot *slot;
 	int			compact_segno;
-	int64		movedTupleCount = 0;
 	ResultRelInfo *resultRelInfo;
 	MemTupleBinding *mt_bind;
 	EState	   *estate;
-	AOTupleId  *aoTupleId;
-	ItemPointerData otid;
 	int64		tupleCount = 0;
 	int64		tuplePerPage = INT_MAX;
+	int64		curr_num_dead_tuples = 0;
+	int64		prev_num_dead_tuples = 0;
+	int64		curr_heap_blks_scanned = 0;
+	int64		prev_heap_blks_scanned = 0;
 
 	Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
 	Assert(RelationIsAoCols(aorel));
@@ -294,34 +298,49 @@ AOCSSegmentFileFullCompaction(Relation aorel,
 	{
 		CHECK_FOR_INTERRUPTS();
 
-		aoTupleId = (AOTupleId *) &slot->tts_tid;
-		otid = slot->tts_tid;
-		if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId))
-		{
-			AOCSMoveTuple(slot,
-						  insertDesc,
-						  resultRelInfo,
-						  estate);
-			movedTupleCount++;
-		}
-		else
-		{
-			/* Tuple is invisible and needs to be dropped */
-			AppendOnlyThrowAwayTuple(aorel, slot, mt_bind);
-		}
+		/*
+ 		 * AppendOnlyVisimap_IsVisible() has already been called in aocs_getnext().
+		 */
+		Assert(AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap,
+										   (AOTupleId *) &slot->tts_tid));
+		AOCSMoveTuple(slot,
+					  insertDesc,
+					  resultRelInfo,
+					  estate);
 
 		if (aocs_compaction_delete_hook)
 			(*aocs_compaction_delete_hook) (aorel, &otid);
 
 		/*
 		 * Check for vacuum delay point after approximatly a var block
 		 */
-		tupleCount++;
-		if (VacuumCostActive && tupleCount % tuplePerPage == 0)
+		moved_tupleCount++;
+		if (VacuumCostActive && moved_tupleCount % tuplePerPage == 0)
 		{
 			vacuum_delay_point();
 		}
+
+		/*
+		 * Report that we are now scanning and compacting segment files.
+		 */
+		curr_num_dead_tuples = scanDesc->cur_seg_row + 1 - moved_tupleCount;
+		if (curr_num_dead_tuples > prev_num_dead_tuples)
+		{
+			pgstat_progress_update_param(PROGRESS_VACUUM_NUM_DEAD_TUPLES,
+										 vacrelstats->num_dead_tuples + curr_num_dead_tuples);
+			prev_num_dead_tuples = curr_num_dead_tuples;
+		}
+
+		curr_heap_blks_scanned = RelationGuessNumberOfBlocksFromSize(scanDesc->totalBytesRead);
+		if (curr_heap_blks_scanned > prev_heap_blks_scanned)
+		{
+			pgstat_progress_update_param(PROGRESS_VACUUM_HEAP_BLKS_SCANNED,
+										 curr_heap_blks_scanned);
+			prev_heap_blks_scanned = curr_heap_blks_scanned;
+		}
 	}
+	/* Accumulate total number dead tuples */
+	vacrelstats->num_dead_tuples += scanDesc->cur_seg_row - moved_tupleCount;
 
 	MarkAOCSFileSegInfoAwaitingDrop(aorel, compact_segno);
 
@@ -340,7 +359,7 @@ AOCSSegmentFileFullCompaction(Relation aorel,
 	elogif(Debug_appendonly_print_compaction, LOG,
 		   "Finished compaction: "
 		   "AO segfile %d, relation %s, moved tuple count " INT64_FORMAT,
-		   compact_segno, relname, movedTupleCount);
+		   compact_segno, relname, tupleCount);
 
 	AppendOnlyVisimap_Finish(&visiMap, NoLock);
 
@@ -361,7 +380,7 @@ AOCSSegmentFileFullCompaction(Relation aorel,
  * The compaction segment file should be locked for this transaction in
  * the appendonlywriter.c code.
  *
- * On exit, *insert_segno will be set to the the segment that was used as the
+ * On exit, *insert_segno will be set to the segment that was used as the
  * insertion target. The segfiles listed in 'avoid_segnos' will not be used
  * for insertion.
  *
@@ -373,7 +392,8 @@ AOCSCompact(Relation aorel,
 			int compaction_segno,
 			int *insert_segno,
 			bool isFull,
-			List *avoid_segnos)
+			List *avoid_segnos,
+			AOVacuumRelStats *vacrelstats)
 {
 	const char *relname;
 	AOCSInsertDesc insertDesc = NULL;
@@ -407,7 +427,8 @@ AOCSCompact(Relation aorel,
 			AOCSSegmentFileFullCompaction(aorel,
 										  insertDesc,
 										  fsinfo,
-										  appendOnlyMetaDataSnapshot);
+										  appendOnlyMetaDataSnapshot,
+										  vacrelstats);
 
 			insertDesc->skipModCountIncrement = true;
 			aocs_insert_finish(insertDesc, NULL);

src/backend/access/aocs/aocsam.c

@@ -592,8 +592,7 @@ aocs_beginscan_internal(Relation relation,
 								 &scan->checksum,
 								 NULL);
 
-	GetAppendOnlyEntryAuxOids(RelationGetRelid(relation),
-							  scan->appendOnlyMetaDataSnapshot,
+	GetAppendOnlyEntryAuxOids(relation,
 							  NULL, NULL, NULL,
 							  &visimaprelid, &visimapidxid);
 
@@ -1008,8 +1007,7 @@ aocs_insert_init(Relation rel, int segno)
                                  &nd);
     desc->compType = NameStr(nd);
 
-    GetAppendOnlyEntryAuxOids(rel->rd_id,
-                              desc->appendOnlyMetaDataSnapshot,
+    GetAppendOnlyEntryAuxOids(rel,
                               &desc->segrelid, &desc->blkdirrelid, NULL,
                               &desc->visimaprelid, &desc->visimapidxid);
 
@@ -1481,8 +1479,7 @@ aocs_fetch_init(Relation relation,
     bool checksum;
     Oid visimaprelid;
     Oid visimapidxid;
-    GetAppendOnlyEntryAuxOids(relation->rd_id,
-                              appendOnlyMetaDataSnapshot,
+    GetAppendOnlyEntryAuxOids(relation,
                               &aocsFetchDesc->segrelid, NULL, NULL,
                               &visimaprelid, &visimapidxid);
 
@@ -1854,8 +1851,7 @@ aocs_delete_init(Relation rel)
 
     Snapshot snapshot = GetCatalogSnapshot(InvalidOid);
 
-    GetAppendOnlyEntryAuxOids(rel->rd_id,
-                              snapshot,
+    GetAppendOnlyEntryAuxOids(rel,
                               NULL, NULL, NULL,
                               &visimaprelid, &visimapidxid);
 

src/backend/access/aocs/aocsam_handler.c

@@ -1326,15 +1326,14 @@ heap_truncate_one_relid(Oid relid)
 static void
 aoco_relation_nontransactional_truncate(Relation rel)
 {
-	Oid			ao_base_relid = RelationGetRelid(rel);
 	Oid			aoseg_relid = InvalidOid;
 	Oid			aoblkdir_relid = InvalidOid;
 	Oid			aovisimap_relid = InvalidOid;
 
 	ao_truncate_one_rel(rel);
 
 	/* Also truncate the aux tables */
-	GetAppendOnlyEntryAuxOids(ao_base_relid, NULL,
+	GetAppendOnlyEntryAuxOids(rel,
 	                          &aoseg_relid,
 	                          &aoblkdir_relid, NULL,
 	                          &aovisimap_relid, NULL);
@@ -1742,8 +1741,8 @@ aoco_index_build_range_scan(Relation heapRelation,
 	Oid blkdirrelid;
 	Oid blkidxrelid;
 
-	GetAppendOnlyEntryAuxOids(RelationGetRelid(aocoscan->rs_base.rs_rd), NULL, NULL,
-	                          &blkdirrelid, &blkidxrelid, NULL, NULL);
+	GetAppendOnlyEntryAuxOids(heapRelation, NULL,
+							  &blkdirrelid, &blkidxrelid, NULL, NULL);
 	/*
 	 * Note that block directory is created during creation of the first
 	 * index.  If it is found empty, it means the block directory was created

src/backend/access/aocs/aocssegfiles.c

@@ -159,8 +159,7 @@ GetAOCSFileSegInfo(Relation prel,
 	bool		isNull;
 
     Oid         segrelid;
-    GetAppendOnlyEntryAuxOids(prel->rd_id,
-                              appendOnlyMetaDataSnapshot,
+    GetAppendOnlyEntryAuxOids(prel,
                               &segrelid, NULL, NULL,
                               NULL, NULL);
 
@@ -273,8 +272,7 @@ GetAllAOCSFileSegInfo(Relation prel,
 
 	Assert(RelationIsAoCols(prel));
 
-	GetAppendOnlyEntryAuxOids(prel->rd_id,
-							  appendOnlyMetaDataSnapshot,
+	GetAppendOnlyEntryAuxOids(prel,
 							  &segrelid, NULL, NULL,
 							  NULL, NULL);
 
@@ -521,8 +519,7 @@ MarkAOCSFileSegInfoAwaitingDrop(Relation prel, int segno)
 	Assert(RelationIsAoCols(prel));
 
 	appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
-	GetAppendOnlyEntryAuxOids(prel->rd_id,
-							  appendOnlyMetaDataSnapshot,
+	GetAppendOnlyEntryAuxOids(prel,
 							  &segrelid, NULL, NULL,
 							  NULL, NULL);
 	UnregisterSnapshot(appendOnlyMetaDataSnapshot);
@@ -614,8 +611,7 @@ ClearAOCSFileSegInfo(Relation prel, int segno)
 		   RelationGetRelationName(prel));
 
 	appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
-	GetAppendOnlyEntryAuxOids(prel->rd_id,
-							  appendOnlyMetaDataSnapshot,
+	GetAppendOnlyEntryAuxOids(prel,
 							  &segrelid, NULL, NULL,
 							  NULL, NULL);
 	UnregisterSnapshot(appendOnlyMetaDataSnapshot);
@@ -905,8 +901,7 @@ AOCSFileSegInfoAddVpe(Relation prel, int32 segno,
 	}
 
     Oid         segrelid;
-    GetAppendOnlyEntryAuxOids(prel->rd_id,
-                              NULL,
+    GetAppendOnlyEntryAuxOids(prel,
                               &segrelid, NULL, NULL,
                               NULL, NULL);
 	segrel = heap_open(segrelid, RowExclusiveLock);
@@ -1024,8 +1019,7 @@ AOCSFileSegInfoAddCount(Relation prel, int32 segno,
 	TupleDesc	tupdesc;
 
     Oid         segrelid;
-    GetAppendOnlyEntryAuxOids(prel->rd_id,
-                              NULL,
+    GetAppendOnlyEntryAuxOids(prel,
                               &segrelid, NULL, NULL,
                               NULL, NULL);
 
@@ -1212,8 +1206,7 @@ gp_aocsseg_internal(PG_FUNCTION_ARGS, Oid aocsRelOid)
 		context->relnatts = aocsRel->rd_rel->relnatts;
 
         Oid         segrelid;
-        GetAppendOnlyEntryAuxOids(aocsRel->rd_id,
-                                  appendOnlyMetaDataSnapshot,
+        GetAppendOnlyEntryAuxOids(aocsRel,
                                   &segrelid, NULL, NULL,
                                   NULL, NULL);
 		pg_aocsseg_rel = heap_open(segrelid, AccessShareLock);
@@ -1425,8 +1418,7 @@ gp_aocsseg_history(PG_FUNCTION_ARGS)
 		context->relnatts = aocsRel->rd_rel->relnatts;
 
         Oid         segrelid;
-        GetAppendOnlyEntryAuxOids(aocsRel->rd_id,
-                                  NULL,
+        GetAppendOnlyEntryAuxOids(aocsRel,
                                   &segrelid, NULL, NULL,
                                   NULL, NULL);
 
@@ -1552,7 +1544,7 @@ aocol_compression_ratio_internal(Relation parentrel)
 
 	Assert(Gp_role == GP_ROLE_DISPATCH || Gp_role == GP_ROLE_UTILITY);
 
-	GetAppendOnlyEntryAuxOids(RelationGetRelid(parentrel), NULL,
+	GetAppendOnlyEntryAuxOids(parentrel,
 							  &segrelid, NULL, NULL, NULL, NULL);
 	Assert(OidIsValid(segrelid));