diff --git a/runtime/gc_vlhgc/IncrementalGenerationalGC.cpp b/runtime/gc_vlhgc/IncrementalGenerationalGC.cpp index d44ace2a729..027c9ddc3c7 100644 --- a/runtime/gc_vlhgc/IncrementalGenerationalGC.cpp +++ b/runtime/gc_vlhgc/IncrementalGenerationalGC.cpp @@ -1021,6 +1021,8 @@ MM_IncrementalGenerationalGC::partialGarbageCollectPostWork(MM_EnvironmentVLHGC PORT_ACCESS_FROM_ENVIRONMENT(env); env->_cycleState->_endTime = j9time_hires_clock(); + verifyHeapSizing(env, false); + reportGCCycleFinalIncrementEnding(env); reportGCIncrementEnd(env); reportPGCEnd(env); @@ -1257,6 +1259,8 @@ MM_IncrementalGenerationalGC::runGlobalGarbageCollection(MM_EnvironmentVLHGC *en PORT_ACCESS_FROM_ENVIRONMENT(env); env->_cycleState->_endTime = j9time_hires_clock(); + verifyHeapSizing(env, true); + reportGCCycleFinalIncrementEnding(env); /* TODO: TEMPORARY: This is a temporary call that should be deleted once the new verbose format is in place */ /* NOTE: May want to move any tracepoints up into this routine */ @@ -1339,7 +1343,13 @@ MM_IncrementalGenerationalGC::preProcessPGCUsingCopyForward(MM_EnvironmentVLHGC * NOTE: A second estimate for free tenure is later made - The lowest estimate for free tenure is used in heap sizing calculations */ _extensions->globalVLHGCStats._heapSizingData.freeTenure = freeMemoryForSurvivor; - + /* in some case free eden size is significant, should not be counted as freeTenure */ + uintptr_t allocatedSinceLastPGC = getAllocatedSinceLastPGC(); + uintptr_t edenSize = getCurrentEdenSizeInBytes(NULL); + if (edenSize >= allocatedSinceLastPGC) { + Assert_MM_true(_extensions->globalVLHGCStats._heapSizingData.freeTenure >= (edenSize - allocatedSinceLastPGC)); + _extensions->globalVLHGCStats._heapSizingData.freeTenure -= edenSize - allocatedSinceLastPGC; + } cycleState->_vlhgcIncrementStats._copyForwardStats._freeMemoryBefore = freeMemoryForSurvivor; cycleState->_vlhgcIncrementStats._copyForwardStats._totalMemoryBefore = _extensions->getHeap()->getMemorySize(); @@ -2611,3 +2621,14 @@ MM_IncrementalGenerationalGC::getBytesScannedInGlobalMarkPhase() } return bytesScanned; } + +void +MM_IncrementalGenerationalGC::verifyHeapSizing(MM_EnvironmentVLHGC *env, bool isGlobalGC) +{ + Assert_GC_true_with_message(env, (((MM_GlobalAllocationManagerTarok *)_extensions->globalAllocationManager)->getFreeRegionCount() >= _schedulingDelegate.getCurrentEdenSizeInRegions(env)), + "verifyHeapSizing freeRegionCount(%zu) is less than EdenRegionCount(%zu), allocatedSinceLastPGC=%zu bytes in current %s\n", + ((MM_GlobalAllocationManagerTarok *)_extensions->globalAllocationManager)->getFreeRegionCount(), + _schedulingDelegate.getCurrentEdenSizeInRegions(env), + getAllocatedSinceLastPGC(), + (isGlobalGC)?"Global GC":"PGC"); +} diff --git a/runtime/gc_vlhgc/IncrementalGenerationalGC.hpp b/runtime/gc_vlhgc/IncrementalGenerationalGC.hpp index be8b335568c..abe1bf0385d 100644 --- a/runtime/gc_vlhgc/IncrementalGenerationalGC.hpp +++ b/runtime/gc_vlhgc/IncrementalGenerationalGC.hpp @@ -147,6 +147,12 @@ class MM_IncrementalGenerationalGC : public MM_GlobalCollector * Tooling report the end of an unload phase. */ void reportClassUnloadingEnd(MM_EnvironmentBase *env); + + /** + * Confirm free region count is equal or more than eden region count. + */ + void verifyHeapSizing(MM_EnvironmentVLHGC *env, bool isGlobalGC); + #endif /* J9VM_GC_DYNAMIC_CLASS_UNLOADING */ protected: @@ -415,6 +421,11 @@ class MM_IncrementalGenerationalGC : public MM_GlobalCollector return _schedulingDelegate.getCurrentEdenSizeInBytes(env); } + MMINLINE void reCalculateEdenSize(MM_EnvironmentVLHGC *env) + { + _schedulingDelegate.reCalculateEdenSize(env); + } + MM_IncrementalGenerationalGC(MM_EnvironmentVLHGC *env, MM_HeapRegionManager *manager); friend class MM_MainGCThread; diff --git a/runtime/gc_vlhgc/MemorySubSpaceTarok.cpp b/runtime/gc_vlhgc/MemorySubSpaceTarok.cpp index aaa4387f870..457ce992e72 100644 --- a/runtime/gc_vlhgc/MemorySubSpaceTarok.cpp +++ b/runtime/gc_vlhgc/MemorySubSpaceTarok.cpp @@ -884,7 +884,7 @@ MM_MemorySubSpaceTarok::collectorExpand(MM_EnvironmentBase *env) Assert_MM_true((0 == expandSize) || (_heapRegionManager->getRegionSize() == expandSize)); _extensions->heap->getResizeStats()->setLastExpandReason(SATISFY_COLLECTOR); - + /* expand by a single region */ /* for the most part the code path is not multi-threaded safe, so we do this under expandLock */ uintptr_t expansionAmount= expand(env, expandSize); @@ -949,8 +949,21 @@ MM_MemorySubSpaceTarok::performResize(MM_EnvironmentBase *env, MM_AllocateDescri resizeAmount = -(intptr_t)performContract(env, allocDescription); } else if (_expansionSize != 0) { resizeAmount = performExpand(env); + } else { + /** + * In case there is no heap resize, check if there is the case that free size is smaller than eden size + * due to the conflict between eden resize and heap resize, reCalculateEdenSize if it happens. + */ + uintptr_t freeBytes = _globalAllocationManagerTarok->getFreeRegionCount()*_heapRegionManager->getRegionSize(); + MM_IncrementalGenerationalGC *collector = (MM_IncrementalGenerationalGC*)_extensions->getGlobalCollector(); + uintptr_t edenSizeInBytes = collector->getCurrentEdenSizeInBytes((MM_EnvironmentVLHGC *)env); + if (edenSizeInBytes > freeBytes) { + collector->reCalculateEdenSize((MM_EnvironmentVLHGC *)env); + edenSizeInBytes = collector->getCurrentEdenSizeInBytes((MM_EnvironmentVLHGC *)env); + } + Assert_MM_true(freeBytes >= edenSizeInBytes); } - + env->popVMstate(oldVMState); return resizeAmount; @@ -1003,6 +1016,17 @@ MM_MemorySubSpaceTarok::checkResize(MM_EnvironmentBase *env, MM_AllocateDescript /* Adjust the heap size by both the required amount for eden AND non-eden. Non-eden size should generally be kept the same size, so that GMP kickoff, and incremental defragmentation timing stays accurate */ heapSizeChange += edenChangeRegionsBytes; + if (edenChangeRegionsBytes > heapSizeChange) { + intptr_t freeBytes = (intptr_t)_globalAllocationManagerTarok->getFreeRegionCount()*_heapRegionManager->getRegionSize(); + MM_IncrementalGenerationalGC *collector = (MM_IncrementalGenerationalGC*)_extensions->getGlobalCollector(); + intptr_t edenSizeInBytes = (intptr_t) collector->getCurrentEdenSizeInBytes(NULL); + + /* Avoid heap change() to cause free memory = 0 (eden size = 0), which then leads to OOM, 0nly in case preserved eden size is not zero. */ + if (0 >= (freeBytes + heapSizeChange) && edenSizeInBytes > 0) { + /* Adjust the heap resize so the previously computed eden size is preserved. */ + heapSizeChange = edenSizeInBytes - freeBytes; + } + } if (0 > heapSizeChange) { _contractionSize = (uintptr_t)(heapSizeChange * -1); diff --git a/runtime/gc_vlhgc/SchedulingDelegate.cpp b/runtime/gc_vlhgc/SchedulingDelegate.cpp index 8a3cf5783b7..1072cebfb00 100644 --- a/runtime/gc_vlhgc/SchedulingDelegate.cpp +++ b/runtime/gc_vlhgc/SchedulingDelegate.cpp @@ -419,7 +419,7 @@ MM_SchedulingDelegate::partialGarbageCollectCompleted(MM_EnvironmentVLHGC *env, /* Check eden size based off of new PGC stats */ checkEdenSizeAfterPgc(env, globalSweepHappened); - calculateEdenSize(env); + calculateEdenSize(env, true); /* Recalculate GMP intermission after (possibly) resizing eden */ calculateAutomaticGMPIntermission(env); estimateMacroDefragmentationWork(env); @@ -1343,7 +1343,7 @@ MM_SchedulingDelegate::updateSurvivalRatesAfterCopyForward(double thisEdenSurviv } void -MM_SchedulingDelegate::calculateEdenSize(MM_EnvironmentVLHGC *env) +MM_SchedulingDelegate::calculateEdenSize(MM_EnvironmentVLHGC *env, bool allowTotalHeapResize) { uintptr_t regionSize = _regionManager->getRegionSize(); uintptr_t previousEdenSize = _edenRegionCount * regionSize; @@ -1362,7 +1362,7 @@ MM_SchedulingDelegate::calculateEdenSize(MM_EnvironmentVLHGC *env) Assert_MM_true(edenMaximumCount >= 1); Assert_MM_true(edenMaximumCount >= edenMinimumCount); - /* Allow eden to expand as much as it wants, as long as the total heap can expand to accomodate it */ + /* Allow eden to expand as much as it wants, as long as the total heap can expand to accommodate it. */ uintptr_t desiredEdenCount = OMR_MAX(edenMaximumCount, edenMinimumCount); intptr_t desiredEdenChangeSize = (intptr_t)desiredEdenCount - (intptr_t)_edenRegionCount; /* Determine if eden should try to grow anyways, or if the heap is tight on memory */ @@ -1372,41 +1372,35 @@ MM_SchedulingDelegate::calculateEdenSize(MM_EnvironmentVLHGC *env) Trc_MM_SchedulingDelegate_calculateEdenSize_dynamic(env->getLanguageVMThread(), desiredEdenCount, _edenSurvivalRateCopyForward, _nonEdenSurvivalCountCopyForward, freeRegions, edenMinimumCount, edenMaximumCount); - /* Make sure that the total heap can expand enough to satisfy the desired change in eden size - * If heap is fully expanded (or close to) make sure that there are enough free regions to satisfy given eden size change - */ - intptr_t maxEdenChange = 0; - uintptr_t maxEdenRegionCount = _regionManager->getTableRegionCount(); - bool edenIsVerySmall = (_edenRegionCount * 64) < maxEdenRegionCount; + /* Eden size can not be bigger than free region size. */ + intptr_t maxEdenChange = freeRegions - _edenRegionCount; + if (allowTotalHeapResize) { - /* Eden will be stealing free regions from the entire heap, without telling the heap to grow. - * Note: the eden sizing logic knows how much free memory is available in the heap, and knows to not grow too much. - * eden size can not be bigger than free region size. - */ - maxEdenChange = freeRegions - _edenRegionCount; - - if (0 == maxHeapExpansionRegions) { - _extensions->globalVLHGCStats._heapSizingData.edenRegionChange = 0; - } else { - /* Eden will inform the total heap resizing logic, that it needs to change total heap size in order to maintain same "tenure" size */ maxEdenChange += maxHeapExpansionRegions; - intptr_t edenChangeWithSurvivorHeadroom = desiredEdenChangeSize; - - /* Total heap needs to be aware that by changing eden size, the amount of survivor space might also need to change */ - if (0 < desiredEdenChangeSize) { - edenChangeWithSurvivorHeadroom = desiredEdenChangeSize + (intptr_t)ceil(((double)desiredEdenChangeSize * _edenSurvivalRateCopyForward)); - } else if ((0 > desiredEdenChangeSize) && !edenIsVerySmall) { - /* If eden is shrinking, only factor adjusting in survivor regions for total heap resizing when eden is not very small. - * Factoring in survivor regions when eden is tiny can lead to some innacuracies, and reduce free non-eden regions, which may impact performance + /* Make sure that the total heap can expand enough to satisfy the desired change in eden size. + * If heap is fully expanded (or close to) make sure that there are enough free regions to satisfy given eden size change. + */ + /* Proportionally adjust survivor area. */ + intptr_t edenChangeWithSurvivorHeadroom = desiredEdenChangeSize + (intptr_t)ceil((double)desiredEdenChangeSize * _edenSurvivalRateCopyForward); + /* Inform the total heap resizing logic, that it needs to change total heap size in order to maintain same "non-eden" size. */ + if (freeRegions > _edenRegionCount) { + _extensions->globalVLHGCStats._heapSizingData.edenRegionChange = OMR_MIN(maxHeapExpansionRegions, edenChangeWithSurvivorHeadroom); + } else { + /* PGC has not recovered enough regions to accommodate even for the current eden size. + * So, let's expand heap by that deficit (capped to the maximum that heap expansion allows), plus whatever the new eden size requires). */ - edenChangeWithSurvivorHeadroom = desiredEdenChangeSize + (intptr_t)floor(((double)desiredEdenChangeSize * _edenSurvivalRateCopyForward)); + _extensions->globalVLHGCStats._heapSizingData.edenRegionChange = OMR_MIN(maxHeapExpansionRegions, edenChangeWithSurvivorHeadroom + (intptr_t)(_edenRegionCount - freeRegions)); } - _extensions->globalVLHGCStats._heapSizingData.edenRegionChange = OMR_MIN(maxEdenChange, edenChangeWithSurvivorHeadroom); } + /** + * We account for maxEdenChange late, since it could have changed based on allowTotalHeapResize value. Meanwhile, + * we had notification for full heap resizing based on old maxEdenChange value, but that is ok, since the notification accounted + * for maxHeapExpansionRegions, what is exactly by how much maxEdenChange is potentially changed. + */ desiredEdenChangeSize = OMR_MIN(maxEdenChange, desiredEdenChangeSize); - - _edenRegionCount = (uintptr_t)OMR_MAX(1, ((intptr_t)_edenRegionCount + desiredEdenChangeSize)); + Assert_MM_true(0 <= ((intptr_t)_edenRegionCount + desiredEdenChangeSize)); + _edenRegionCount = _edenRegionCount + desiredEdenChangeSize; Trc_MM_SchedulingDelegate_calculateEdenSize_Exit(env->getLanguageVMThread(), (_edenRegionCount * regionSize)); } diff --git a/runtime/gc_vlhgc/SchedulingDelegate.hpp b/runtime/gc_vlhgc/SchedulingDelegate.hpp index 0033d1cc096..d9b3b17fe7d 100644 --- a/runtime/gc_vlhgc/SchedulingDelegate.hpp +++ b/runtime/gc_vlhgc/SchedulingDelegate.hpp @@ -271,10 +271,11 @@ class MM_SchedulingDelegate : public MM_BaseNonVirtual /** * Following a GC, recalculate the Eden size for the next PGC. * This is typically the same as GCExtensions->tarokeEdenSize, but may be smaller if - * insufficient memory is available + * insufficient memory is available. * @param env[in] the main GC thread + * @param allowTotalHeapResize[in] if true, allow total heap to resize to accommodate for eden resizing (typically after a PGC) */ - void calculateEdenSize(MM_EnvironmentVLHGC *env); + void calculateEdenSize(MM_EnvironmentVLHGC *env, bool allowTotalHeapResize = false); /** * Compute what the ideal eden size should be, and return by how many regions eden should change @@ -659,6 +660,14 @@ class MM_SchedulingDelegate : public MM_BaseNonVirtual */ void heapReconfigured(MM_EnvironmentVLHGC *env); + /** + * call calculateEdenSize() without allowTotalHeapResize option + */ + void reCalculateEdenSize(MM_EnvironmentVLHGC *env) + { + calculateEdenSize(env); + } + double getAvgEdenSurvivalRateCopyForward(MM_EnvironmentVLHGC *env) { return _edenSurvivalRateCopyForward; } MM_SchedulingDelegate(MM_EnvironmentVLHGC *env, MM_HeapRegionManager *manager);