diff --git a/runtime/gc_vlhgc/ConfigurationIncrementalGenerational.cpp b/runtime/gc_vlhgc/ConfigurationIncrementalGenerational.cpp index d88f0cd0c81..b7cf0c9a9ed 100644 --- a/runtime/gc_vlhgc/ConfigurationIncrementalGenerational.cpp +++ b/runtime/gc_vlhgc/ConfigurationIncrementalGenerational.cpp @@ -304,6 +304,9 @@ MM_ConfigurationIncrementalGenerational::initialize(MM_EnvironmentBase *env) #define DEFAULT_MAX_NURSERY_AGE 1 #define DEFAULT_MAX_AGE_FOR_ALLOCATION_BASED 5 + /* enable AllocationAge by default for local testing */ + extensions->tarokAllocationAgeEnabled = true; + /* set default region maximum age if it is not specified yet */ if (0 == extensions->tarokRegionMaxAge) { if (extensions->tarokAllocationAgeEnabled) { diff --git a/runtime/gc_vlhgc/CopyForwardScheme.cpp b/runtime/gc_vlhgc/CopyForwardScheme.cpp index 8e503fccfbe..b3d50181b86 100644 --- a/runtime/gc_vlhgc/CopyForwardScheme.cpp +++ b/runtime/gc_vlhgc/CopyForwardScheme.cpp @@ -1848,7 +1848,7 @@ MM_CopyForwardScheme::stopCopyingIntoCache(MM_EnvironmentVLHGC *env, uintptr_t c MM_HeapRegionDescriptorVLHGC * region = (MM_HeapRegionDescriptorVLHGC *)_regionManager->tableDescriptorForAddress(copyCache->cacheBase); /* atomically add (age * usedBytes) product from this cache to the regions product */ - double newAllocationAgeSizeProduct = region->atomicIncrementAllocationAgeSizeProduct(copyCache->_allocationAgeSizeProduct); + double newAllocationAgeSizeProduct = region->atomicIncrementAllocationAgeSizeProduct(copyCache->_allocationAgeSizeProduct, copyCache->_objectSize); region->updateAgeBounds(copyCache->_lowerAgeBound, copyCache->_upperAgeBound); /* Return any remaining memory to the pool */ @@ -5638,7 +5638,7 @@ MM_CopyForwardScheme::setRegionAsSurvivor(MM_EnvironmentVLHGC *env, MM_HeapRegio (double)region->getAllocationAge() / (1024 * 1024), (double)usedBytes / (1024 * 1024), allocationAgeSizeProduct / (1024 * 1024) / (1024 * 1024)); Assert_MM_true(0.0 == region->getAllocationAgeSizeProduct()); - region->setAllocationAgeSizeProduct(allocationAgeSizeProduct); + region->setAllocationAgeSizeProduct(allocationAgeSizeProduct, usedBytes); if (freshSurvivor) { region->resetAgeBounds(); } @@ -5652,13 +5652,18 @@ MM_CopyForwardScheme::setRegionAsSurvivor(MM_EnvironmentVLHGC *env, MM_HeapRegio void MM_CopyForwardScheme::setAllocationAgeForMergedRegion(MM_EnvironmentVLHGC *env, MM_HeapRegionDescriptorVLHGC *region) { - uintptr_t compactGroup = MM_CompactGroupManager::getCompactGroupNumber(env, region); - uintptr_t usedBytes = region->getSize() - region->getMemoryPool()->getFreeMemoryAndDarkMatterBytes(); + /* "Dark matter" refers to memory that appears allocated but isn't reachable. + * This can cause inaccurate projections and potential Boundaries Overflow. + * so use actual copied byte instead of calculating from getFreeMemoryAndDarkMatterBytes() for usedBytes. + */ +// uintptr_t usedBytes = region->getSize() - region->getMemoryPool()->getFreeMemoryAndDarkMatterBytes(); + uintptr_t usedBytes = region->getUsedAllocationBytes(); Assert_MM_true(0 != usedBytes); /* convert allocation age product (usedBytes * age) back to pure age */ uint64_t newAllocationAge = (uint64_t)(region->getAllocationAgeSizeProduct() / (double)usedBytes); + uintptr_t compactGroup = MM_CompactGroupManager::getCompactGroupNumber(env, region); Trc_MM_CopyForwardScheme_setAllocationAgeForMergedRegion(env->getLanguageVMThread(), _regionManager->mapDescriptorToRegionTableIndex(region), compactGroup, region->getAllocationAgeSizeProduct() / (1024 * 1024) / (1024 * 1024), (double)usedBytes / (1024 * 1024), (double)newAllocationAge / (1024 * 1024), @@ -5666,7 +5671,17 @@ MM_CopyForwardScheme::setAllocationAgeForMergedRegion(MM_EnvironmentVLHGC *env, if (_extensions->tarokAllocationAgeEnabled) { Assert_MM_true(newAllocationAge < _extensions->compactGroupPersistentStats[compactGroup]._maxAllocationAge); - Assert_MM_true((MM_CompactGroupManager::getRegionAgeFromGroup(env, compactGroup) == 0) || (newAllocationAge >= _extensions->compactGroupPersistentStats[compactGroup - 1]._maxAllocationAge)); + + if ((MM_CompactGroupManager::getRegionAgeFromGroup(env, compactGroup) != 0) /*eden region */ + && (_extensions->compactGroupPersistentStats[compactGroup - 1]._maxAllocationAge != _extensions->tarokMaximumAgeInBytes) /* tenure region */ + && (newAllocationAge < _extensions->compactGroupPersistentStats[compactGroup - 1]._maxAllocationAge)) { + PORT_ACCESS_FROM_ENVIRONMENT(env); + j9tty_printf(PORTLIB, "setAllocationAgeForMergedRegion newAllocationAge=%zu, region->getLowerAgeBound()=%zu, _extensions->compactGroupPersistentStats[%zu]._maxAllocationAge=%zu, _extensions->compactGroupPersistentStats[%zu]._maxAllocationAge=%zu, _extensions->tarokMaximumAgeInBytes=%zu\n", + newAllocationAge, region->getLowerAgeBound(), compactGroup - 1, _extensions->compactGroupPersistentStats[compactGroup - 1]._maxAllocationAge, compactGroup, _extensions->compactGroupPersistentStats[compactGroup]._maxAllocationAge, _extensions->tarokMaximumAgeInBytes); + Assert_MM_true((MM_CompactGroupManager::getRegionAgeFromGroup(env, compactGroup) == 0) /*eden region */ + || _extensions->compactGroupPersistentStats[compactGroup - 1]._maxAllocationAge == _extensions->tarokMaximumAgeInBytes /* tenure region */ + || (newAllocationAge >= _extensions->compactGroupPersistentStats[compactGroup - 1]._maxAllocationAge)); /*survivior region */ + } } uintptr_t logicalAge = 0; @@ -5677,8 +5692,9 @@ MM_CopyForwardScheme::setAllocationAgeForMergedRegion(MM_EnvironmentVLHGC *env, } region->setAge(newAllocationAge, logicalAge); - /* reset aging auxiliary datea for future usage */ - region->setAllocationAgeSizeProduct(0.0); + + /* reset aging auxiliary data for future usage */ + region->setAllocationAgeSizeProduct(0.0, 0); } bool diff --git a/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.cpp b/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.cpp index 505a1f44255..bce298568ce 100644 --- a/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.cpp +++ b/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.cpp @@ -50,6 +50,7 @@ MM_HeapRegionDescriptorVLHGC::MM_HeapRegionDescriptorVLHGC(MM_EnvironmentVLHGC * ,_extensions(MM_GCExtensions::getExtensions(env)) ,_allocationAge(0) ,_allocationAgeSizeProduct(0.0) + ,_usedAllocationBytes(0) ,_age(0) ,_rememberedSetCardList() ,_rsclBufferPool(NULL) diff --git a/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.hpp b/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.hpp index 3e8bede4a8f..792a064214f 100644 --- a/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.hpp +++ b/runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.hpp @@ -94,6 +94,9 @@ class MM_HeapRegionDescriptorVLHGC : public MM_HeapRegionDescriptor U_64 _lowerAgeBound; /**< lowest possible age of any object in this region */ U_64 _upperAgeBound; /**< highest possible age of any object in this region */ double _allocationAgeSizeProduct; /**< sum of (age * size) products for each object in the region. used for age merging math in survivor regions */ + + uintptr_t _usedAllocationBytes; + uintptr_t _age; /**< logical allocation age (number of GC cycles since the last attempted allocation) */ MM_RememberedSetCardList _rememberedSetCardList; /**< remembered set card list */ MM_RememberedSetCard *_rsclBufferPool; /**< RSCL Buffer pool owned by this region (Buffers can still be shared among other regions) */ @@ -128,23 +131,32 @@ class MM_HeapRegionDescriptorVLHGC : public MM_HeapRegionDescriptor return _allocationAgeSizeProduct; } + MMINLINE uintptr_t getUsedAllocationBytes() + { + return _usedAllocationBytes; + } + /** - * set current value of (age * size) product used for survivor regions (used for setting its initial value) + * Set current value of (age × size) product used for survivor regions + * @param allocationAgeSizeProduct age to increment with + * @param usedAllocationBytes actual bytes allocated (copied) in the region */ - MMINLINE void setAllocationAgeSizeProduct(double allocationAgeSizeProduct) + MMINLINE void setAllocationAgeSizeProduct(double allocationAgeSizeProduct, uintptr_t usedAllocationBytes) { + _usedAllocationBytes = usedAllocationBytes; _allocationAgeSizeProduct = allocationAgeSizeProduct; } - /** * increment atomically allocation (age * size) product * @param allocationAgeSizeProduct age to increment with + * @param copiedBytes * @return new allocationAgeSizeProduct value */ - MMINLINE double atomicIncrementAllocationAgeSizeProduct(double allocationAgeSizeProduct) + MMINLINE double atomicIncrementAllocationAgeSizeProduct(double allocationAgeSizeProduct, uintptr_t copiedBytes) { - return MM_AtomicOperations::addDouble(&_allocationAgeSizeProduct, allocationAgeSizeProduct); + MM_AtomicOperations::add(&_usedAllocationBytes, copiedBytes); + return MM_AtomicOperations::addDouble(&_allocationAgeSizeProduct, allocationAgeSizeProduct); } /**