JEP: Automatic Heap Sizing for ZGC

Author: fisk

src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp

@@ -28,6 +28,8 @@
 
 void ZNUMA::pd_initialize() {
   _enabled = false;
+  _node_count = 1;
+  _bound_node_count = 1;
   _count = !FLAG_IS_DEFAULT(ZFakeNUMA)
       ? ZFakeNUMA
       : 1;

src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp

@@ -92,7 +92,7 @@ bool ZPhysicalMemoryBacking::is_initialized() const {
   return _initialized;
 }
 
-void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
+void ZPhysicalMemoryBacking::warn_commit_limits(size_t expected_capacity, size_t max_capacity) const {
   // Does nothing
 }
 
@@ -178,3 +178,7 @@ void ZPhysicalMemoryBacking::unmap(zaddress_unsafe addr, size_t size) const {
     fatal("Failed to map memory (%s)", err.to_string());
   }
 }
+
+void ZPhysicalMemoryBacking::collapse(zaddress_unsafe addr, size_t size) const {
+  ShouldNotReachHere();
+}

src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp

@@ -38,13 +38,15 @@ class ZPhysicalMemoryBacking {
 
   bool is_initialized() const;
 
-  void warn_commit_limits(size_t max_capacity) const;
+  void warn_commit_limits(size_t expected_capacity, size_t max_capacity) const;
 
   size_t commit(zbacking_offset offset, size_t length, uint32_t numa_id) const;
   size_t uncommit(zbacking_offset offset, size_t length) const;
 
   void map(zaddress_unsafe addr, size_t size, zbacking_offset offset) const;
   void unmap(zaddress_unsafe addr, size_t size) const;
+
+  void collapse(zaddress_unsafe addr, size_t size) const;
 };
 
 #endif // OS_BSD_GC_Z_ZPHYSICALMEMORYBACKING_BSD_HPP

src/hotspot/os/linux/gc/z/zLargePages_linux.cpp

@@ -21,21 +21,59 @@
  * questions.
  */
 
+#include "gc/z/zAdaptiveHeap.inline.hpp"
 #include "gc/z/zLargePages.hpp"
 #include "hugepages.hpp"
 #include "os_linux.hpp"
 #include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
+
+#include <sys/mman.h>
+
+static bool madv_collapse_available() {
+  const size_t size = 2 * M;
+  void* const res = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+  if (res == MAP_FAILED) {
+    return false;
+  }
+
+  assert(size >= os::vm_page_size(), "Unexpected page size");
+  os::pretouch_memory(res, (void*)(((char*)res) + os::vm_page_size()));
+
+  bool result = os::Linux::madvise_collapse_transparent_huge_pages(res, size);
+
+  munmap(res, size);
+
+  return result;
+}
 
 void ZLargePages::pd_initialize() {
+  bool can_collapse = ZMemoryHeating && madv_collapse_available();
+
   if (os::Linux::thp_requested()) {
+    if (can_collapse) {
+      _state = Collapse;
+      return;
+    }
     // Check if the OS config turned off transparent huge pages for shmem.
     _os_enforced_transparent_mode = HugePages::shmem_thp_info().is_disabled();
     _state = _os_enforced_transparent_mode ? Disabled : Transparent;
     return;
   }
 
   if (UseLargePages) {
-    _state = Explicit;
+    if (!ZAdaptiveHeap::can_adapt() || ZAdaptiveHeap::explicit_max_capacity()) {
+      _state = Explicit;
+      return;
+    }
+
+    log_warning(gc, init)("UseLargePages requires a max heap size to be set (-Xmx) when running with Automatic Heap Sizing. "
+                          "Disabling the use of explicit large pages for the heap");
+  }
+
+  if (FLAG_IS_DEFAULT(UseTransparentHugePages) && can_collapse) {
+    _state = Collapse;
     return;
   }
 

src/hotspot/os/linux/gc/z/zNUMA_linux.cpp

@@ -38,24 +38,32 @@ static uint32_t* z_node_to_numa_id = nullptr;
 void ZNUMA::pd_initialize() {
   _enabled = UseNUMA;
 
-  size_t configured_nodes = 0;
+  uint32_t configured_nodes = 1;
+  const int max_nodes = os::Linux::numa_num_configured_nodes();
 
-  if (UseNUMA) {
-    const size_t max_nodes = os::Linux::numa_num_configured_nodes();
-    z_numa_id_to_node = NEW_C_HEAP_ARRAY(uint, max_nodes, mtGC);
-    configured_nodes = os::numa_get_leaf_groups(z_numa_id_to_node, 0);
+  if (max_nodes != -1) {
+    assert(max_nodes >= 1, "bad max_nodes value: %d", max_nodes);
 
-    z_node_to_numa_id = NEW_C_HEAP_ARRAY(uint32_t, max_nodes, mtGC);
+    z_numa_id_to_node = NEW_C_HEAP_ARRAY(uint, (size_t)max_nodes, mtGC);
+    configured_nodes = (uint32_t)os::numa_get_leaf_groups(z_numa_id_to_node, 0);
+
+    z_node_to_numa_id = NEW_C_HEAP_ARRAY(uint32_t, (size_t)max_nodes, mtGC);
 
     // Fill the array with invalid NUMA ids
-    for (uint32_t i = 0; i < max_nodes; i++) {
+    for (uint32_t i = 0; i < (size_t)max_nodes; i++) {
       z_node_to_numa_id[i] = (uint32_t)-1;
     }
 
     // Fill the reverse mappings
     for (uint32_t i = 0; i < configured_nodes; i++) {
       z_node_to_numa_id[z_numa_id_to_node[i]] = i;
     }
+
+    _node_count = (uint32_t)max_nodes;
+    _bound_node_count = configured_nodes;
+  } else {
+    _node_count = 1;
+    _bound_node_count = 1;
   }
 
   // UseNUMA and is_faked() are mutually excluded in zArguments.cpp.

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp

@@ -22,6 +22,7 @@
  */
 
 #include "gc/shared/gcLogPrecious.hpp"
+#include "gc/z/zAdaptiveHeap.inline.hpp"
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zErrno.hpp"
@@ -35,6 +36,8 @@
 #include "hugepages.hpp"
 #include "logging/log.hpp"
 #include "os_linux.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
 #include "runtime/init.hpp"
 #include "runtime/os.hpp"
 #include "runtime/safefetch.hpp"
@@ -116,6 +119,8 @@ static const char* ZPreferredHugetlbfsMountpoints[] = {
 static int z_fallocate_hugetlbfs_attempts = 3;
 static bool z_fallocate_supported = true;
 
+static size_t z_max_map_count = 0;
+
 ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity)
   : _fd(-1),
     _filesystem(0),
@@ -151,6 +156,14 @@ ZPhysicalMemoryBacking::ZPhysicalMemoryBacking(size_t max_capacity)
   _block_size = buf.f_bsize;
   _available = buf.f_bavail * _block_size;
 
+  // Note that the available space on a tmpfs or a hugetlbfs filesystem
+  // will be zero if no size limit was specified when it was mounted.
+  if (_available == 0) {
+    log_info_p(gc, init)("Available space on backing filesystem: N/A");
+  } else {
+    log_info_p(gc, init)("Available space on backing filesystem: %zuM", _available / M);
+  }
+
   log_info_p(gc, init)("Heap Backing Filesystem: %s (" UINT64_FORMAT_X ")",
                        is_tmpfs() ? ZFILESYSTEM_TMPFS : is_hugetlbfs() ? ZFILESYSTEM_HUGETLBFS : "other", _filesystem);
 
@@ -308,12 +321,9 @@ void ZPhysicalMemoryBacking::warn_available_space(size_t max_capacity) const {
   // will be zero if no size limit was specified when it was mounted.
   if (_available == 0) {
     // No size limit set, skip check
-    log_info_p(gc, init)("Available space on backing filesystem: N/A");
     return;
   }
 
-  log_info_p(gc, init)("Available space on backing filesystem: %zuM", _available / M);
-
   // Warn if the filesystem doesn't currently have enough space available to hold
   // the max heap size. The max heap size will be capped if we later hit this limit
   // when trying to expand the heap.
@@ -328,7 +338,11 @@ void ZPhysicalMemoryBacking::warn_available_space(size_t max_capacity) const {
   }
 }
 
-void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const {
+void ZPhysicalMemoryBacking::compute_max_map_count() const {
+  if (z_max_map_count != 0) {
+    return;
+  }
+
   const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT;
   FILE* const file = os::fopen(filename, "r");
   if (file == nullptr) {
@@ -337,38 +351,50 @@ void ZPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const {
     return;
   }
 
-  size_t actual_max_map_count = 0;
-  const int result = fscanf(file, "%zu", &actual_max_map_count);
+  const int result = fscanf(file, "%zu", &z_max_map_count);
   fclose(file);
   if (result != 1) {
     // Failed to read file, skip check
     log_debug_p(gc, init)("Failed to read %s", filename);
     return;
   }
+}
+
+void ZPhysicalMemoryBacking::warn_max_map_count(size_t expected_capacity, size_t max_capacity) const {
+  // In the worst case, ZGC needs 2 mappings per granule. The reason is that if the
+  // fragmentation is at maximum, the mappings will be interleaved between being
+  // backed by memory vs not being backed by memory.
+  const size_t mappings_per_granule = 2;
 
   // The required max map count is impossible to calculate exactly since subsystems
   // other than ZGC are also creating memory mappings, and we have no control over that.
   // However, ZGC tends to create the most mappings and dominate the total count.
-  // In the worst cases, ZGC will map each granule three times, i.e. once per heap view.
-  // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory.
-  const size_t required_max_map_count = (max_capacity / ZGranuleSize) * 3 * 1.2;
-  if (actual_max_map_count < required_max_map_count) {
-    log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
-    log_warning_p(gc)("The system limit on number of memory mappings per process might be too low for the given");
-    log_warning_p(gc)("max Java heap size (%zuM). Please adjust %s to allow for at",
-                      max_capacity / M, filename);
-    log_warning_p(gc)("least %zu mappings (current limit is %zu). Continuing execution "
-                      "with the current", required_max_map_count, actual_max_map_count);
-    log_warning_p(gc)("limit could lead to a premature OutOfMemoryError being thrown, due to failure to map memory.");
+  const size_t required_max_map_count = (expected_capacity / ZGranuleSize) * mappings_per_granule * 1.2;
+  if (z_max_map_count >= required_max_map_count) {
+    return;
   }
+
+  const size_t recommended_max_map_count = (max_capacity / ZGranuleSize) * mappings_per_granule * 1.2;
+
+  const char* const filename = ZFILENAME_PROC_MAX_MAP_COUNT;
+  log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+  log_warning_p(gc)("The system limit on number of memory mappings per process might be too low for the");
+  log_warning_p(gc)("Java heap size (%zuM). Please adjust %s to allow for at",
+                    expected_capacity / M, filename);
+  log_warning_p(gc)("least %zu mappings (current limit is %zu). Continuing execution "
+                    "with the current", recommended_max_map_count, z_max_map_count);
+  log_warning_p(gc)("limit could lead to a premature OutOfMemoryError being thrown, due to failure to map memory.");
 }
 
-void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
+void ZPhysicalMemoryBacking::warn_commit_limits(size_t expected_capacity, size_t max_capacity) const {
   // Warn if available space is too low
-  warn_available_space(max_capacity);
+  warn_available_space(expected_capacity);
+
+  // Compute max map count the first time (during bootstrapping)
+  compute_max_map_count();
 
   // Warn if max map count is too low
-  warn_max_map_count(max_capacity);
+  warn_max_map_count(expected_capacity, max_capacity);
 }
 
 bool ZPhysicalMemoryBacking::is_tmpfs() const {
@@ -513,7 +539,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(zbacking_offset offset, size_
   // some point touch these segments, otherwise we can not punch hole in them.
   // Also note that we need to use compat mode when using transparent huge pages,
   // since we need to use madvise(2) on the mapping before the page is allocated.
-  if (z_fallocate_supported && !ZLargePages::is_enabled()) {
+  if (z_fallocate_supported && !ZLargePages::is_explicit()) {
      const ZErrno err = fallocate_fill_hole_syscall(offset, length);
      if (!err) {
        // Success
@@ -615,8 +641,12 @@ bool ZPhysicalMemoryBacking::commit_inner(zbacking_offset offset, size_t length)
       goto retry;
     }
 
+    static Atomic<bool> warned_failed_commit{false};
+    if (warned_failed_commit.compare_exchange(false, true) == false) {
+      log_error_p(gc)("Failed to commit memory (%s)", err.to_string());
+    }
+
     // Failed
-    log_error_p(gc)("Failed to commit memory (%s)", err.to_string());
     return false;
   }
 
@@ -705,3 +735,7 @@ void ZPhysicalMemoryBacking::unmap(zaddress_unsafe addr, size_t size) const {
     fatal("Failed to map memory (%s)", err.to_string());
   }
 }
+
+void ZPhysicalMemoryBacking::collapse(zaddress_unsafe addr, size_t size) const {
+  os::Linux::madvise_collapse_transparent_huge_pages((void*)untype(addr), size);
+}

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp

@@ -31,14 +31,14 @@ class ZErrno;
 class ZPhysicalMemoryBacking {
 private:
   int      _fd;
-  size_t   _size;
   uint64_t _filesystem;
   size_t   _block_size;
   size_t   _available;
   bool     _initialized;
 
   void warn_available_space(size_t max_capacity) const;
-  void warn_max_map_count(size_t max_capacity) const;
+  void compute_max_map_count() const;
+  void warn_max_map_count(size_t warn_capacity, size_t max_capacity) const;
 
   int create_mem_fd(const char* name) const;
   int create_file_fd(const char* name) const;
@@ -67,13 +67,15 @@ class ZPhysicalMemoryBacking {
 
   bool is_initialized() const;
 
-  void warn_commit_limits(size_t max_capacity) const;
+  void warn_commit_limits(size_t expected_capacity, size_t max_capacity) const;
 
   size_t commit(zbacking_offset offset, size_t length, uint32_t numa_id) const;
   size_t uncommit(zbacking_offset offset, size_t length) const;
 
   void map(zaddress_unsafe addr, size_t size, zbacking_offset offset) const;
   void unmap(zaddress_unsafe addr, size_t size) const;
+
+  void collapse(zaddress_unsafe addr, size_t size) const;
 };
 
 #endif // OS_LINUX_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_HPP

src/hotspot/os/linux/osContainer_linux.cpp

@@ -90,11 +90,11 @@ void OSContainer::init() {
     //  1.) On a physical Linux system without any limit
     //  2.) On a physical Linux system with a limit enforced by other means (like systemd slice)
     physical_memory_size_type mem_limit_val;
-    any_mem_cpu_limit_present = any_mem_cpu_limit_present || memory_limit_in_bytes(mem_limit_val);
-    any_mem_cpu_limit_present = any_mem_cpu_limit_present || memory_throttle_limit_in_bytes(mem_limit_val);
-    any_mem_cpu_limit_present = any_mem_cpu_limit_present || memory_soft_limit_in_bytes(mem_limit_val);
+    any_mem_cpu_limit_present = any_mem_cpu_limit_present || (memory_limit_in_bytes(mem_limit_val) && mem_limit_val != value_unlimited);
+    any_mem_cpu_limit_present = any_mem_cpu_limit_present || (memory_throttle_limit_in_bytes(mem_limit_val) && mem_limit_val != value_unlimited);
+    any_mem_cpu_limit_present = any_mem_cpu_limit_present || (memory_soft_limit_in_bytes(mem_limit_val) && mem_limit_val != value_unlimited && mem_limit_val != 0);
     double cpus;
-    any_mem_cpu_limit_present = any_mem_cpu_limit_present || active_processor_count(cpus);
+    any_mem_cpu_limit_present = any_mem_cpu_limit_present || (active_processor_count(cpus) && cpus < double(os::Linux::active_processor_count()));
     if (any_mem_cpu_limit_present) {
       reason = " because either a cpu or a memory limit is present";
     } else {

src/hotspot/os/linux/os_linux.cpp

@@ -3265,6 +3265,8 @@ bool os::Linux::libnuma_init() {
                                                    libnuma_dlsym(handle, "numa_num_configured_nodes")));
       set_numa_available(CAST_TO_FN_PTR(numa_available_func_t,
                                         libnuma_dlsym(handle, "numa_available")));
+      set_numa_node_size64(CAST_TO_FN_PTR(numa_node_size64_func_t,
+                                          libnuma_dlsym(handle, "numa_node_size64")));
       set_numa_tonode_memory(CAST_TO_FN_PTR(numa_tonode_memory_func_t,
                                             libnuma_dlsym(handle, "numa_tonode_memory")));
       set_numa_interleave_memory(CAST_TO_FN_PTR(numa_interleave_memory_func_t,
@@ -3537,6 +3539,7 @@ os::Linux::numa_node_to_cpus_func_t os::Linux::_numa_node_to_cpus;
 os::Linux::numa_node_to_cpus_v2_func_t os::Linux::_numa_node_to_cpus_v2;
 os::Linux::numa_max_node_func_t os::Linux::_numa_max_node;
 os::Linux::numa_num_configured_nodes_func_t os::Linux::_numa_num_configured_nodes;
+os::Linux::numa_node_size64_func_t os::Linux::_numa_node_size64;
 os::Linux::numa_available_func_t os::Linux::_numa_available;
 os::Linux::numa_tonode_memory_func_t os::Linux::_numa_tonode_memory;
 os::Linux::numa_interleave_memory_func_t os::Linux::_numa_interleave_memory;