Add alloc_align

Signed-off-by: Petr Shumilov <p.shumilov@vkteam.ru>

Author: PetrShumilov

runtime-common/core/allocator/script-malloc-interface.h

@@ -19,23 +19,92 @@ namespace memory {
 
 namespace script {
 
-constexpr int64_t MALLOC_REPLACER_SIZE_OFFSET = sizeof(size_t);
-constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00;
+constexpr uint64_t MALLOC_REPLACER_MAX_ALLOC = 0xFFFFFF00; // 4GiB
+
+namespace details {
+struct control_block {
+  static constexpr auto SIZE_FIELD_BITSIZE{48};
+  static constexpr auto BASE_OFFSET_FIELD_BITSIZE{16};
+  static constexpr uint64_t BLOCK_SIZE_MASK{(1UL << SIZE_FIELD_BITSIZE) - 1};
+  static constexpr uint64_t BASE_OFFSET_MASK{(1UL << BASE_OFFSET_FIELD_BITSIZE) - 1};
+
+  static constexpr uint64_t max_size() noexcept {
+    return 1UL << SIZE_FIELD_BITSIZE;
+  }
+
+  static constexpr uint64_t max_alignment() noexcept {
+    return 1UL << BASE_OFFSET_FIELD_BITSIZE;
+  }
+
+  uint64_t raw() const noexcept {
+    return (static_cast<std::uint64_t>(base_offset) << SIZE_FIELD_BITSIZE) | (static_cast<std::uint64_t>(size) & BLOCK_SIZE_MASK);
+  }
+
+  static control_block from_raw(std::uint64_t raw) noexcept {
+    return control_block{.size = raw & BLOCK_SIZE_MASK, .base_offset = static_cast<uint16_t>((raw >> SIZE_FIELD_BITSIZE) & BASE_OFFSET_MASK)};
+  }
+
+  uint64_t size : SIZE_FIELD_BITSIZE;
+  uint16_t base_offset : BASE_OFFSET_FIELD_BITSIZE;
+};
+
+inline bool is_power_of_2(uint64_t v) noexcept {
+  return v && !(v & (v - 1));
+}
+
+static_assert(sizeof(control_block) == sizeof(uint64_t), "Control block's size must be equal to uint64");
+
+} // namespace details
 
 inline void* alloc(size_t size) noexcept {
-  if (unlikely(size > MALLOC_REPLACER_MAX_ALLOC - MALLOC_REPLACER_SIZE_OFFSET)) {
-    php_warning("attempt to allocate too much memory by malloc replacer : %lu", size);
+  const size_t cb_size{sizeof(details::control_block)};
+  if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - cb_size)) {
+    php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size);
+    return nullptr;
+  }
+  const size_t total_size{size + cb_size};
+  void* base{RuntimeAllocator::get().alloc_script_memory(total_size)};
+  if (unlikely(base == nullptr)) {
+    php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size);
+    return base;
+  }
+  *(reinterpret_cast<std::uint64_t*>(base)) = details::control_block{.size = total_size, .base_offset = cb_size}.raw();
+  return reinterpret_cast<void*>(reinterpret_cast<uint64_t>(base) + cb_size); // NOLINT
+}
+
+inline void* alloc_aligned(size_t size, std::align_val_t alignment) noexcept {
+  // Check that provided alignment is power of two
+  const size_t align{static_cast<uint64_t>(alignment)};
+  if (unlikely(align == 0 || !details::is_power_of_2(align) || align >= details::control_block::max_alignment())) {
+    php_warning("allocation alignment have to be non-zero power of two and not greater than %" PRIi64 ", got : %lu", details::control_block::max_alignment(), align);
     return nullptr;
   }
-  const size_t real_size{size + MALLOC_REPLACER_SIZE_OFFSET};
-  void* ptr{RuntimeAllocator::get().alloc_script_memory(real_size)};
 
-  if (unlikely(ptr == nullptr)) {
-    php_warning("not enough script memory to allocate: %lu", size);
-    return ptr;
+  // Check that memory is enough
+  const size_t cb_size{sizeof(details::control_block)};
+  if (unlikely(size > std::min(details::control_block::max_size(), MALLOC_REPLACER_MAX_ALLOC) - (align - 1) - cb_size)) {
+    php_warning("attempt to allocate too much memory by malloc replacer, requested : %lu", size);
+    return nullptr;
+  }
+
+  // Request mem from underlying memory manager
+  const size_t total_size{size + (align - 1) + cb_size};
+  void* base{RuntimeAllocator::get().alloc_script_memory(total_size)};
+  if (unlikely(base == nullptr)) {
+    php_warning("not enough script memory to allocate, requested : %lu, actual requested: %lu", size, total_size);
+    return base;
   }
-  *static_cast<size_t*>(ptr) = real_size;
-  return static_cast<std::byte*>(ptr) + MALLOC_REPLACER_SIZE_OFFSET;
+
+  const uint64_t base_u{reinterpret_cast<uint64_t>(base)};
+  // The smallest multiple of `align` greater than or equal to requested memory
+  const uint64_t aligned_u{((base_u + cb_size) + (align - 1)) & ~(align - 1)};
+  const uint64_t base_offset_u{aligned_u - base_u};
+
+  // Save control block
+  *(reinterpret_cast<std::uint64_t*>(aligned_u - cb_size)) = // NOLINT
+      details::control_block{.size = total_size, .base_offset = static_cast<std::uint16_t>(base_offset_u)}.raw();
+
+  return reinterpret_cast<void*>(aligned_u); // NOLINT
 }
 
 inline void* calloc(size_t num, size_t size) noexcept {
@@ -47,10 +116,17 @@ inline void* calloc(size_t num, size_t size) noexcept {
 }
 
 inline void free(void* ptr) noexcept {
-  if (likely(ptr != nullptr)) {
-    void* real_ptr{static_cast<std::byte*>(ptr) - MALLOC_REPLACER_SIZE_OFFSET};
-    RuntimeAllocator::get().free_script_memory(real_ptr, *static_cast<size_t*>(real_ptr));
+  if (unlikely(ptr == nullptr)) {
+    return;
   }
+
+  const size_t cb_size{sizeof(details::control_block)};
+  const auto mem{reinterpret_cast<uint64_t>(ptr)};
+
+  const auto cb{details::control_block::from_raw(*reinterpret_cast<uint64_t*>(mem - cb_size))}; // NOLINT
+  void* base{reinterpret_cast<void*>(mem - cb.base_offset)};                                    // NOLINT
+
+  RuntimeAllocator::get().free_script_memory(base, cb.size);
 }
 
 inline void* realloc(void* ptr, size_t new_size) noexcept {
@@ -63,13 +139,18 @@ inline void* realloc(void* ptr, size_t new_size) noexcept {
     return nullptr;
   }
 
-  void* real_ptr{static_cast<std::byte*>(ptr) - sizeof(size_t)};
-  const size_t old_size{*static_cast<size_t*>(real_ptr)};
+  const size_t cb_size{sizeof(details::control_block)};
+  const auto mem{reinterpret_cast<uint64_t>(ptr)};
+
+  const auto cb{details::control_block::from_raw(*reinterpret_cast<uint64_t*>(mem - cb_size))}; // NOLINT
+
+  void* old_base{reinterpret_cast<void*>(mem - cb.base_offset)}; // NOLINT
+  const size_t old_size{cb.size};
 
   void* new_ptr{kphp::memory::script::alloc(new_size)};
   if (likely(new_ptr != nullptr)) {
     std::memcpy(new_ptr, ptr, std::min(new_size, old_size));
-    RuntimeAllocator::get().free_script_memory(real_ptr, old_size);
+    RuntimeAllocator::get().free_script_memory(old_base, old_size);
   }
   return new_ptr;
 }