perf: optimize Highway row kernels

Author: msg7086

src/core/delogo_processor.cpp

@@ -61,6 +61,7 @@ void process_opaque_row(
   std::span<Pixel> row,
   std::span<const int> colors,
   std::span<const int> depths,
+  std::span<const std::uint32_t> erase_reciprocals,
   int bit_depth
 ) {
   if (backend == RowKernelBackend::Scalar) {
@@ -75,7 +76,7 @@ void process_opaque_row(
   if (operation == LogoOperation::Add) {
     process_add_row_hwy(row, colors, depths, bit_depth);
   } else {
-    process_erase_row_hwy(row, colors, depths, bit_depth);
+    process_erase_row_hwy(row, colors, depths, erase_reciprocals, bit_depth);
   }
 }
 
@@ -187,6 +188,7 @@ void DelogoProcessor::process_plane(
       row,
       coefficients.c_row(i).first(row_width),
       coefficients.d_row(i).first(row_width),
+      coefficients.d_reciprocal_row(i).first(row_width),
       bit_depth_
     );
   }

src/core/prepared_logo.cpp

@@ -2,6 +2,7 @@
 
 #include "core/canonical_math.hpp"
 #include "core/delogo_processor.hpp"
+#include "core/reciprocal_math.hpp"
 
 #include <algorithm>
 #include <optional>
@@ -22,6 +23,14 @@ int chroma_color_for_block(std::int64_t weighted_color, int sum_depth) {
   return round_divide_signed(weighted_color, sum_depth);
 }
 
+std::uint32_t erase_reciprocal_for_depth(int depth) {
+  int alpha = clamp_logo_depth(depth);
+  if (alpha >= LOGO_MAX_DP) {
+    alpha = LOGO_MAX_DP - 1;
+  }
+  return kLogoReciprocalTable32[static_cast<std::size_t>(LOGO_MAX_DP - alpha)];
+}
+
 std::optional<LogoImage> shift_logo(LogoImage image, int left, int top) {
   const int oldl = image.header.x + left;
   const int oldt = image.header.y + top;
@@ -126,10 +135,11 @@ void LogoPlaneCoefficients::reset(int width, int height) {
   height_ = height;
   c_.reset(static_cast<std::size_t>(width_) * height_);
   d_.reset(static_cast<std::size_t>(width_) * height_);
+  d_reciprocal_.reset(static_cast<std::size_t>(width_) * height_);
 }
 
 bool LogoPlaneCoefficients::active() const noexcept {
-  return !c_.empty() && !d_.empty();
+  return !c_.empty() && !d_.empty() && !d_reciprocal_.empty();
 }
 
 int LogoPlaneCoefficients::width() const noexcept {
@@ -154,6 +164,13 @@ std::span<int> LogoPlaneCoefficients::d_row(int y) noexcept {
   );
 }
 
+std::span<std::uint32_t> LogoPlaneCoefficients::d_reciprocal_row(int y) noexcept {
+  return d_reciprocal_.subspan(
+    static_cast<std::size_t>(y) * width_,
+    static_cast<std::size_t>(width_)
+  );
+}
+
 std::span<const int> LogoPlaneCoefficients::c_row(int y) const noexcept {
   return c_.subspan(
     static_cast<std::size_t>(y) * width_,
@@ -168,6 +185,13 @@ std::span<const int> LogoPlaneCoefficients::d_row(int y) const noexcept {
   );
 }
 
+std::span<const std::uint32_t> LogoPlaneCoefficients::d_reciprocal_row(int y) const noexcept {
+  return d_reciprocal_.subspan(
+    static_cast<std::size_t>(y) * width_,
+    static_cast<std::size_t>(width_)
+  );
+}
+
 PreparedLogo::PreparedLogo(const DelogoProcessorConfig& config)
   : subsampling_w_(config.subsampling_w),
     subsampling_h_(config.subsampling_h),
@@ -214,6 +238,7 @@ void PreparedLogo::convert(LogoImage& image, bool mono) {
   for (int y = 0; y < logo_header_.h; ++y) {
     auto y_c = planes_[0].c_row(y);
     auto y_d = planes_[0].d_row(y);
+    auto y_d_reciprocal = planes_[0].d_reciprocal_row(y);
     for (int x = 0; x < logo_header_.w; ++x) {
       const auto index = (static_cast<std::size_t>(y) * logo_header_.w) + x;
       LOGO_PIXEL& pixel = image.pixels[index];
@@ -225,6 +250,7 @@ void PreparedLogo::convert(LogoImage& image, bool mono) {
       pixel.dp_y = static_cast<int16_t>(clamp_logo_depth(pixel.dp_y));
       y_c[x] = luma_to_internal_color(pixel.y);
       y_d[x] = pixel.dp_y;
+      y_d_reciprocal[x] = erase_reciprocal_for_depth(pixel.dp_y);
       if (mono) {
         pixel.cb = 0;
         pixel.cr = 0;
@@ -245,8 +271,10 @@ void PreparedLogo::convert(LogoImage& image, bool mono) {
     const int dst_y = y / hstep;
     auto u_c_row = planes_[1].c_row(dst_y);
     auto u_d_row = planes_[1].d_row(dst_y);
+    auto u_d_reciprocal_row = planes_[1].d_reciprocal_row(dst_y);
     auto v_c_row = planes_[2].c_row(dst_y);
     auto v_d_row = planes_[2].d_row(dst_y);
+    auto v_d_reciprocal_row = planes_[2].d_reciprocal_row(dst_y);
 
     for (int x = 0; x < logo_header_.w; x += wstep) {
       const int dst_x = x / wstep;
@@ -281,6 +309,8 @@ void PreparedLogo::convert(LogoImage& image, bool mono) {
       v_c_row[dst_x] = chroma_to_internal_color(vc);
       u_d_row[dst_x] = ud;
       v_d_row[dst_x] = vd;
+      u_d_reciprocal_row[dst_x] = erase_reciprocal_for_depth(ud);
+      v_d_reciprocal_row[dst_x] = erase_reciprocal_for_depth(vd);
     }
   }
 }

src/core/prepared_logo.hpp

@@ -4,6 +4,7 @@
 #include "core/logo_file.hpp"
 
 #include <cstddef>
+#include <cstdint>
 #include <span>
 
 namespace delogohd::core {
@@ -22,14 +23,17 @@ class LogoPlaneCoefficients {
   [[nodiscard]] int height() const noexcept;
   std::span<int> c_row(int y) noexcept;
   std::span<int> d_row(int y) noexcept;
+  std::span<std::uint32_t> d_reciprocal_row(int y) noexcept;
   [[nodiscard]] std::span<const int> c_row(int y) const noexcept;
   [[nodiscard]] std::span<const int> d_row(int y) const noexcept;
+  [[nodiscard]] std::span<const std::uint32_t> d_reciprocal_row(int y) const noexcept;
 
 private:
   int width_ = 0;
   int height_ = 0;
   AlignedBuffer<int, kLogoAlignment> c_;
   AlignedBuffer<int, kLogoAlignment> d_;
+  AlignedBuffer<std::uint32_t, kLogoAlignment> d_reciprocal_;
 };
 
 class PreparedLogo {

src/core/reciprocal_math.hpp

@@ -21,6 +21,20 @@ constexpr std::array<std::uint64_t, LOGO_MAX_DP + 1> make_logo_reciprocal_table(
 
 inline constexpr auto kLogoReciprocalTable = make_logo_reciprocal_table();
 
+constexpr std::array<std::uint32_t, LOGO_MAX_DP + 1> make_logo_reciprocal_table32() noexcept {
+  std::array<std::uint32_t, LOGO_MAX_DP + 1> table{};
+  // 1 << 32 does not fit in u32; the correction step still yields exact / 1.
+  table[1] = 0xFFFFFFFFu;
+  for (int denominator = 2; denominator <= LOGO_MAX_DP; ++denominator) {
+    table[denominator] = static_cast<std::uint32_t>(
+      kReciprocalScale / static_cast<std::uint64_t>(denominator)
+    );
+  }
+  return table;
+}
+
+inline constexpr auto kLogoReciprocalTable32 = make_logo_reciprocal_table32();
+
 inline std::uint64_t divide_floor_by_reciprocal(
   std::uint64_t numerator,
   int denominator

src/core/row_kernel.hpp

@@ -46,12 +46,14 @@ void process_erase_row_hwy(
   std::span<std::uint8_t> row,
   std::span<const int> colors,
   std::span<const int> depths,
+  std::span<const std::uint32_t> reciprocals,
   int bit_depth
 );
 void process_erase_row_hwy(
   std::span<std::uint16_t> row,
   std::span<const int> colors,
   std::span<const int> depths,
+  std::span<const std::uint32_t> reciprocals,
   int bit_depth
 );