Copybara import of the project:

bench/BUILD.bazel

@@ -172,6 +172,7 @@ xnnpack_cxx_library(
     ],
 ) for kernel in [
     "bf16_gemm",
+    "qd8_bf16_qb4w_gemm",
     "qd8_f16_qb4w_gemm",
     "qd8_f32_qb4w_gemm",
     "qd8_f16_qc8w_gemm",

bench/CMakeLists.txt

@@ -74,6 +74,7 @@ SET(MICROKERNEL_BENCHMARKS
     pf16-gemm-minmax
     pf32-gemm-minmax
     qb4-packw
+    qd8-bf16-qb4w-gemm
     qd8-f16-qb4w-gemm
     qd8-f16-qc4w-gemm
     qd8-f16-qc8w-gemm

bench/gemm-benchmark.cc

@@ -627,6 +627,96 @@ void GEMMBenchmark(benchmark::State& state,
                          benchmark::Counter::kIsRate);
 }
 
+void GEMMBenchmark(benchmark::State& state,
+                   xnn_qd8_bf16_qb4w_gemm_ukernel_fn gemm,
+                   xnn_init_bf16_qb4w_minmax_params_fn init_params,
+                   xnn_pack_qs8_qb4w_gemm_fn pack, size_t mr, size_t nr,
+                   size_t kr, size_t sr,
+                   uint64_t arch_flags) {
+  if (!benchmark::utils::CheckArchFlags(state, arch_flags)) {
+    return;
+  }
+
+  const size_t mc = state.range(0);
+  const size_t nc = state.range(1);
+  const size_t bl = state.range(3);
+  const size_t kc = round_up(state.range(2), bl);
+
+  xnnpack::ReplicableRandomDevice rng;
+  auto scalerng = std::bind(std::uniform_real_distribution<float>(0.5f, 2.f),
+                            std::ref(rng));
+
+  const size_t planes = 2;  // 4 bit is 2 planes - low nibbles and high nibbles
+  const size_t k2 = round_up_po2(kc, 2);  // tester assumes byte aligned rows
+  const size_t packed_k2 =
+      round_up_po2(kc, kr * sr * planes);  // 2 blocks for nibbles
+
+  const size_t packed_k_bytes = (packed_k2 + 1) / 2;
+  const size_t num_blocks = packed_k2 / bl;
+  const size_t packed_n = round_up_po2(nc, nr);
+
+  xnnpack::Buffer<int8_t> a(mc * kc, xnnpack::XnnExtraBytes);
+  xnnpack::fill_uniform_random_bits(a.data(), a.size(), rng);
+  xnnpack::Buffer<uint8_t> k(nc * kc / 2);
+  xnnpack::fill_uniform_random_bits(k.data(), k.size(), rng);
+  xnnpack::Buffer<xnn_bfloat16> kernel_scale2d(nc * k2 / bl);
+  xnnpack::fill_uniform_random_bits(k.data(), k.size(), rng);
+  std::generate(kernel_scale2d.begin(), kernel_scale2d.end(),
+                [&]() { return scalerng(); });
+
+  xnnpack::Buffer<xnn_qd8_quantization_params> quantization_params(
+      mc + XNN_EXTRA_QUANTIZATION_PARAMS);
+  const size_t w_bytes = packed_n * packed_k_bytes +
+                         /* vksum */ packed_n * sizeof(float) +
+                         /* scales */ packed_n * num_blocks * sizeof(float) +
+                         /* bias */ packed_n * sizeof(float);
+
+  const size_t c_elements = mc * nc;
+  const size_t num_buffers =
+      1 + benchmark::utils::DivideRoundUp<size_t>(
+              benchmark::utils::GetMaxCacheSize(),
+              w_bytes + sizeof(xnn_bfloat16) * c_elements);
+
+  xnnpack::Buffer<char, XNN_ALLOCATION_ALIGNMENT> w(w_bytes * num_buffers);
+
+  const xnn_qs8_qc4w_packing_params packing_params = {/*input_zero_point=*/1,
+                                                      /*kernel_zero_point=*/8};
+  pack(1, nc, k2, nr, kr, sr, bl, k.data(), /*bias=*/nullptr,
+       /*scale=*/kernel_scale2d.data(), w.data(), sizeof(float) * nr,
+       sizeof(float) * nr, &packing_params);
+  xnnpack::Buffer<xnn_bfloat16> c(c_elements * num_buffers);
+
+  // Prepare parameters.
+  xnn_bf16_qb4w_minmax_params params;
+  init_params(&params, std::numeric_limits<int8_t>::min(),
+              std::numeric_limits<int8_t>::max(), 8, bl);
+
+  size_t buffer_index = 0;
+  for (auto _ : state) {
+    state.PauseTiming();
+    benchmark::utils::PrefetchToL1(a.data(), a.size());
+    buffer_index = (buffer_index + 1) % num_buffers;
+    state.ResumeTiming();
+
+    for (uint32_t m = 0; m < mc; m += mr) {
+      const uint32_t mb = min(mc - m, mr);
+      gemm(mb, nc, kc * sizeof(int8_t), a.data() + m * kc, kc * sizeof(int8_t),
+           w.data() + w_bytes * buffer_index,
+           c.data() + (buffer_index * mc + m) * nc, nc * sizeof(xnn_bfloat16),
+           nr * sizeof(xnn_bfloat16), &params, quantization_params.data() + m);
+    }
+  }
+
+  const uint64_t cpu_frequency = benchmark::utils::GetCurrentCpuFrequency();
+  if (cpu_frequency != 0) {
+    state.counters["cpufreq"] = cpu_frequency;
+  }
+
+  state.counters["OPS"] =
+      benchmark::Counter(uint64_t(state.iterations()) * 2 * mc * nc * kc,
+                         benchmark::Counter::kIsRate);
+}
+
 void GEMMBenchmark(benchmark::State& state,
                    xnn_qd8_f16_qc4w_gemm_ukernel_fn gemm,
                    xnn_init_f16_qc4w_minmax_params_fn init_params,

bench/gemm-benchmark.h

@@ -79,6 +79,13 @@ void GEMMBenchmark(benchmark::State& state,
                    size_t kr, size_t sr,
                    uint64_t arch_flags = 0);
 
+void GEMMBenchmark(benchmark::State& state,
+                   xnn_qd8_bf16_qb4w_gemm_ukernel_fn gemm,
+                   xnn_init_bf16_qb4w_minmax_params_fn init_params,
+                   xnn_pack_qs8_qb4w_gemm_fn pack, size_t mr, size_t nr,
+                   size_t kr, size_t sr,
+                   uint64_t arch_flags = 0);
+
 void GEMMBenchmark(benchmark::State& state,
                    xnn_qd8_f16_qc4w_gemm_ukernel_fn gemm,
                    xnn_init_f16_qc4w_minmax_params_fn init_params,

bench/qd8-bf16-qb4w-gemm.cc

@@ -0,0 +1,110 @@
+// clang-format off
+// Copyright 2023 Google LLC
+//
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree.
+//
+// Auto-generated file. Do not edit!
+//   Specification: test/qd8-bf16-qb4w-gemm-minmax.yaml
+//   Generator: tools/generate-gemm-test.py
+
+#include <cstdint>
+#include <functional>
+
+#include <benchmark/benchmark.h>
+#include "bench/gemm-benchmark.h"
+#include "bench/utils.h"
+#include "src/xnnpack/common.h"
+#include "src/xnnpack/gemm.h"
+#include "src/xnnpack/hardware-config.h"
+#include "src/xnnpack/microfnptr.h"
+#include "src/xnnpack/microparams-init.h"
+#include "src/xnnpack/pack.h"
+#include "src/xnnpack/packw.h"
+
+namespace {
+
+
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_1x2__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_1x2__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/1, /*nr=*/2, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_1x2__scalar)
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_1x4__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_1x4__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/1, /*nr=*/4, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_1x4__scalar)
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_1x8__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_1x8__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/1, /*nr=*/8, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_1x8__scalar)
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_2x2__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_2x2__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/2, /*nr=*/2, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_2x2__scalar)
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_2x4__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_2x4__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/2, /*nr=*/4, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_2x4__scalar)
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_2x8__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_2x8__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/2, /*nr=*/8, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_2x8__scalar)
+
+static void qd8_bf16_qb4w_gemm_minmax_ukernel_4x4__scalar(benchmark::State& state) {
+  GEMMBenchmark(state,
+    xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_4x4__scalar,
+    xnn_init_bf16_qb4w_minmax_scalar_params,
+    xnn_pack_qs8_qb4w_gemm_goi_w,
+    /*mr=*/4, /*nr=*/4, /*kr=*/1, /*sr=*/1,
+    /*arch_flags=*/0);
+}
+
+BENCHMARK_GEMM(qd8_bf16_qb4w_gemm_minmax_ukernel_4x4__scalar)
+
+}  // namespace
+
+#ifndef XNNPACK_BENCHMARK_NO_MAIN
+XNN_BENCHMARK_MAIN();
+#endif

cmake/gen/scalar_microkernels.cmake

@@ -136,6 +136,8 @@ SET(PROD_SCALAR_MICROKERNEL_SRCS
   src/f32-vunary/gen/f32-vabs-scalar.c
   src/f32-vunary/gen/f32-vneg-scalar.c
   src/f32-vunary/gen/f32-vsqr-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x4-minmax-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-4x4-minmax-scalar.c
   src/qd8-f16-qc2w-gemm/gen/qd8-f16-qc2w-gemm-1x2-minmax-scalar.c
   src/qd8-f32-qb4w-gemm/gen/qd8-f32-qb4w-gemm-1x4-minmax-scalar.c
   src/qd8-f32-qb4w-gemm/gen/qd8-f32-qb4w-gemm-4x4-minmax-scalar.c
@@ -539,6 +541,11 @@ SET(NON_PROD_SCALAR_MICROKERNEL_SRCS
   src/f32-vsigmoid/gen/f32-vsigmoid-scalar-rr2-p5-div-u4.c
   src/qb4-packw/gen/qb4-packw-x16c4-gemm-goi-scalar.c
   src/qb4-packw/gen/qb4-packw-x16c8-gemm-goi-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x2-minmax-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x8-minmax-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x2-minmax-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x4-minmax-scalar.c
+  src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x8-minmax-scalar.c
   src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-1x2-minmax-scalar.c
   src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-1x4-minmax-scalar.c
   src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-1x8-minmax-scalar.c

gen/scalar_microkernels.bzl

@@ -132,6 +132,8 @@ PROD_SCALAR_MICROKERNEL_SRCS = [
     "src/f32-vunary/gen/f32-vabs-scalar.c",
     "src/f32-vunary/gen/f32-vneg-scalar.c",
     "src/f32-vunary/gen/f32-vsqr-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x4-minmax-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-4x4-minmax-scalar.c",
     "src/qd8-f16-qc2w-gemm/gen/qd8-f16-qc2w-gemm-1x2-minmax-scalar.c",
     "src/qd8-f32-qb4w-gemm/gen/qd8-f32-qb4w-gemm-1x4-minmax-scalar.c",
     "src/qd8-f32-qb4w-gemm/gen/qd8-f32-qb4w-gemm-4x4-minmax-scalar.c",
@@ -536,6 +538,11 @@ NON_PROD_SCALAR_MICROKERNEL_SRCS = [
     "src/f32-vsigmoid/gen/f32-vsigmoid-scalar-rr2-p5-div-u4.c",
     "src/qb4-packw/gen/qb4-packw-x16c4-gemm-goi-scalar.c",
     "src/qb4-packw/gen/qb4-packw-x16c8-gemm-goi-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x2-minmax-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x8-minmax-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x2-minmax-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x4-minmax-scalar.c",
+    "src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x8-minmax-scalar.c",
     "src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-1x2-minmax-scalar.c",
     "src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-1x4-minmax-scalar.c",
     "src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-1x8-minmax-scalar.c",

include/xnnpack.h

@@ -4353,6 +4353,35 @@ enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qb4w(
     void* workspace,
     const struct xnn_quantization_params* quantization_params);
 
+enum xnn_status xnn_create_fully_connected_nc_qd8_bf16_qb4w(
+    size_t input_channels,
+    size_t output_channels,
+    size_t input_stride,
+    size_t output_stride,
+    size_t block_size,
+    uint8_t kernel_zero_point,
+    const uint16_t* kernel_scale,
+    const void* kernel,
+    const float* bias,
+    float output_min,
+    float output_max,
+    uint32_t flags,
+    xnn_weights_cache_t weights_cache,
+    xnn_operator_t* fully_connected_op_out);
+
+enum xnn_status xnn_reshape_fully_connected_nc_qd8_bf16_qb4w(
+    xnn_operator_t fully_connected_op,
+    size_t batch_size,
+    size_t* workspace_size,
+    pthreadpool_t threadpool);
+
+enum xnn_status xnn_setup_fully_connected_nc_qd8_bf16_qb4w(
+    xnn_operator_t fully_connected_op,
+    const int8_t* input,
+    void* output,
+    void* workspace,
+    const struct xnn_quantization_params* quantization_params);
+
 enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qc4w(
   size_t input_channels,
   size_t output_channels,

scripts/generate-qs8-gemm.sh

@@ -56,6 +56,14 @@ tools/xngen src/qs8-gemm/scalar.c.in -D MR=2 -D NR=4 -D REQUANTIZATION= -D VARIA
 tools/xngen src/qs8-gemm/scalar.c.in -D MR=2 -D NR=8 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_F16 -D WASM=0 -o src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-2x8-minmax-scalar.c &
 tools/xngen src/qs8-gemm/scalar.c.in -D MR=4 -D NR=4 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_F16 -D WASM=0 -o src/qd8-f16-qb4w-gemm/gen/qd8-f16-qb4w-gemm-4x4-minmax-scalar.c &
 
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=1 -D NR=2 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x2-minmax-scalar.c &
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=1 -D NR=4 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x4-minmax-scalar.c &
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=1 -D NR=8 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-1x8-minmax-scalar.c &
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=2 -D NR=2 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x2-minmax-scalar.c &
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=2 -D NR=4 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x4-minmax-scalar.c &
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=2 -D NR=8 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-2x8-minmax-scalar.c &
+tools/xngen src/qs8-gemm/scalar.c.in -D MR=4 -D NR=4 -D REQUANTIZATION= -D VARIANT= -D DATATYPE=QB4_BF16 -D WASM=0 -o src/qd8-bf16-qb4w-gemm/gen/qd8-bf16-qb4w-gemm-4x4-minmax-scalar.c &
+
 tools/xngen src/qs8-gemm/scalar.c.in -D MR=1 -D NR=2 -D REQUANTIZATION=FP32 -D VARIANT=FMAGIC -D DATATYPE=QC8 -D WASM=0 -o src/qs8-qc8w-gemm/gen/qs8-qc8w-gemm-1x2-minmax-fp32-scalar-fmagic.c &
 tools/xngen src/qs8-gemm/scalar.c.in -D MR=2 -D NR=2 -D REQUANTIZATION=FP32 -D VARIANT=FMAGIC -D DATATYPE=QC8 -D WASM=0 -o src/qs8-qc8w-gemm/gen/qs8-qc8w-gemm-2x2-minmax-fp32-scalar-fmagic.c &
 tools/xngen src/qs8-gemm/scalar.c.in -D MR=3 -D NR=2 -D REQUANTIZATION=FP32 -D VARIANT=FMAGIC -D DATATYPE=QC8 -D WASM=0 -o src/qs8-qc8w-gemm/gen/qs8-qc8w-gemm-3x2-minmax-fp32-scalar-fmagic.c &

src/configs/gemm-config.c

@@ -50,6 +50,7 @@ static struct xnn_gemm_config qd8_f16_qb4w_gemm_config = {0};
 static struct xnn_gemm_config qd8_f16_qc4w_gemm_config = {0};
 static struct xnn_gemm_config qd8_f16_qc8w_gemm_config = {0};
 static struct xnn_gemm_config qd8_f16_qc8w_igemm_config = {0};
+static struct xnn_gemm_config qd8_bf16_qb4w_gemm_config = {0};
 static struct xnn_gemm_config qd8_f32_qb4w_gemm_config = {0};
 static struct xnn_gemm_config qd8_f32_qc4w_gemm_config = {0};
 static struct xnn_gemm_config qd8_f32_qc2w_gemm_config = {0};
@@ -82,6 +83,7 @@ XNN_INIT_ONCE_GUARD(pqs8_qc8w_gemm);
 XNN_INIT_ONCE_GUARD(qd8_f16_qc2w_gemm);
 XNN_INIT_ONCE_GUARD(qdu8_f16_qc2w_gemm);
 XNN_INIT_ONCE_GUARD(qd8_f16_qb4w_gemm);
+XNN_INIT_ONCE_GUARD(qd8_bf16_qb4w_gemm);
 XNN_INIT_ONCE_GUARD(qd8_f16_qc4w_gemm);
 XNN_INIT_ONCE_GUARD(qd8_f16_qc8w_gemm);
 XNN_INIT_ONCE_GUARD(qd8_f16_qc8w_igemm);
@@ -2501,6 +2503,27 @@ static void init_qd8_f16_qb4w_gemm_config(void) {
   assert(qd8_f16_qb4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1));
 }
 
+static void init_qd8_bf16_qb4w_gemm_config(void) {
+  // Common parameters.
+  qd8_bf16_qb4w_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_INT8_T;
+  qd8_bf16_qb4w_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_UINT8_T;
+  qd8_bf16_qb4w_gemm_config.log2_filter_element_bit_size = XNN_LOG2_BIT_SIZEOF_INT4;
+  qd8_bf16_qb4w_gemm_config.bias_element_size = sizeof(float);
+  qd8_bf16_qb4w_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_qb4_weights_and_biases;
+  qd8_bf16_qb4w_gemm_config.pack_weights_and_biases = xnn_pack_qb4_weights_and_biases;
+
+  // Scalar-only for now.
+  qd8_bf16_qb4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_1x4__scalar);
+  qd8_bf16_qb4w_gemm_config.minmax.dqgemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_DQGEMM_UKERNEL(xnn_qd8_bf16_qb4w_gemm_minmax_ukernel_4x4__scalar);
+  qd8_bf16_qb4w_gemm_config.init.bf16_qb4w = xnn_init_bf16_qb4w_minmax_scalar_params;
+  qd8_bf16_qb4w_gemm_config.mr = 4;
+  qd8_bf16_qb4w_gemm_config.nr = 4;
+  qd8_bf16_qb4w_gemm_config.planes = 2;
+
+  assert(qd8_bf16_qb4w_gemm_config.mr <= XNN_MAX_MR);
+  assert(qd8_bf16_qb4w_gemm_config.mr <= (XNN_EXTRA_QUANTIZATION_PARAMS + 1));
+}
+
 static void init_qd8_f32_qc2w_gemm_config(void) {
   // Common parameters.
   qd8_f32_qc2w_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_INT8_T;
@@ -6278,6 +6301,14 @@ const struct xnn_gemm_config* xnn_init_qd8_f16_qb4w_gemm_config() {
   return qd8_f16_qb4w_gemm_config.arch ? &qd8_f16_qb4w_gemm_config : NULL;
 }
 
+const struct xnn_gemm_config* xnn_init_qd8_bf16_qb4w_gemm_config() {
+  if (xnn_init_hardware_config() == NULL) {
+    return NULL;
+  }
+  XNN_INIT_ONCE(qd8_bf16_qb4w_gemm);
+  return &qd8_bf16_qb4w_gemm_config;
+}
+
 const struct xnn_gemm_config* xnn_init_qd8_f32_qc4w_gemm_config() {
   if (xnn_init_hardware_config() == NULL) {
     return NULL;