gemm block quantization infrastructure for llm decoder style

CMakeLists.txt

@@ -90,6 +90,7 @@ option(NCNN_ASAN "build for address sanitizer" OFF)
 option(NCNN_BUILD_BENCHMARK "build benchmark" ON)
 option(NCNN_PYTHON "build python api" OFF)
 option(NCNN_INT8 "int8 inference" ON)
+option(NCNN_WEIGHT_QUANT "weight quantized inference" ON)
 option(NCNN_BF16 "bf16 inference" ON)
 option(NCNN_FORCE_INLINE "force inline some function" ON)
 

docs/developer-guide/operation-param-weight-table.md

@@ -131,6 +131,25 @@
 ||2|expand_c|0|
 ||3|axes|[ ]|
 |Flatten|||
+|Gemm|0|alpha|1.f|A_data B_data C_data A_data_int8_scales B_data_int8_scales/B_data_quantize_scales B_data_input_scales|
+||1|beta|1.f|
+||2|transA|0|
+||3|transB|0|
+||4|constantA|0|
+||5|constantB|0|
+||6|constantC|0|
+||7|constantM|0|
+||8|constantN|0|
+||9|constantK|0|
+||10|constant_broadcast_type_C|0|
+||11|output_N1M|0|
+||12|output_elempack|0|
+||13|output_elemtype|0|
+||14|output_transpose|0|
+||18|quantize_term|0|
+||20|constant_TILE_M|0|
+||21|constant_TILE_N|0|
+||22|constant_TILE_K|0|
 |HardSigmoid|0|alpha|0.2f||
 ||1|beta|0.5f|
 |HardSwish|0|alpha|0.2f||

docs/developer-guide/operators.md

@@ -1151,7 +1151,7 @@ y = (gemm(a, b) + c * beta) * alpha
 | 12        | output_elempack | int | 0         |                   |
 | 13        | output_elemtype | int | 0         |                   |
 | 14        | output_transpose | int| 0         |                   |
-| 18        | int8_scale_term | int | 0         |                   |
+| 18        | quantize_term | int | 0         | 0=no quant, nonzero non-block value below 400 is legacy Gemm int8 except obsolete 4/5/6, 400/401/402=int4 block32/64/128, 410/411/412=int4 block32/64/128 with input scale, 600/601/602=int6 block32/64/128, 610/611/612=int6 block32/64/128 with input scale, 800/801/802=int8 block32/64/128, 810/811/812=int8 block32/64/128 with input scale |
 | 20        | constant_TILE_M | int | 0         |                   |
 | 21        | constant_TILE_N | int | 0         |                   |
 | 22        | constant_TILE_K | int | 0         |                   |
@@ -1163,6 +1163,16 @@ y = (gemm(a, b) + c * beta) * alpha
 | C_data        | float | [1], [M] or [N] or [1, M] or [N,1] or [N, M] |
 | A_data_int8_scales| float | [M]               |
 | B_data_int8_scales| float | [1]               |
+| B_data_quantize_scales| float | [ceil(K / block_size), N] for block quantized constant B |
+| B_data_input_scales| float | [K] for block quantized constant B with input scale |
+
+For weight-only block quantized Gemm:
+
+* `constantA=0`, `constantB=1`, `transA=0`, `transB=1`
+* output is fp32 pack1 with `output_N1M=0`, `output_elempack=0`, `output_transpose=0`
+* `B_data` is tagged int8 bytes with shape `[ceil(K * weight_bits / 8), N]`
+* scales and optional input scales are raw fp32 data
+* packing is signed symmetric scale-only, no zero point
 
 # GridSample
 ```
@@ -1575,7 +1585,7 @@ y = affine(out)
 | 5         | attn_mask     | int   | 0         |                   |
 | 6         | scale         | float | 1.f / sqrt(embed_dim / num_heads) | |
 | 7         | kv_cache      | int   | 0         |                   |
-| 18        | int8_scale_term | int | 0         |                   |
+| 18        | quantize_term | int | 0         | 0=no quant, nonzero non-block value below 400 is legacy MultiHeadAttention int8 except obsolete 4/5/6, 400/401/402=int4 block32/64/128, 410/411/412=int4 block32/64/128 with input scale, 600/601/602=int6 block32/64/128, 610/611/612=int6 block32/64/128 with input scale, 800/801/802=int8 block32/64/128, 810/811/812=int8 block32/64/128 with input scale |
 
 | weight        | type  | shape                 |
 | ------------- | ----- | --------------------- |
@@ -1591,6 +1601,16 @@ y = affine(out)
 | k_weight_data_int8_scales| float | [embed_dim] |
 | v_weight_data_int8_scales| float | [embed_dim] |
 | out_weight_data_int8_scales| float | [1]      |
+| q_weight_data_quantize_scales| float | [ceil(qdim / block_size), embed_dim] for block quantized weight |
+| k_weight_data_quantize_scales| float | [ceil(kdim / block_size), embed_dim] for block quantized weight |
+| v_weight_data_quantize_scales| float | [ceil(vdim / block_size), embed_dim] for block quantized weight |
+| out_weight_data_quantize_scales| float | [ceil(embed_dim / block_size), qdim] for block quantized weight |
+| q_weight_data_input_scales| float | [qdim] for block quantized weight with input scale |
+| k_weight_data_input_scales| float | [kdim] for block quantized weight with input scale |
+| v_weight_data_input_scales| float | [vdim] for block quantized weight with input scale |
+| out_weight_data_input_scales| float | [embed_dim] for block quantized weight with input scale |
+
+Weight-only block quantized MultiHeadAttention stores q/k/v/out weights as tagged int8 bytes with signed symmetric int4/int6/int8 packing. Activations and output are fp32. Input-scale terms add per-input-channel multipliers for q/k/v/out.
 
 # MVN
 ```

docs/how-to-use-and-FAQ/quantized-int8-inference.md

@@ -101,6 +101,120 @@ ncnn2table can generate static weight scales without a calibration dataset for R
 ./ncnn2int8 mobilenet-opt.param mobilenet-opt.bin mobilenet-int8.param mobilenet-int8.bin mobilenet.table
 ```
 
+## Weight-only block quantized Gemm and MultiHeadAttention
+
+LLM-oriented `Gemm` and `MultiHeadAttention` weight-only block quantization is separate from the post training int8 activation/weight inference flow above. It stores weights as signed symmetric int4/int6/int8 blocks with one fp32 scale per K block and output remains fp32.
+
+The recommended workflow mirrors `ncnn2table` and `ncnn2int8`:
+
+```shell
+./ncnnllm2table in.param in.bin model.llm.table method=minmax bits=6 block=64
+./ncnnllm2int468 in.param in.bin out.param out.bin model.llm.table
+```
+
+`ncnnllm2table` options follow the same trailing key-value style as `ncnn2table`:
+
+| key | values | default | description |
+| --- | ------ | ------- | ----------- |
+| `method` | `minmax`, `mseclip`, `awq`, `gptq` | `minmax` | offline quantization method |
+| `bits` | `4`, `6`, `8` | `6` | signed weight bit width |
+| `block` | `32`, `64`, `128` | `64` | K block size |
+| `thread` | positive integer | `1` | worker threads |
+| `type` | `1` | `1` | calibration input file type, npy only |
+| `shape` | `[w,h,...]` | none | calibration input shape in ncnn order |
+| `awq_steps` | non-negative integer | `20` | AWQ input-scale search steps |
+| `awq_samples` | positive integer | `128` | max activation rows used by AWQ search |
+| `awq_max_scale` | float > 1 | `16` | AWQ input scale clamp |
+| `awq_inner` | `minmax`, `mseclip` | `minmax` | weight scale method used inside AWQ |
+| `gptq_samples` | positive integer | `128` | max activation rows used by GPTQ |
+| `gptq_damp` | non-negative float | `0.01` | GPTQ Hessian damping ratio |
+
+`method=minmax` uses per-block absmax scaling.
+
+`method=mseclip` searches clipped absmax candidates and picks the scale with the smallest block weight reconstruction error. It is calibration-free and still exports the same symmetric scale-only runtime format. It is not AWQ and does not add activation rescaling metadata.
+
+`method=awq` requires calibration data. It exports block scale rows plus `_input_scale` rows. Runtime still computes signed symmetric block dequantization; `method=awq` is only table provenance.
+
+```shell
+./ncnnllm2table in.param in.bin calib.list awq.llm.table method=awq bits=4 block=64 type=1 shape=[...]
+./ncnnllm2int468 in.param in.bin awq.param awq.bin awq.llm.table
+```
+
+`method=gptq` requires calibration data. It uses Fixed-Scale GPTQ: the runtime block scale is chosen from the original weight block, then block-local Hessian error compensation writes exact qweight sidecar files. The converter imports those qvalues directly instead of requantizing fp32 weights.
+
+```shell
+./ncnnllm2table in.param in.bin calib.list gptq.llm.table method=gptq bits=4 block=128 type=1 shape=[...]
+./ncnnllm2int468 in.param in.bin gptq.param gptq.bin gptq.llm.table
+```
+
+The calibration list format follows `ncnn2table`: one npy path per line for one input, or comma-separated list files for multiple inputs. All input lists must have the same sample count.
+
+The llm table uses one line for each quantized weight:
+
+```text
+gemm_name_param_1 bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_0  bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_1  bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_2  bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_3  bits=4 block=64 method=mseclip scale0 scale1 ...
+```
+
+For `MultiHeadAttention`, `_param_0/_param_1/_param_2/_param_3` are q/k/v/out weights. All four rows must use the same bits and block size.
+
+`bits` and `block` are row-local, so one table can mix int4/int6/int8 and block32/block64/block128 for different layers. `method` is written for readability and logging. Unknown `key=value` fields are rejected.
+
+`out.llm.table` is a text file and may be edited before conversion. A missing Gemm row skips that Gemm. For `MultiHeadAttention`, all four q/k/v/out rows must exist together; deleting all four skips the layer.
+
+The scale-only table format is:
+
+```text
+layer_param_x bits=4/6/8 block=32/64/128 method=minmax/mseclip/awq scale0 scale1 ...
+```
+
+Exact qweight import uses:
+
+```text
+gemm_name_param_1 bits=4 block=128 method=gptq qweight=gemm.qweight scale0 scale1 ...
+```
+
+The `qweight` path is relative to the table directory unless absolute. qweight data is packed in runtime layout: output rows `N`, input columns `K`, signed int4/int6/int8. The converter validates qweight byte count, tail padding bits, unsupported negative sentinel values, and scale coefficient count.
+
+The runtime quantize term is stored in param id `18`.
+
+```text
+400/401/402  int4 block=32/64/128
+600/601/602  int6 block=32/64/128
+800/801/802  int8 block=32/64/128
+```
+
+Optional per-input-channel multipliers can be stored by adding separate `_input_scale` rows. They are applied to the current input channel inside the dot product:
+
+```text
+sum += (x[k] * input_scale[k]) * (qweight[n,k] / block_scale[n,k/block])
+```
+
+```text
+gemm_name_param_1_input_scale method=awq coeff0 coeff1 ...
+mha_name_param_0_input_scale  method=awq coeff0 coeff1 ...
+mha_name_param_1_input_scale  method=awq coeff0 coeff1 ...
+mha_name_param_2_input_scale  method=awq coeff0 coeff1 ...
+mha_name_param_3_input_scale  method=awq coeff0 coeff1 ...
+```
+
+For `Gemm`, the input scale count is `constantK`. For `MultiHeadAttention`, q/k/v/out input scale counts are qdim/kdim/vdim/embed_dim, and either all four rows exist or none exist. `ncnnllm2table method=minmax/mseclip` does not write input scale rows; `method=awq` writes them.
+
+Input-scale models use the same bits/block terms with tens digit `1`: `410/411/412`, `610/611/612`, and `810/811/812`.
+
+For quick conversion without saving a table, `ncnnllm2int468` can still compute scales directly:
+
+```shell
+./ncnnllm2int468 in.param in.bin out.param out.bin method=minmax bits=6 block=64
+```
+
+AWQ and GPTQ are offline tool methods and must not become runtime `quantize_term` values. SmoothQuant is not implemented here; it needs separate offline graph rewrite or scale folding rules.
+
+This format does not use zero point or asymmetric dequantization metadata.
+
 ## use ncnn int8 inference
 
 the ncnn library would use int8 inference automatically, nothing changed in your code

src/layer/arm/gemm_arm.cpp

@@ -4584,8 +4584,13 @@ static int gemm_AT_BT_arm(const Mat& AT, const Mat& BT, const Mat& C, Mat& top_b
 
 int Gemm_arm::create_pipeline(const Option& opt)
 {
+    if (weight_block_quantize)
+    {
+        return 0;
+    }
+
 #if NCNN_INT8
-    if (int8_scale_term)
+    if (quantize_term)
     {
         return create_pipeline_int8(opt);
     }
@@ -4742,8 +4747,13 @@ int Gemm_arm::create_pipeline(const Option& opt)
 
 int Gemm_arm::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
 {
+    if (weight_block_quantize)
+    {
+        return Gemm::forward(bottom_blobs, top_blobs, opt);
+    }
+
 #if NCNN_INT8
-    if (int8_scale_term)
+    if (quantize_term)
     {
         return forward_int8(bottom_blobs, top_blobs, opt);
     }

src/layer/arm/multiheadattention_arm.cpp

@@ -4,6 +4,7 @@
 #include "multiheadattention_arm.h"
 
 #include "cpu.h"
+#include "gemm.h"
 #include "layer_type.h"
 
 namespace ncnn {
@@ -34,6 +35,9 @@ MultiHeadAttention_arm::MultiHeadAttention_arm()
 
 int MultiHeadAttention_arm::create_pipeline(const Option& _opt)
 {
+    if (weight_block_quantize)
+        return 0;
+
     Option opt = _opt;
     if (int8_scale_term)
     {
@@ -265,6 +269,9 @@ int MultiHeadAttention_arm::create_pipeline(const Option& _opt)
 
 int MultiHeadAttention_arm::destroy_pipeline(const Option& _opt)
 {
+    if (weight_block_quantize)
+        return 0;
+
     Option opt = _opt;
     if (int8_scale_term)
     {
@@ -330,6 +337,9 @@ int MultiHeadAttention_arm::destroy_pipeline(const Option& _opt)
 
 int MultiHeadAttention_arm::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& _opt) const
 {
+    if (weight_block_quantize)
+        return MultiHeadAttention::forward(bottom_blobs, top_blobs, _opt);
+
     int q_blob_i = 0;
     int k_blob_i = 0;
     int v_blob_i = 0;