Skip to content

Commit d6588da

Browse files
lhezlhez
authored
opencl: support bf16 by converting to f16 (ggml-org#23839)
1 parent d38d50e commit d6588da

2 files changed

Lines changed: 121 additions & 2 deletions

File tree

‎ggml/src/ggml-opencl/ggml-opencl.cpp‎

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,7 @@ struct ggml_backend_opencl_context {
585585
cl_kernel kernel_convert_block_mxfp4_trans4_ns, kernel_restore_block_mxfp4_trans4_ns;
586586
cl_kernel kernel_convert_block_q8_0, kernel_restore_block_q8_0, kernel_restore_block_q8_0_trans;
587587
cl_kernel kernel_convert_block_q6_K_noshuffle, kernel_restore_block_q6_K_noshuffle;
588+
cl_kernel kernel_convert_bf16_to_f16, kernel_convert_f16_to_bf16;
588589
cl_kernel kernel_mul_mat_q4_0_f32_8x_flat;
589590
cl_kernel kernel_convert_block_q4_0_noshuffle;
590591
cl_kernel kernel_restore_block_q4_0_noshuffle;
@@ -1175,6 +1176,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
11751176
CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl", &err), err));
11761177
CL_CHECK((backend_ctx->kernel_convert_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_nl_noshuffle", &err), err));
11771178
CL_CHECK((backend_ctx->kernel_restore_block_iq4_nl_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_nl_noshuffle", &err), err));
1179+
CL_CHECK((backend_ctx->kernel_convert_bf16_to_f16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_bf16_to_f16", &err), err));
1180+
CL_CHECK((backend_ctx->kernel_convert_f16_to_bf16 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_f16_to_bf16", &err), err));
11781181
GGML_LOG_CONT(".");
11791182
}
11801183

@@ -5019,6 +5022,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
50195022
case GGML_OP_MUL_MAT:
50205023
if (op->src[0]->type == GGML_TYPE_F16) {
50215024
return true;
5025+
} else if (op->src[0]->type == GGML_TYPE_BF16) {
5026+
return true;
50225027
} else if (op->src[0]->type == GGML_TYPE_F32) {
50235028
return op->src[1]->type == GGML_TYPE_F32;
50245029
} else if (op->src[0]->type == GGML_TYPE_Q4_0 || op->src[0]->type == GGML_TYPE_Q4_1 ||
@@ -6828,6 +6833,40 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
68286833
}
68296834
#endif // GGML_OPENCL_SOA_Q
68306835

6836+
// convert bf16 to f16 and store as f16 in device buffer
6837+
if (tensor->type == GGML_TYPE_BF16) {
6838+
GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0
6839+
&& "Offset and size must be multiples of 2 for bf16 tensors");
6840+
6841+
ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
6842+
GGML_ASSERT(extra);
6843+
6844+
cl_ulong n_elements = size / sizeof(ggml_fp16_t);
6845+
cl_ulong off_dst = (extra->offset + offset) / sizeof(ggml_fp16_t);
6846+
6847+
cl_int err;
6848+
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR,
6849+
size, (void *) data, &err);
6850+
CL_CHECK(err);
6851+
6852+
cl_kernel kernel = backend_ctx->kernel_convert_bf16_to_f16;
6853+
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
6854+
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->data_device));
6855+
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_ulong), &off_dst));
6856+
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements));
6857+
6858+
size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 };
6859+
size_t local_work_size[] = { 64, 1, 1 };
6860+
6861+
cl_event evt;
6862+
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
6863+
CL_CHECK(clWaitForEvents(1, &evt));
6864+
CL_CHECK(clReleaseMemObject(data_device));
6865+
CL_CHECK(clReleaseEvent(evt));
6866+
6867+
return;
6868+
}
6869+
68316870
ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
68326871
GGML_ASSERT(extra);
68336872

@@ -7676,6 +7715,41 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
76767715
}
76777716
#endif // GGML_OPENCL_SOA_Q
76787717

7718+
if (tensor->type == GGML_TYPE_BF16) {
7719+
GGML_ASSERT(offset % sizeof(ggml_fp16_t) == 0 && size % sizeof(ggml_fp16_t) == 0
7720+
&& "Offset and size must be multiples of 2 for bf16 tensors");
7721+
7722+
ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
7723+
GGML_ASSERT(extra);
7724+
7725+
cl_ulong n_elements = size / sizeof(ggml_fp16_t);
7726+
cl_ulong off_src = (extra->offset + tensor->view_offs + offset) / sizeof(ggml_fp16_t);
7727+
7728+
cl_int err;
7729+
cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, size, NULL, &err);
7730+
CL_CHECK(err);
7731+
7732+
cl_kernel kernel = backend_ctx->kernel_convert_f16_to_bf16;
7733+
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->data_device));
7734+
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &off_src));
7735+
CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));
7736+
CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &n_elements));
7737+
7738+
size_t global_work_size[] = { (size_t)CEIL_DIV(n_elements, 64)*64, 1, 1 };
7739+
size_t local_work_size[] = { 64, 1, 1 };
7740+
7741+
cl_event evt;
7742+
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
7743+
CL_CHECK(clWaitForEvents(1, &evt));
7744+
CL_CHECK(clReleaseEvent(evt));
7745+
7746+
CL_CHECK(clEnqueueReadBuffer(
7747+
queue, data_device, CL_TRUE, 0, size, data, 0, NULL, NULL));
7748+
CL_CHECK(clReleaseMemObject(data_device));
7749+
7750+
return;
7751+
}
7752+
76797753
ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
76807754

76817755
CL_CHECK(clEnqueueReadBuffer(
@@ -8165,6 +8239,7 @@ static void ggml_cl_copy_to_contiguous(ggml_backend_t backend, const ggml_tensor
81658239
kernel = backend_ctx->kernel_cpy_f32_f32;
81668240
break;
81678241
case GGML_TYPE_F16:
8242+
case GGML_TYPE_BF16: // stored as f16 on device
81688243
kernel = backend_ctx->kernel_cpy_f16_f16;
81698244
break;
81708245
default:
@@ -11125,7 +11200,8 @@ static bool ggml_cl_can_use_adreno_xmem_gemm_f16_f32(
1112511200
if (backend_ctx->gpu_family != GPU_FAMILY::ADRENO) {
1112611201
return false;
1112711202
}
11128-
if (src0->type != GGML_TYPE_F16 || src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
11203+
if ((src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_BF16) ||
11204+
src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
1112911205
return false;
1113011206
}
1113111207
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
@@ -12843,7 +12919,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
1284312919
GGML_ASSERT(dst);
1284412920
GGML_ASSERT(dst->extra);
1284512921

12846-
const enum ggml_type src0t = src0->type;
12922+
// bf16 is stored as f16 on device
12923+
const enum ggml_type src0t = (src0->type == GGML_TYPE_BF16) ? GGML_TYPE_F16 : src0->type;
1284712924
const enum ggml_type src1t = src1->type;
1284812925

1284912926
ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;

‎ggml/src/ggml-opencl/kernels/cvt.cl‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,48 @@ struct block_iq4_nl
117117
uint8_t qs[QK4_NL / 2];
118118
};
119119

120+
//------------------------------------------------------------------------------
121+
// bf16 to f16
122+
//------------------------------------------------------------------------------
123+
kernel void kernel_convert_bf16_to_f16(
124+
global const ushort * src,
125+
global half * dst,
126+
ulong off_dst,
127+
ulong n
128+
) {
129+
uint i = get_global_id(0);
130+
if (i >= n) {
131+
return;
132+
}
133+
134+
dst[i + off_dst] = (half) as_float((uint) src[i] << 16);
135+
}
136+
137+
//------------------------------------------------------------------------------
138+
// f16 to bf16
139+
//------------------------------------------------------------------------------
140+
kernel void kernel_convert_f16_to_bf16(
141+
global const half * src,
142+
ulong off_src,
143+
global ushort * dst,
144+
ulong n
145+
) {
146+
uint i = get_global_id(0);
147+
if (i >= n) {
148+
return;
149+
}
150+
151+
float f = (float) src[i + off_src];
152+
uint bits = as_uint(f);
153+
if ((bits & 0x7fffffffu) > 0x7f800000u) {
154+
// nan to quiet nan
155+
dst[i] = (ushort)((bits >> 16) | 0x40u);
156+
} else {
157+
uint rounded = bits + 0x7fffu + ((bits >> 16) & 1u);
158+
dst[i] = (ushort)(rounded >> 16);
159+
}
160+
}
161+
120162
//------------------------------------------------------------------------------
121163
// kernel_convert_block_q4_0
122164
// Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA).

0 commit comments

Comments
 (0)