Skip to content

Commit c0e9cfa

Browse files
bashbaugbashbaug
authored
matrix experiments (#167)
1 parent 309d6c2 commit c0e9cfa

22 files changed

Lines changed: 6583 additions & 42 deletions

‎include/bfloat16.hpp‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
// Copyright (c) 2024-2026 Ben Ashbaugh
3+
//
4+
// SPDX-License-Identifier: MIT
5+
*/
6+
#pragma once
7+
8+
#include <cmath>
9+
#include <cstdint>
10+
11+
class bfloat16;
12+
13+
class bfloat16 {
14+
using StorageType = uint16_t;
15+
StorageType value;
16+
17+
static StorageType from_float(const float &a) {
18+
if (std::isnan(a))
19+
return 0xffc1;
20+
union {
21+
uint32_t intStorage;
22+
float floatValue;
23+
};
24+
floatValue = a;
25+
// Do RNE and truncate
26+
uint32_t roundingBias = ((intStorage >> 16) & 0x1) + 0x00007FFF;
27+
return static_cast<StorageType>((intStorage + roundingBias) >> 16);
28+
}
29+
30+
static float to_float(const StorageType &a) {
31+
union {
32+
uint32_t intStorage;
33+
float floatValue;
34+
};
35+
intStorage = a << 16;
36+
return floatValue;
37+
}
38+
39+
public:
40+
bfloat16() = default;
41+
bfloat16(const bfloat16 &) = default;
42+
~bfloat16() = default;
43+
44+
// Implicit conversion from float to bfloat16
45+
bfloat16(const float &a) { value = from_float(a); }
46+
47+
bfloat16 &operator=(const float &rhs) {
48+
value = from_float(rhs);
49+
return *this;
50+
}
51+
52+
// Implicit conversion from bfloat16 to float
53+
operator float() const { return to_float(value); }
54+
55+
// Logical operators (!,||,&&) are covered if we can cast to bool
56+
explicit operator bool() const { return to_float(value) != 0.0f; }
57+
58+
// Unary minus operator overloading
59+
friend bfloat16 operator-(const bfloat16 &lhs) {
60+
return -to_float(lhs.value);
61+
}
62+
63+
// Increment and decrement operators overloading
64+
#define OP(op) \
65+
friend bfloat16 &operator op(bfloat16 &lhs) { \
66+
float f = to_float(lhs.value); \
67+
lhs.value = from_float(op f); \
68+
return lhs; \
69+
} \
70+
friend bfloat16 operator op(bfloat16 &lhs, int) { \
71+
bfloat16 old = lhs; \
72+
operator op(lhs); \
73+
return old; \
74+
}
75+
OP(++)
76+
OP(--)
77+
#undef OP
78+
79+
// Assignment operators overloading
80+
#define OP(op) \
81+
friend bfloat16 &operator op(bfloat16 &lhs, const bfloat16 &rhs) { \
82+
float f = static_cast<float>(lhs); \
83+
f op static_cast<float>(rhs); \
84+
return lhs = f; \
85+
} \
86+
template <typename T> \
87+
friend bfloat16 &operator op(bfloat16 &lhs, const T &rhs) { \
88+
float f = static_cast<float>(lhs); \
89+
f op static_cast<float>(rhs); \
90+
return lhs = f; \
91+
} \
92+
template <typename T> friend T &operator op(T &lhs, const bfloat16 &rhs) { \
93+
float f = static_cast<float>(lhs); \
94+
f op static_cast<float>(rhs); \
95+
return lhs = f; \
96+
}
97+
OP(+=)
98+
OP(-=)
99+
OP(*=)
100+
OP(/=)
101+
#undef OP
102+
103+
// Binary operators overloading
104+
#define OP(type, op) \
105+
friend type operator op(const bfloat16 &lhs, const bfloat16 &rhs) { \
106+
return type{static_cast<float>(lhs) op static_cast<float>(rhs)}; \
107+
} \
108+
template <typename T> \
109+
friend type operator op(const bfloat16 &lhs, const T &rhs) { \
110+
return type{static_cast<float>(lhs) op static_cast<float>(rhs)}; \
111+
} \
112+
template <typename T> \
113+
friend type operator op(const T &lhs, const bfloat16 &rhs) { \
114+
return type{static_cast<float>(lhs) op static_cast<float>(rhs)}; \
115+
}
116+
OP(bfloat16, +)
117+
OP(bfloat16, -)
118+
OP(bfloat16, *)
119+
OP(bfloat16, /)
120+
OP(bool, ==)
121+
OP(bool, !=)
122+
OP(bool, <)
123+
OP(bool, >)
124+
OP(bool, <=)
125+
OP(bool, >=)
126+
#undef OP
127+
128+
// Bitwise(|,&,~,^), modulo(%) and shift(<<,>>) operations are not supported
129+
// for floating-point types.
130+
};

‎include/util.hpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@
66
#pragma once
77

88
#include <CL/opencl.hpp>
9+
10+
#include <cctype>
11+
#include <cstdio>
12+
#include <cstring>
13+
#include <fstream>
14+
#include <iterator>
915
#include <string>
1016

1117
static cl_version getDeviceOpenCLVersion(
@@ -68,6 +74,22 @@ static bool checkDeviceForExtension(
6874
return supported;
6975
}
7076

77+
static std::string readStringFromFile(
78+
const std::string& filename )
79+
{
80+
std::ifstream is(filename, std::ios::binary);
81+
if (!is.good()) {
82+
printf("Couldn't open file '%s'!\n", filename.c_str());
83+
return "";
84+
}
85+
86+
std::string source{
87+
std::istreambuf_iterator<char>(is),
88+
std::istreambuf_iterator<char>() };
89+
90+
return source;
91+
}
92+
7193
static bool checkPlatformIndex(
7294
const std::vector<cl::Platform>& platforms,
7395
int platformIndex)

‎samples/05_kernelfromfile/main.cpp‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,6 @@
1313

1414
#include "util.hpp"
1515

16-
static std::string readStringFromFile(
17-
const std::string& filename )
18-
{
19-
std::ifstream is(filename, std::ios::binary);
20-
if (!is.good()) {
21-
printf("Couldn't open file '%s'!\n", filename.c_str());
22-
return "";
23-
}
24-
25-
size_t filesize = 0;
26-
is.seekg(0, std::ios::end);
27-
filesize = (size_t)is.tellg();
28-
is.seekg(0, std::ios::beg);
29-
30-
std::string source{
31-
std::istreambuf_iterator<char>(is),
32-
std::istreambuf_iterator<char>() };
33-
34-
return source;
35-
}
36-
3716
int main(
3817
int argc,
3918
char** argv )

‎samples/06_ndrangekernelfromfile/main.cpp‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,6 @@
1313

1414
#include "util.hpp"
1515

16-
static std::string readStringFromFile(
17-
const std::string& filename )
18-
{
19-
std::ifstream is(filename, std::ios::binary);
20-
if (!is.good()) {
21-
printf("Couldn't open file '%s'!\n", filename.c_str());
22-
return "";
23-
}
24-
25-
size_t filesize = 0;
26-
is.seekg(0, std::ios::end);
27-
filesize = (size_t)is.tellg();
28-
is.seekg(0, std::ios::beg);
29-
30-
std::string source{
31-
std::istreambuf_iterator<char>(is),
32-
std::istreambuf_iterator<char>() };
33-
34-
return source;
35-
}
36-
3716
int main(
3817
int argc,
3918
char** argv )

‎samples/20_matrixexperiments-bf16/CMakeLists.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Copyright (c) 2024-2026 Ben Ashbaugh
2+
#
3+
# SPDX-License-Identifier: MIT
4+
5+
add_opencl_sample(
6+
TEST
7+
NUMBER 20
8+
TARGET matrixexperiments-bf16
9+
VERSION 200 # for clSetKernelExecInfo
10+
SOURCES main.cpp
11+
KERNELS matrix_helpers_bf16.cl matrix_kernels_bf16.cl matrix_kernel_tiled_bf16.cl)

‎samples/20_matrixexperiments-bf16/README.md‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# matrixexperiments-bf16
2+
3+
## Sample Purpose
4+
5+
This sample demonstrates various techniques to perform a large matrix multiplication where the matrix elements contain 16-bit `bfloat16` data.
6+
The sample includes many different implementations:
7+
8+
1. The "naive" implementation is a very simple implementation.
9+
It is not very fast, but it is easy to understand, and it has no extension dependencies so it will run on many devices.
10+
2. The "dpas" kernels use sub-group extensions to improve performance.
11+
On some devices, they will also use specialized matrix multiplication extensions to further improve performance.
12+
Because these kernels require certain extensions or a specific sub-group size, they may not run on all devices.
13+
3. The "dpas blockread" kernels use additional sub-group extensions to further improve performance.
14+
15+
Most of the optimized kernels operate on fixed size tiles of matrix data.
16+
For some of these kernels, parameters such as the number of matrix tiles per-sub-group or the number of sub-groups per work-group may be modified via program build options.
17+
Experiment with different options to see what performs the best!
18+
19+
A good place to start for some devices is:
20+
21+
```sh
22+
./matrixexperiments-bf16 -m4096 --options="-DSGS_PER_WG_X=4 -DSGS_PER_WG_Y=8 -DKK=2 -cl-intel-256-GRF-per-thread" --zero
23+
```
24+
25+
## Key APIs and Concepts
26+
27+
This sample will optionally use the following OpenCL extensions:
28+
29+
* cl_intel_bfloat16_conversions
30+
* cl_intel_required_subgroup_size
31+
* cl_intel_split_work_group_barrier
32+
* cl_intel_subgroup_2d_block_io
33+
* cl_intel_subgroup_matrix_multiply_accumulate
34+
* cl_intel_subgroups
35+
* cl_intel_subgroups_short
36+
37+
## Command Line Options
38+
39+
| Option | Default Value | Description |
40+
|:--|:-:|:--|
41+
| `-p <index>` | 0 | Specify the index of the OpenCL platform to execute the sample on.
42+
| `-d <index>` | 0 | Specify the index of the OpenCL device in the platform to execute on the sample on.
43+
| `--file <string>` | `matrix_kernels_bf16.cl` | Specify the name of the file with the OpenCL kernel source.
44+
| `--options <string>` | None | Specify optional program build options.
45+
| `--matrixsize <int>` | 512 | Specify the dimensions of the matrix.
46+
| `--iterations <int>` | 16 | Specify the number of iterations for performance testing.
47+
| `--validate` | n/a | Validate results for correctness.
48+
| `--zero` | n/a | Initialize all matrices to zero.
49+
| `--identity` | n/a | Initialize all matrices to one.
50+
| `--fixed` | n/a | Initialize all matrices to values computed from the matrix row and column.
51+
| `--emulate` | n/a | Do not use specialized matrix multiplication extensions.
52+
| `--wallclock` | n/a | Measure performance using wallclock time instead of event profiling.
53+
| `--skipinit` | n/a | Skip initialization of source matrices.
54+
| `--roundrobin` | n/a | Use round robin thread scheduling.
55+
| `--threshold <float>` | 0.01 | Set the threshold used when validating results.
56+
| `--mask <int>` | ~0 | Set a mask to only run a subset of tests.
57+
58+
By default, the source matrices are populated with random data.
59+
When validating results, it is recommended to use either "fixed" or "identity" data.
60+
For best performance, use "zero" data.

0 commit comments

Comments
 (0)