Skip to content
This repository was archived by the owner on Dec 18, 2025. It is now read-only.

Commit ed78d6b

Browse files
felix-johnnyfelix-johnny
authored
CMSIS-NN: Re-deliver block process for DW conv (#1550)
A force push error in PR #1545 meant that the actual change got overwritten. This is a re-delivery of the change. DW conv for MVE is changed to process in smaller blocks. For most cases this results in lower scratch buffer requirement.
1 parent e656bce commit ed78d6b

42 files changed

Lines changed: 2686 additions & 278 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CMSIS/NN/Include/arm_nnsupportfunctions.h‎

Lines changed: 40 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@
2121
* Title: arm_nnsupportfunctions.h
2222
* Description: Public header file of support functions for CMSIS NN Library
2323
*
24-
* $Date: 4 Aug 2022
25-
* $Revision: V.9.0.1
24+
* $Date: 8 August 2022
25+
* $Revision: V.10.0.0
2626
*
2727
* Target Processor: Cortex-M CPUs
2828
* -------------------------------------------------------------------- */
@@ -50,6 +50,14 @@ extern "C" {
5050
#define CLAMP(x, h, l) MAX(MIN((x), (h)), (l))
5151
#define REDUCE_MULTIPLIER(_mult) ((_mult < 0x7FFF0000) ? ((_mult + (1 << 15)) >> 16) : 0x7FFF)
5252

53+
// Number of channels processed in a block for DW Conv(MVE)
54+
// Requirement: Greater than 0 & less than 128
55+
// This can be fine tuned to match number of input channels for best performance.
56+
// A layer with lower number of channels than CH_IN_BLOCK_MVE will result in higher
57+
// scratch buffer usage and a layer with higher number of channels than CH_IN_BLOCK_MVE
58+
// will result in lower scratch buffer usage.
59+
#define CH_IN_BLOCK_MVE (124)
60+
5361
/**
5462
* @brief definition to pack four 8 bit values.
5563
*/
@@ -495,7 +503,8 @@ arm_cmsis_nn_status arm_nn_vec_mat_mult_t_svdf_s8(const q7_t *lhs,
495503
* @param[in] lhs Input left-hand side matrix
496504
* @param[in] rhs Input right-hand side matrix (transposed)
497505
* @param[in] lhs_offset LHS matrix offset(input offset). Range: -127 to 128
498-
* @param[in] num_ch Number of channels in LHS/RHS
506+
* @param[in] active_ch Subset of total_ch processed
507+
* @param[in] total_ch Number of channels in LHS/RHS
499508
* @param[in] out_shift Per channel output shift. Length of vector is equal to number of channels
500509
* @param[in] out_mult Per channel output multiplier. Length of vector is equal to number of channels
501510
* @param[in] out_offset Offset to be added to the output values. Range: -127 to 128
@@ -516,18 +525,19 @@ arm_cmsis_nn_status arm_nn_vec_mat_mult_t_svdf_s8(const q7_t *lhs,
516525
* - Output bias
517526
* - rhs
518527
*/
519-
q7_t *arm_nn_depthwise_conv_nt_t_padded_s8(const q7_t *lhs,
520-
const q7_t *rhs,
521-
const int32_t lhs_offset,
522-
const uint16_t num_ch,
523-
const int32_t *out_shift,
524-
const int32_t *out_mult,
525-
const int32_t out_offset,
526-
const int32_t activation_min,
527-
const int32_t activation_max,
528-
const uint16_t row_x_col,
529-
const int32_t *const output_bias,
530-
q7_t *out);
528+
arm_cmsis_nn_status arm_nn_depthwise_conv_nt_t_padded_s8(const q7_t *lhs,
529+
const q7_t *rhs,
530+
const int32_t lhs_offset,
531+
const int32_t active_ch,
532+
const int32_t total_ch,
533+
const int32_t *out_shift,
534+
const int32_t *out_mult,
535+
const int32_t out_offset,
536+
const int32_t activation_min,
537+
const int32_t activation_max,
538+
const uint16_t row_x_col,
539+
const int32_t *const output_bias,
540+
q7_t *out);
531541

532542
/**
533543
* @brief Depthwise convolution of transposed rhs matrix with 4 lhs matrices. To be used in non-padded cases.
@@ -536,7 +546,8 @@ q7_t *arm_nn_depthwise_conv_nt_t_padded_s8(const q7_t *lhs,
536546
* @param[in] lhs Input left-hand side matrix
537547
* @param[in] rhs Input right-hand side matrix (transposed)
538548
* @param[in] lhs_offset LHS matrix offset(input offset). Range: -127 to 128
539-
* @param[in] num_ch Number of channels in LHS/RHS
549+
* @param[in] active_ch Subset of total_ch processed
550+
* @param[in] total_ch Number of channels in LHS/RHS
540551
* @param[in] out_shift Per channel output shift. Length of vector is equal to number of channels.
541552
* @param[in] out_mult Per channel output multiplier. Length of vector is equal to number of channels.
542553
* @param[in] out_offset Offset to be added to the output values. Range: -127 to 128
@@ -557,18 +568,19 @@ q7_t *arm_nn_depthwise_conv_nt_t_padded_s8(const q7_t *lhs,
557568
* - Output bias
558569
* - rhs
559570
*/
560-
q7_t *arm_nn_depthwise_conv_nt_t_s8(const q7_t *lhs,
561-
const q7_t *rhs,
562-
const int32_t lhs_offset,
563-
const uint16_t num_ch,
564-
const int32_t *out_shift,
565-
const int32_t *out_mult,
566-
const int32_t out_offset,
567-
const int32_t activation_min,
568-
const int32_t activation_max,
569-
const uint16_t row_x_col,
570-
const int32_t *const output_bias,
571-
q7_t *out);
571+
arm_cmsis_nn_status arm_nn_depthwise_conv_nt_t_s8(const q7_t *lhs,
572+
const q7_t *rhs,
573+
const int32_t lhs_offset,
574+
const int32_t active_ch,
575+
const int32_t total_ch,
576+
const int32_t *out_shift,
577+
const int32_t *out_mult,
578+
const int32_t out_offset,
579+
const int32_t activation_min,
580+
const int32_t activation_max,
581+
const uint16_t row_x_col,
582+
const int32_t *const output_bias,
583+
q7_t *out);
572584

573585
/**
574586
* @brief Depthwise convolution of transposed rhs matrix with 4 lhs matrices. To be used in non-padded cases.

‎CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_s8_opt.c‎

Lines changed: 106 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@
2222
* Description: Optimized s8 depthwise separable convolution function for
2323
* channel multiplier of 1.
2424
*
25-
* $Date: 6 July 2022
26-
* $Revision: V.3.0.1
25+
* $Date: 27 July 2022
26+
* $Revision: V.3.1.0
2727
*
2828
* Target Processor: Cortex-M CPUs
2929
*
@@ -64,7 +64,7 @@ arm_cmsis_nn_status arm_depthwise_conv_s8_opt(const cmsis_nn_context *ctx,
6464
const int32_t input_ch = input_dims->c;
6565
const int32_t output_ch = output_dims->c;
6666

67-
/* Check input constraints input_ch == output_ch */
67+
/* Check depth multiplier is 1 */
6868
if (input_ch != output_ch)
6969
{
7070
return ARM_CMSIS_NN_ARG_ERROR;
@@ -102,116 +102,130 @@ arm_cmsis_nn_status arm_depthwise_conv_s8_opt(const cmsis_nn_context *ctx,
102102
int buffer_count = 0;
103103
const int32_t kernel_size = kernel_x * kernel_y;
104104

105-
/* This part implements the im2col function */
106-
for (int i_out_y = 0, base_idx_y = -pad_y; i_out_y < output_y; base_idx_y += stride_y, i_out_y++)
105+
const int32_t ch_loop = (input_ch + (CH_IN_BLOCK_MVE - 1)) / CH_IN_BLOCK_MVE;
106+
int32_t remaining_ch = output_ch;
107+
int32_t active_ch = MIN(CH_IN_BLOCK_MVE, remaining_ch);
108+
remaining_ch -= CH_IN_BLOCK_MVE;
109+
110+
for (int i_ch = 0; i_ch < ch_loop; i_ch++)
107111
{
108-
for (int i_out_x = 0, base_idx_x = -pad_x; i_out_x < output_x; base_idx_x += stride_x, i_out_x++)
112+
out = output + i_ch * CH_IN_BLOCK_MVE;
113+
const int8_t *input_slice = input + (i_ch * CH_IN_BLOCK_MVE);
114+
115+
for (int i_out_y = 0, base_idx_y = -pad_y; i_out_y < output_y; base_idx_y += stride_y, i_out_y++)
109116
{
110-
for (int i_ker_y = base_idx_y; i_ker_y < base_idx_y + kernel_y; i_ker_y++)
117+
for (int i_out_x = 0, base_idx_x = -pad_x; i_out_x < output_x; base_idx_x += stride_x, i_out_x++)
111118
{
112-
for (int i_ker_x = base_idx_x; i_ker_x < base_idx_x + kernel_x; i_ker_x++)
119+
for (int i_ker_y = base_idx_y; i_ker_y < base_idx_y + kernel_y; i_ker_y++)
120+
{
121+
for (int i_ker_x = base_idx_x; i_ker_x < base_idx_x + kernel_x; i_ker_x++)
122+
{
123+
if (i_ker_y < 0 || i_ker_y >= input_y || i_ker_x < 0 || i_ker_x >= input_x)
124+
{
125+
arm_memset_q7(lhs_buffer, (int8_t)-input_offset, (uint32_t)active_ch);
126+
padded = 1;
127+
}
128+
else
129+
{
130+
arm_memcpy_q7(lhs_buffer,
131+
input_slice + (i_ker_y * input_x + i_ker_x) * input_ch,
132+
(uint32_t)active_ch);
133+
}
134+
lhs_buffer += CH_IN_BLOCK_MVE;
135+
}
136+
}
137+
buffer_count++;
138+
139+
if (buffer_count == 4)
113140
{
114-
if (i_ker_y < 0 || i_ker_y >= input_y || i_ker_x < 0 || i_ker_x >= input_x)
141+
const int32_t block_offset = i_ch * CH_IN_BLOCK_MVE;
142+
lhs_buffer = (q7_t *)buffer_a;
143+
if (padded == 0)
115144
{
116-
arm_memset_q7(lhs_buffer, (int8_t)-input_offset, (uint32_t)input_ch);
117-
padded = 1;
145+
arm_nn_depthwise_conv_nt_t_s8(lhs_buffer,
146+
kernel + block_offset,
147+
input_offset,
148+
active_ch,
149+
input_ch,
150+
output_shift + block_offset,
151+
output_mult + block_offset,
152+
output_offset,
153+
output_activation_min,
154+
output_activation_max,
155+
kernel_size,
156+
bias + block_offset,
157+
out);
118158
}
119159
else
120160
{
121-
arm_memcpy_q7(lhs_buffer, input + (i_ker_y * input_x + i_ker_x) * input_ch, (uint32_t)input_ch);
161+
arm_nn_depthwise_conv_nt_t_padded_s8(lhs_buffer,
162+
kernel + block_offset,
163+
input_offset,
164+
active_ch,
165+
input_ch,
166+
output_shift + block_offset,
167+
output_mult + block_offset,
168+
output_offset,
169+
output_activation_min,
170+
output_activation_max,
171+
kernel_size,
172+
bias + block_offset,
173+
out);
174+
padded = 0;
122175
}
123-
lhs_buffer += input_ch;
176+
out += (4 * input_ch);
177+
buffer_count = 0;
124178
}
125179
}
126-
buffer_count++;
180+
}
181+
/* Handle left over buffers */
182+
lhs_buffer = (q7_t *)buffer_a;
127183

128-
if (buffer_count == 4)
184+
int8_t *out_base = out;
185+
for (int i_buf = 0; i_buf < buffer_count; i_buf++)
186+
{
187+
int32_t loop_count = (active_ch + 3) / 4;
188+
int32_t num_ch_to_process = active_ch;
189+
out = out_base + (i_buf * input_ch);
190+
for (int i_loop_cnt = 0, offset = i_ch * CH_IN_BLOCK_MVE; i_loop_cnt < loop_count;
191+
num_ch_to_process -= 4, offset += 4, i_loop_cnt++)
129192
{
130-
lhs_buffer = (q7_t *)buffer_a;
131-
if (padded == 0)
132-
{
133-
out = arm_nn_depthwise_conv_nt_t_s8(lhs_buffer,
134-
kernel,
135-
input_offset,
136-
input_ch,
137-
output_shift,
138-
output_mult,
139-
output_offset,
140-
output_activation_min,
141-
output_activation_max,
142-
kernel_size,
143-
bias,
144-
out);
145-
}
146-
else
193+
const int8_t *col_0 = lhs_buffer + (kernel_size * CH_IN_BLOCK_MVE * i_buf) + (i_loop_cnt * 4);
194+
const int8_t *row_0 = kernel + offset;
195+
int32x4_t out_0 = vdupq_n_s32(0);
196+
if (bias)
147197
{
148-
out = arm_nn_depthwise_conv_nt_t_padded_s8(lhs_buffer,
149-
kernel,
150-
input_offset,
151-
input_ch,
152-
output_shift,
153-
output_mult,
154-
output_offset,
155-
output_activation_min,
156-
output_activation_max,
157-
kernel_size,
158-
bias,
159-
out);
160-
padded = 0;
198+
out_0 = vldrwq_s32(&bias[offset]);
161199
}
162-
buffer_count = 0;
163-
}
164-
}
165-
}
166200

167-
/* Handle left over buffers */
168-
lhs_buffer = (q7_t *)buffer_a;
169-
170-
for (int i_buf = 0; i_buf < buffer_count; i_buf++)
171-
{
172-
int32_t loop_count = (input_ch + 3) / 4;
173-
int32_t num_ch_to_process = input_ch;
201+
for (int i_ker = 0; i_ker < kernel_size; i_ker++)
202+
{
203+
const int32x4_t ker_0 = vldrbq_s32(row_0);
204+
int32x4_t ip_0 = vldrbq_s32(col_0);
205+
ip_0 = vaddq_n_s32(ip_0, input_offset);
206+
out_0 += vmulq_s32(ip_0, ker_0);
174207

175-
for (int i_loop_cnt = 0, offset = 0; i_loop_cnt < loop_count; num_ch_to_process -= 4, offset += 4, i_loop_cnt++)
176-
{
177-
const int8_t *col_0 = lhs_buffer + (kernel_size * input_ch * i_buf) + offset;
178-
const int8_t *row_0 = kernel + offset;
179-
int32x4_t out_0 = vdupq_n_s32(0);
180-
if (bias)
181-
{
182-
out_0 = vldrwq_s32(&bias[offset]);
183-
}
208+
col_0 += CH_IN_BLOCK_MVE;
209+
row_0 += input_ch;
210+
}
184211

185-
for (int i_ker = 0; i_ker < kernel_size; i_ker++)
186-
{
187-
const int32x4_t ker_0 = vldrbq_s32(row_0);
212+
const int32x4_t mult = vldrwq_s32(&output_mult[offset]);
213+
const int32x4_t shift = vldrwq_s32(&output_shift[offset]);
188214

189-
int32x4_t ip_0 = vldrbq_s32(col_0);
190-
ip_0 = vaddq_n_s32(ip_0, input_offset);
191-
out_0 += vmulq_s32(ip_0, ker_0);
215+
out_0 = arm_requantize_mve_32x4(out_0, mult, shift);
216+
out_0 = vaddq_n_s32(out_0, output_offset);
217+
out_0 = vmaxq_s32(out_0, vdupq_n_s32(output_activation_min));
218+
out_0 = vminq_s32(out_0, vdupq_n_s32(output_activation_max));
219+
mve_pred16_t p = vctp32q((uint32_t)num_ch_to_process);
220+
vstrbq_p_s32(out, out_0, p);
192221

193-
col_0 += input_ch;
194-
row_0 += input_ch;
222+
out += 4;
195223
}
196-
197-
const int32x4_t mult = vldrwq_s32(&output_mult[offset]);
198-
const int32x4_t shift = vldrwq_s32(&output_shift[offset]);
199-
200-
out_0 = arm_requantize_mve_32x4(out_0, mult, shift);
201-
out_0 = vaddq_n_s32(out_0, output_offset);
202-
out_0 = vmaxq_s32(out_0, vdupq_n_s32(output_activation_min));
203-
out_0 = vminq_s32(out_0, vdupq_n_s32(output_activation_max));
204-
mve_pred16_t p = vctp32q((uint32_t)num_ch_to_process);
205-
vstrbq_p_s32(out, out_0, p);
206-
207-
out += 4;
208224
}
225+
buffer_count = 0;
209226

210-
const int tail_ch = input_ch & 0x3;
211-
if (tail_ch != 0)
212-
{
213-
out -= (4 - tail_ch);
214-
}
227+
active_ch = MIN(CH_IN_BLOCK_MVE, remaining_ch);
228+
remaining_ch -= CH_IN_BLOCK_MVE;
215229
}
216230

217231
#else // ARM_MATH_DSP
@@ -431,8 +445,8 @@ arm_cmsis_nn_status arm_depthwise_conv_s8_opt(const cmsis_nn_context *ctx,
431445
int32_t arm_depthwise_conv_s8_opt_get_buffer_size(const cmsis_nn_dims *input_dims, const cmsis_nn_dims *filter_dims)
432446
{
433447
#if defined(ARM_MATH_MVEI)
434-
/* The + 4 accounts for out of bounds read of the lhs buffers in the *_nt_t_* functions. */
435-
return (4 * input_dims->c * filter_dims->w * filter_dims->h) * (int32_t)sizeof(int8_t) + 4;
448+
(void)input_dims;
449+
return (4 * CH_IN_BLOCK_MVE * filter_dims->w * filter_dims->h) * (int32_t)sizeof(int8_t);
436450
#elif defined(ARM_MATH_DSP)
437451
return (input_dims->c * filter_dims->w * filter_dims->h) * sizeof(int16_t);
438452
#else

0 commit comments

Comments
 (0)