Skip to content

Add HiFi int8/int32 optimized paths for LeakyRelu, Pad, and SVDF on Xtensa #3644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
unmeshna017 wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Add HiFi int8/int32 optimized paths for LeakyRelu, Pad, and SVDF on Xtensa #3644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dbc8f1b
Add HiFi int8 optimized LeakyRelu for Xtensa
unmeshna017 Jul 31, 2026
a3edca6
Add HiFi int8/int32 optimized Pad for Xtensa
unmeshna017 Jul 31, 2026
aa4e828
Enable Xtensa SVDF optimized paths for HIFI_IQ target
unmeshna017 Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions tensorflow/lite/micro/kernels/xtensa/leaky_relu.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,35 @@ TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) {
tflite::micro::GetTensorData<float>(output));
return kTfLiteOk;
} break;
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
case kTfLiteInt8: {
int err;
const signed char *input_data_ptr;
signed char *output_data_ptr;
const int flat_size = MatchingFlatSize(tflite::micro::GetTensorShape(input), tflite::micro::GetTensorShape(output));
input_data_ptr = tflite::micro::GetTensorData<int8_t>(input);
output_data_ptr = tflite::micro::GetTensorData<int8_t>(output);

err = xa_nn_vec_leaky_relu_asym8s_asym8s(output_data_ptr,
input_data_ptr,
data.input_zero_point,
data.output_multiplier_alpha,
data.output_shift_alpha,
data.output_multiplier_identity,
data.output_shift_identity,
data.output_zero_point,
flat_size);
TF_LITE_ENSURE(context, err == 0);
return kTfLiteOk;
} break;
#else
case kTfLiteInt8: {
QuantizeLeakyRelu<int8_t>(data, input, output);
return kTfLiteOk;
} break;
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
case kTfLiteInt16: {
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
const RuntimeShape& input_shape = tflite::micro::GetTensorShape(input);
const RuntimeShape& output_shape = tflite::micro::GetTensorShape(output);
const int flat_size = MatchingFlatSize(input_shape, output_shape);
Expand All @@ -89,7 +112,7 @@ TfLiteStatus LeakyReluEval(TfLiteContext* context, TfLiteNode* node) {
if (err != 0) return kTfLiteError;
#else
QuantizeLeakyRelu<int16_t>(data, input, output);
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
return kTfLiteOk;
} break;
default:
Expand Down
83 changes: 76 additions & 7 deletions tensorflow/lite/micro/kernels/xtensa/pad.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,13 +190,54 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
case kTfLiteInt8: {
#if defined(VISION_P6)
PadEvalVision(*op_data_xtensa, input, output);
#else
#else // defined(VISION_P6)
int8_t pad_value;
if (constant_values == nullptr) {
pad_value = static_cast<uint8_t>(data->output_zero_point);
} else {
pad_value = *tflite::micro::GetTensorData<int8_t>(constant_values);
}
#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ))
if((input->dims->size <= 4))
{
int err;
const int8_t *input_data_ptr;
int8_t *output_data_ptr;

input_data_ptr = tflite::micro::GetTensorData<int8_t>(input);
output_data_ptr = tflite::micro::GetTensorData<int8_t>(output);

const TfLiteEvalTensor* paddings = tflite::micro::GetEvalInput(context, node, /*index=*/1);
const int32_t* paddings_data_ptr = tflite::micro::GetTensorData<int32_t>(paddings);

err = xa_nn_pad_8_8(output_data_ptr
,output->dims->data
,input_data_ptr
,input->dims->data
,paddings_data_ptr
,paddings->dims->data
,output->dims->size
,input->dims->size
,paddings->dims->size
,pad_value);
TF_LITE_ENSURE(context, (err==0) );
}
else
{
if (data->params.resizing_category == ResizingCategory::kImageStyle) {
reference_ops::PadImageStyle(
data->params, tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<int8_t>(input), &pad_value,
tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<int8_t>(output));
} else {
reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<int8_t>(input),
&pad_value, tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<int8_t>(output));
}
}
#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
if (data->params.resizing_category == ResizingCategory::kImageStyle) {
reference_ops::PadImageStyle(
data->params, tflite::micro::GetTensorShape(input),
Expand All @@ -209,16 +250,17 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
&pad_value, tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<int8_t>(output));
}
#endif
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
#endif // defined(VISION_P6)
} break;
case kTfLiteInt16: {
int16_t pad_value =
constant_values == nullptr
? 0
: *tflite::micro::GetTensorData<int16_t>(constant_values);
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ))
/* NNLib currently only supports up to 4D input tensors */
if (tflite::micro::GetTensorShape(input).DimensionsCount() == 4) {
if (tflite::micro::GetTensorShape(input).DimensionsCount() <= 4) {
const TfLiteEvalTensor* paddings =
tflite::micro::GetEvalInput(context, node, /*index=*/1);
int32_t err = xa_nn_pad_16_16(
Expand All @@ -234,24 +276,51 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
pad_value);
if (err != 0) return kTfLiteError;
} else {
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<int16_t>(input),
&pad_value, tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<int16_t>(output));
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ))
}
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
} break;
case kTfLiteInt32: {
int32_t pad_value =
constant_values == nullptr
? 0
: *tflite::micro::GetTensorData<int32_t>(constant_values);
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
if(tflite::micro::GetTensorShape(input).DimensionsCount() <= 4)
{
const TfLiteEvalTensor* paddings =
tflite::micro::GetEvalInput(context, node, /*index=*/1);
int32_t err = xa_nn_pad_32_32(
tflite::micro::GetTensorData<int32_t>(output),
tflite::micro::GetTensorShape(output).DimsData(),
tflite::micro::GetTensorData<int32_t>(input),
tflite::micro::GetTensorShape(input).DimsData(),
tflite::micro::GetTensorData<int32_t>(paddings),
tflite::micro::GetTensorShape(paddings).DimsData(),
tflite::micro::GetTensorShape(output).DimensionsCount(),
tflite::micro::GetTensorShape(input).DimensionsCount(),
tflite::micro::GetTensorShape(paddings).DimensionsCount(),
pad_value);
if (err != 0) return kTfLiteError;
}
else
{
reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<int32_t>(input),
&pad_value, tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<int32_t>(output));
}
#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
reference_ops::Pad(data->params, tflite::micro::GetTensorShape(input),
tflite::micro::GetTensorData<int32_t>(input),
&pad_value, tflite::micro::GetTensorShape(output),
tflite::micro::GetTensorData<int32_t>(output));
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
} break;
default:

Expand Down
22 changes: 10 additions & 12 deletions tensorflow/lite/micro/kernels/xtensa/svdf.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ limitations under the License.
namespace tflite {
namespace {

#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)

TfLiteStatus EvalIntegerSvdfHifi(TfLiteContext* context, TfLiteNode* node,
const TfLiteEvalTensor* input_tensor,
Expand All @@ -60,11 +60,11 @@ TfLiteStatus EvalIntegerSvdfHifi(TfLiteContext* context, TfLiteNode* node,

// Left shift the activation_state.
int num_bytes = sizeof(*state_ptr) * (n_batch * n_filter * n_memory - 1);
#if defined(HIFI5)
#if defined(HIFI5) || defined(HIFI_IQ)
memcpy(state_ptr, state_ptr + 1, num_bytes);
#else
xa_nn_memmove_16(state_ptr, state_ptr + 1, (num_bytes >> 1));
#endif // defined(HIFI5)
#endif // defined(HIFI5) || defined(HIFI_IQ)

// Note: no need to clear the latest activation, matmul is not accumulative.

Expand Down Expand Up @@ -108,15 +108,15 @@ TfLiteStatus EvalIntegerSvdfHifi(TfLiteContext* context, TfLiteNode* node,
}
return kTfLiteOk;
}
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)

void* Init(TfLiteContext* context, const char* buffer, size_t length) {
TFLITE_DCHECK(context != nullptr);
return context->AllocatePersistentBuffer(context, sizeof(OpDataSvdf));
}

TfLiteStatus PrepareInt8(TfLiteContext* context, TfLiteNode* node) {
#if defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#if defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
TFLITE_DCHECK(node->builtin_data != nullptr);
const auto* params = static_cast<const TfLiteSVDFParams*>(node->builtin_data);

Expand Down Expand Up @@ -252,12 +252,11 @@ TfLiteStatus PrepareInt8(TfLiteContext* context, TfLiteNode* node) {
return kTfLiteOk;
#else
return PrepareSvdf(context, node);
#endif // defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) ||
// defined(HIFI5)
#endif // defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
}

TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
#if defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#if defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)

MicroContext* micro_context = GetMicroContext(context);
TfLiteTensor* input =
Expand All @@ -278,8 +277,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
return status;
#else
return PrepareSvdf(context, node);
#endif // defined(HIFIMINI) || defined(HIFI3) || defined(HIFI4) ||
// defined(HIFI5)
#endif
}

TfLiteStatus EvalInt8(TfLiteContext* context, TfLiteNode* node) {
Expand Down Expand Up @@ -308,15 +306,15 @@ TfLiteStatus EvalInt8(TfLiteContext* context, TfLiteNode* node) {
return EvalIntegerSvdfHifimini(context, node, input, weights_feature,
weights_time, bias, params, activation_state,
output, data);
#elif defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#elif defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
return EvalIntegerSvdfHifi(context, node, input, weights_feature,
weights_time, bias, params, activation_state,
output, data);
#else
EvalInt16SvdfReference(context, node, input, weights_feature, weights_time,
bias, params, activation_state, output, data);
return kTfLiteOk;
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5)
#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)
}

TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
Expand Down
Loading