diff --git a/tensorflow/lite/micro/kernels/xtensa/add.cc b/tensorflow/lite/micro/kernels/xtensa/add.cc index d3c23713cfb..a053ad7eb16 100644 --- a/tensorflow/lite/micro/kernels/xtensa/add.cc +++ b/tensorflow/lite/micro/kernels/xtensa/add.cc @@ -49,6 +49,31 @@ TfLiteStatus EvalAdd(TfLiteContext* context, TfLiteNode* node, tflite::ArithmeticParams op_params; SetActivationParams(data->output_activation_min_f32, data->output_activation_max_f32, &op_params); +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) + int err; + const RuntimeShape extended_input1_shape = + RuntimeShape::ExtendedShape(4, tflite::micro::GetTensorShape(input1)); + const RuntimeShape extended_input2_shape = + RuntimeShape::ExtendedShape(4, tflite::micro::GetTensorShape(input2)); + const RuntimeShape extended_output_shape = + RuntimeShape::ExtendedShape(4, tflite::micro::GetTensorShape(output)); + + err = xa_nn_elm_add_broadcast_4D_f32xf32_f32( + tflite::micro::GetTensorData(output), + extended_output_shape.DimsData(), + tflite::micro::GetTensorData(input1), + extended_input1_shape.DimsData(), + tflite::micro::GetTensorData(input2), + extended_input2_shape.DimsData()); + TF_LITE_ENSURE(context, err == 0); + + err = xa_nn_vec_activation_min_max_f32_f32( + tflite::micro::GetTensorData(output), + tflite::micro::GetTensorData(output), + data->output_activation_min_f32, data->output_activation_max_f32, + extended_output_shape.FlatSize()); + TF_LITE_ENSURE(context, err == 0); +#else // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) if (data->requires_broadcast) { reference_ops::BroadcastAdd4DSlow( op_params, tflite::micro::GetTensorShape(input1), @@ -65,6 +90,7 @@ TfLiteStatus EvalAdd(TfLiteContext* context, TfLiteNode* node, tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); } +#endif // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) } break; case kTfLiteInt32: { tflite::ArithmeticParams op_params; @@ -113,11 +139,11 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, op_params.output_shift = data->output_shift; SetActivationParams(data->output_activation_min, data->output_activation_max, &op_params); -#if !(defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) +#if !(defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) bool need_broadcast = reference_ops::ProcessBroadcastShapes( tflite::micro::GetTensorShape(input1), tflite::micro::GetTensorShape(input2), &op_params); -#endif // !defined(HIFI3) && !defined(HIFI4) && !defined(HIFI5) +#endif switch (output->type) { case kTfLiteInt8: { @@ -126,7 +152,7 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, *(reinterpret_cast(node->user_data)); AddEvalQuantizedVision(context, node, *params, op_data, input1, input2, output); -#elif defined(HIFI3) || defined(HIFI4) || defined(HIFI5) // defined(VISION_P6) +#elif defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) // #if defined(VISION_P6) int err; const RuntimeShape extended_input1_shape = RuntimeShape::ExtendedShape(4, tflite::micro::GetTensorShape(input1)); @@ -145,12 +171,12 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, extended_input1_shape.DimsData(), op_params.input1_offset, op_params.input1_shift, op_params.input1_multiplier, tflite::micro::GetTensorData(input2), - extended_input2_shape.DimsData(), op_params.input2_offset, - op_params.input2_shift, op_params.input2_multiplier, - op_params.left_shift); + extended_input2_shape.DimsData(), + op_params.input2_offset, op_params.input2_shift, + op_params.input2_multiplier, op_params.left_shift); TF_LITE_ENSURE(context, err == 0); -#else // defined(VISION_P6) +#else // defined(VISION_P6) if (need_broadcast) { reference_integer_ops::BroadcastAdd4DSlow( op_params, tflite::micro::GetTensorShape(input1), @@ -168,11 +194,11 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); } -#endif // defined(VISION_P6) +#endif // defined(VISION_P6) break; } case kTfLiteInt16: { -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) int err; const RuntimeShape extended_input1_shape = RuntimeShape::ExtendedShape(4, tflite::micro::GetTensorShape(input1)); @@ -196,7 +222,7 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, op_params.left_shift); TF_LITE_ENSURE(context, err == 0); -#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#else // #if defined(HIFI4) || defined(HIFI5) if (need_broadcast) { reference_ops::BroadcastAdd4DSlow( op_params, tflite::micro::GetTensorShape(input1), @@ -214,7 +240,7 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, tflite::micro::GetTensorData(output), false); } -#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#endif // #if defined(HIFI4) || defined(HIFI5) break; } default: diff --git a/tensorflow/lite/micro/kernels/xtensa/dequantize.cc b/tensorflow/lite/micro/kernels/xtensa/dequantize.cc index f2f4a7d9cb7..14d7721f6d2 100644 --- a/tensorflow/lite/micro/kernels/xtensa/dequantize.cc +++ b/tensorflow/lite/micro/kernels/xtensa/dequantize.cc @@ -47,7 +47,7 @@ TfLiteStatus DequantizeEval(TfLiteContext* context, TfLiteNode* node) { switch (input->type) { case kTfLiteInt8: { -#if HIFI_VFPU && (defined(HIFI5) || defined(HIFI4) || defined(HIFI3)) +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) int err; const int8_t* input_data_ptr; float* output_data_ptr; @@ -61,17 +61,17 @@ TfLiteStatus DequantizeEval(TfLiteContext* context, TfLiteNode* node) { output_data_ptr, input_data_ptr, data->quantization_params.zero_point, data->quantization_params.scale, flat_size); TF_LITE_ENSURE(context, (err == 0)); -#else // HIFI_VFPU && (defined(HIFI5) || defined(HIFI4) || defined(HIFI3)) +#else // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) reference_ops::Dequantize(data->quantization_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); -#endif // HIFI_VFPU && (defined(HIFI5) || defined(HIFI4) || defined(HIFI3)) +#endif // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) break; } case kTfLiteInt16: { -#if HIFI_VFPU && (defined(HIFI5) || defined(HIFI4) || defined(HIFI3)) +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) int err; const int16_t* input_data_ptr; float* output_data_ptr; @@ -84,13 +84,13 @@ TfLiteStatus DequantizeEval(TfLiteContext* context, TfLiteNode* node) { output_data_ptr, input_data_ptr, data->quantization_params.zero_point, data->quantization_params.scale, flat_size); TF_LITE_ENSURE(context, (err == 0)); -#else // HIFI_VFPU && (defined(HIFI5) || defined(HIFI4) || defined(HIFI3)) +#else // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) reference_ops::Dequantize(data->quantization_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); -#endif // HIFI_VFPU && (defined(HIFI5) || defined(HIFI4) || defined(HIFI3)) +#endif // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) break; } case kTfLiteUInt8: diff --git a/tensorflow/lite/micro/kernels/xtensa/hifimini/fixedpoint_utils.h b/tensorflow/lite/micro/kernels/xtensa/hifimini/fixedpoint_utils.h new file mode 100644 index 00000000000..42bf971b360 --- /dev/null +++ b/tensorflow/lite/micro/kernels/xtensa/hifimini/fixedpoint_utils.h @@ -0,0 +1,139 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_MICRO_KERNELS_XTENSA_HIFIMINI_FIXEDPOINT_UTILS_H_ +#define TENSORFLOW_LITE_MICRO_KERNELS_XTENSA_HIFIMINI_FIXEDPOINT_UTILS_H_ + +#if defined(HIFIMINI) +#include + +#include +#include +#include + +#include "tensorflow/lite/kernels/internal/compatibility.h" +#include "tensorflow/lite/micro/kernels/xtensa/xtensa.h" + +namespace tflite { + +// INT24 MIN/MAX +#define INT24_MIN -8388608 +#define INT24_MAX 8388607 + +// Multiply 24bit value by a quantized multiplier (w/ shift) and returns a 48bit +// aligned value in the QR register. +inline ae_q56s MultiplyByQuantizedMultiplier(ae_p24x2s x_24x2, + int32_t quantized_multiplier, + int shift) { + // A value with 1 sign bit, N integer bits and M fractional bits is + // represented as QN+1.M since the sign bit is included in the integer bits. + // + // The Q notation in this method explains the values represented in each + // variable, along with an implicit division since the quantized_multiplier + // represents a value between 0.5 and 1.0 (Q1.X-1 where X is the bit precision + // of the type). + // + // Load the quantized multiplier into the PR register. + // NOTE: This method assumes that this param has been calculated for 24bit + // space - not 32bits. + // Q32.0 / 2^23 -> Q24.0 / 2^23 representing a Q1.23 multiplier. + ae_p24x2s quantized_multiplier_24x2 = AE_MOVPA24(quantized_multiplier); + // Shift right by 23 - 16 bits minus the specified shift. This is because we + // keep 16 fractional bits until the end to perform rounding. Subtract shift + // since shift is a left shift, and the 23-16 is a right shift. + int shift_amount = 7 - shift; + + // Find the product of x and the quantized_multiplier. + // Q24.0 / 2^23 * Q24.0 = Q48.0 / 2^23 + // Q48.0 / 2^23 >> 7 = Q48.0 / 2^16 + ae_q56s result_56 = AE_MULP24S_HH(x_24x2, quantized_multiplier_24x2); + + // Shift right if shift amount is positive, left if shift amount is negative. + if (shift_amount >= 0) { + result_56 = AE_Q56S_SRA(result_56, shift_amount); + } else { + result_56 = AE_Q56S_SLA(result_56, -shift_amount); + } + + // Round off the bottom 16 bits. + // Q48.0 / 2^16 -> Q32.0 aligned to 48 bits. + result_56 = AE_ROUNDSQ32SYM(result_56); + return result_56; +} + +// Multiply 32bit value by a quantized multiplier (w/ shift) and returns a 48bit +// aligned value in the QR register. +inline ae_q56s MultiplyByQuantizedMultiplierResult48Bit( + int32_t x, int32_t quantized_multiplier, int shift) { + // Convert x into a 2x24bit PR register file. If x is outside the numerical + // limits of a 24bit integer, the "fractional" or lower 8bits are discarded. + // If x is within the range of a 24 bit integer, the "signed" or upper 8bits + // are discarded. + ae_p24x2s x_24x2; + if (x > INT24_MIN && x < INT24_MAX) { + x_24x2 = AE_MOVPA24(x); + } else { + x_24x2 = static_cast(*reinterpret_cast(&x)); + shift += 8; + } + + return MultiplyByQuantizedMultiplier(x_24x2, quantized_multiplier, shift); +} + +// Calculate quantization params for 24bit runtimes. +inline void QuantizeMultiplierForInt24(float multiplier, + int32_t* quantized_multiplier, + int* shift) { + if (multiplier == 0.0f) { + *quantized_multiplier = 0; + *shift = 0; + return; + } + + // Special cased to 24bit: + const float q = std::frexp(multiplier, shift); + auto q_fixed = static_cast(std::round(q * (1 << 23))); + + TFLITE_CHECK(q_fixed <= (1 << 23)); + if (q_fixed == (1 << 23)) { + q_fixed /= 2; + ++*shift; + } + TFLITE_CHECK_LE(q_fixed, INT24_MAX); + + // Ensure shift does not exceed 24-bit range. + TFLITE_CHECK_LE(*shift, 23); + if (*shift < -23) { + *shift = 0; + q_fixed = 0; + } + *quantized_multiplier = static_cast(q_fixed); +} + +// Convert a floating point number to a Q representation for 24 bit integers. +inline int CreateQConstantForInt24(int integer_bits, float f) { + const float min_bounds = static_cast(INT24_MIN); + const float max_bounds = static_cast(INT24_MAX); + + int fractional_bits = 23 - integer_bits; + float raw = std::round(f * static_cast(1 << fractional_bits)); + raw = std::max(raw, min_bounds); + raw = std::min(raw, max_bounds); + return static_cast(raw); +} + +} // namespace tflite +#endif // defined(HIFIMINI) +#endif // TENSORFLOW_LITE_MICRO_KERNELS_XTENSA_HIFIMINI_FIXEDPOINT_UTILS_H_ diff --git a/tensorflow/lite/micro/kernels/xtensa/hifimini/fully_connected.cc b/tensorflow/lite/micro/kernels/xtensa/hifimini/fully_connected.cc new file mode 100644 index 00000000000..b63c5001a91 --- /dev/null +++ b/tensorflow/lite/micro/kernels/xtensa/hifimini/fully_connected.cc @@ -0,0 +1,118 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if defined(HIFIMINI) +#include "tensorflow/lite/micro/kernels/fully_connected.h" + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/reference/fully_connected.h" +#include "tensorflow/lite/kernels/internal/reference/integer_ops/fully_connected.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/xtensa/hifimini/fixedpoint_utils.h" +#include "tensorflow/lite/micro/kernels/xtensa/xtensa.h" + +namespace tflite { + +void FullyConnectedEvalHifimini( + const FullyConnectedParams& params, const RuntimeShape& input_shape, + const int8_t* input_data, const RuntimeShape& filter_shape, + const int8_t* filter_data, const RuntimeShape& bias_shape, + const int32_t* bias_data, const RuntimeShape& output_shape, + int8_t* output_data) { + const int32_t input_offset = params.input_offset; + const int32_t filter_offset = params.weights_offset; + const int32_t output_offset = params.output_offset; + const int32_t output_multiplier = params.output_multiplier; + const int output_shift = params.output_shift; + const int32_t output_activation_min = params.quantized_activation_min; + const int32_t output_activation_max = params.quantized_activation_max; + + const int filter_dim_count = filter_shape.DimensionsCount(); + const int batches = output_shape.Dims(0); + const int output_depth = output_shape.Dims(1); + const int accum_depth = filter_shape.Dims(filter_dim_count - 1); + const int accum_depth_iters = accum_depth / 2; + + ae_p24x2s offsets_input_24x2 = AE_MOVPA24(input_offset); + ae_p24x2s offsets_filter_24x2 = AE_MOVPA24(filter_offset); + ae_q56s output_offset_56 = AE_CVTQ48A32S(output_offset); + ae_q56s output_activation_max_56 = AE_CVTQ48A32S(output_activation_max); + ae_q56s output_activation_min_56 = AE_CVTQ48A32S(output_activation_min); + + for (int b = 0; b < batches; ++b) { + for (int out_c = 0; out_c < output_depth; ++out_c) { + // Load intrinsics advance pointer before loading so backoff data pointers + // by two before loading: + const int8_t* input_ptr = (input_data + b * accum_depth) - 2; + const int8_t* filter_ptr = (filter_data + out_c * accum_depth) - 2; + + // Main accumulator register entry for loop: + ae_q56s sum_56 = AE_ZEROQ56(); + + for (int d = 0; d < accum_depth_iters; d++) { + // Load the signed 8bit values into the PR register: + ae_p24x2s input_24x2; + ae_p24x2s filter_24x2; + AE_LP8X2F_IU(input_24x2, input_ptr, 2); + AE_LP8X2F_IU(filter_24x2, filter_ptr, 2); + + // Right shift the signed 8bit values to expand to signed 24bit values: + input_24x2 = AE_P24X2S_SRAI(input_24x2, 16); + filter_24x2 = AE_P24X2S_SRAI(filter_24x2, 16); + + // Add offsets to data values (24 bit aligned): + input_24x2 = AE_P24S_ADDS_P24X2S(offsets_input_24x2, input_24x2); + filter_24x2 = AE_P24S_ADDS_P24X2S(offsets_filter_24x2, filter_24x2); + + // 24x2 signed integer dual MAC w/ addition into 56bit accumulator (48 + // bit aligned): + AE_MULAAP24S_HH_LL(sum_56, input_24x2, filter_24x2); + } + + // Left shift to get back into 32bit space (right padded to 48bit): + sum_56 = AE_Q56S_SLAI(sum_56, 16); + + // Add bias data if needed: + if (bias_data) { + ae_q56s bias_56 = AE_CVTQ48A32S(bias_data[out_c]); + sum_56 = AE_ADDQ56(sum_56, bias_56); + } + + // Shift left into 24bit space and place back on PR register: + sum_56 = AE_Q56S_SLAI(sum_56, 8); + ae_p24x2s sum_24x2 = AE_TRUNCP24Q48(sum_56); + + // MultiplyByQuantizedMultiplier returns a 48bit aligned value + sum_56 = MultiplyByQuantizedMultiplier(sum_24x2, output_multiplier, + output_shift); + + // Add output_offset and cap min/max values: + sum_56 = AE_ADDQ56(sum_56, output_offset_56); + sum_56 = AE_MINQ56S(sum_56, output_activation_max_56); + sum_56 = AE_MAXQ56S(sum_56, output_activation_min_56); + + output_data[out_c + output_depth * b] = + static_cast(AE_TRUNCA32Q48(sum_56)); + } + } +} + +} // namespace tflite +#endif // defined(HIFIMINI) diff --git a/tensorflow/lite/micro/kernels/xtensa/hifimini/svdf.cc b/tensorflow/lite/micro/kernels/xtensa/hifimini/svdf.cc new file mode 100644 index 00000000000..08ef4d9bb0c --- /dev/null +++ b/tensorflow/lite/micro/kernels/xtensa/hifimini/svdf.cc @@ -0,0 +1,237 @@ +/* Copyright 2023 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if defined(HIFIMINI) +#include "tensorflow/lite/micro/kernels/svdf.h" + +#include + +#include "tensorflow/lite/c/builtin_op_data.h" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/quantization_util.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/kernels/op_macros.h" +#include "tensorflow/lite/micro/kernels/activation_utils.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/kernels/xtensa/hifimini/fixedpoint_utils.h" +#include "tensorflow/lite/micro/kernels/xtensa/xtensa.h" +#include "tensorflow/lite/micro/kernels/xtensa/xtensa_svdf.h" + +namespace tflite { + +/** + * This version of SVDF is specific to TFLite Micro. It contains only a full + * integer receipe with optimizations for the Xtensa HiFiMini platform. + * + * Note: passing OpDataSvdf by value might seem like an oversight but it helps + * reduce the latency. See b/155656675 for more details. + */ +TfLiteStatus EvalIntegerSvdfHifimini( + TfLiteContext* context, TfLiteNode* node, + const TfLiteEvalTensor* input_tensor, + const TfLiteEvalTensor* weights_feature_tensor, + const TfLiteEvalTensor* weights_time_tensor, + const TfLiteEvalTensor* bias_tensor, const TfLiteSVDFParams* params, + TfLiteEvalTensor* activation_state_tensor, TfLiteEvalTensor* output_tensor, + OpDataSvdf data) { + const int n_rank = params->rank; + const int n_batch = input_tensor->dims->data[0]; + const int n_input = input_tensor->dims->data[1]; + const int n_filter = weights_feature_tensor->dims->data[0]; + const int n_unit = n_filter / n_rank; + const int n_memory = weights_time_tensor->dims->data[1]; + + TFLITE_DCHECK(context != nullptr); + TFLITE_DCHECK(context->GetScratchBuffer != nullptr); + + int32_t* scratch_tensor = static_cast( + context->GetScratchBuffer(context, data.scratch_tensor_index)); + TFLITE_DCHECK(scratch_tensor != nullptr); + int32_t* scratch_output_tensor = static_cast( + context->GetScratchBuffer(context, data.scratch_output_tensor_index)); + TFLITE_DCHECK(scratch_output_tensor != nullptr); + + // Shift states. + int16_t* const state_ptr = + tflite::micro::GetTensorData(activation_state_tensor); + + // Left shift the activation_state. + { + int16_t* new_state_start = state_ptr; + const int16_t* old_state_start = state_ptr + 1; + const int16_t* old_state_end = state_ptr + n_batch * n_filter * n_memory; + while (old_state_start != old_state_end) { + *new_state_start++ = *old_state_start++; + } + } + + // Note: no need to clear the latest activation, matmul is not accumulative. + + // Feature matmul. + { + const int8_t* input = tflite::micro::GetTensorData(input_tensor); + const int8_t* weight_feature = + tflite::micro::GetTensorData(weights_feature_tensor); + int16_t* result_in_batch = state_ptr + (n_memory - 1); + + ae_q56s output_int16_max_56 = AE_CVTQ48A32S(INT16_MAX); + ae_q56s output_int16_min_56 = AE_CVTQ48A32S(INT16_MIN); + ae_p24x2s input_zp_24x2 = AE_MOVPA24(data.input_zero_point); + + for (int b = 0; b < n_batch; b++) { + const int8_t* weight_feature_ptr = weight_feature - 2; + + for (int r = 0; r < n_filter; r++) { + ae_q56s dot_prod_56 = AE_ZEROQ56(); + + const int8_t* input_batch_ptr = input + b * n_input; + const int8_t* offset_input_batch_ptr = input_batch_ptr - 2; + + int num_iters = n_input / 2; + for (int c = 0; c < num_iters; c++) { + // Load 2 sets of values: + ae_p24x2s weight_feature_ptr_24x2; + ae_p24x2s input_batch_ptr_24x2; + AE_LP8X2F_IU(weight_feature_ptr_24x2, weight_feature_ptr, 2); + AE_LP8X2F_IU(input_batch_ptr_24x2, offset_input_batch_ptr, 2); + + // Right shift the signed 8bit values to expand to signed 24bit + // values: + weight_feature_ptr_24x2 = AE_P24X2S_SRAI(weight_feature_ptr_24x2, 16); + input_batch_ptr_24x2 = AE_P24X2S_SRAI(input_batch_ptr_24x2, 16); + + // First subtract input_zp from input_batch_ptr_24x2: + input_batch_ptr_24x2 = + AE_SUBSP24S(input_batch_ptr_24x2, input_zp_24x2); + + // Multiply accum: + AE_MULAAP24S_HH_LL(dot_prod_56, weight_feature_ptr_24x2, + input_batch_ptr_24x2); + } + + // Left shift 48bit value into 24bit space and place on the PR register: + dot_prod_56 = AE_Q56S_SLAI(dot_prod_56, 24); + ae_p24x2s dot_prod_24x2 = AE_TRUNCP24Q48(dot_prod_56); + + dot_prod_56 = MultiplyByQuantizedMultiplier( + dot_prod_24x2, data.effective_scale_1_a, data.effective_scale_1_b); + + // Cap min/max and convert to int32_t: + dot_prod_56 = AE_MAXQ56S(dot_prod_56, output_int16_min_56); + dot_prod_56 = AE_MINQ56S(dot_prod_56, output_int16_max_56); + // Truncate immediately since the QR register is already 32 bit aligned: + // This assumes state is symmetrically quantized. Otherwise last bit of + // state should be initialized to its zero point and accumulate the + // dot_prod. + // Equivalent as the following: + // result_in_batch = zero point, which happens to be zero. + // result_in_batch += dot_prod_56. + *result_in_batch = AE_TRUNCA32Q48(dot_prod_56); + result_in_batch += n_memory; + } + } + } + + // Time. + { + for (int b = 0; b < n_batch; ++b) { + int32_t* scratch_ptr_batch = scratch_tensor + b * n_filter; + + // Perform batched vector dot product: + const int16_t* vector1_ptr = + tflite::micro::GetTensorData(weights_time_tensor); + const int16_t* vector2_ptr = state_ptr + b * n_memory * n_filter; + + const ae_p16x2s* offset_vector1 = + reinterpret_cast(vector1_ptr - 2); + const ae_p16x2s* offset_vector2 = + reinterpret_cast(vector2_ptr - 2); + + for (int i = 0; i < n_filter; i++) { + *scratch_ptr_batch = 0; + + ae_q56s sum_56 = AE_ZEROQ56(); + int num_iters = n_memory / 2; + for (int j = 0; j < num_iters; j++) { + ae_p24x2s vector1_24x2; + ae_p24x2s vector2_24x2; + AE_LP16X2F_IU(vector1_24x2, offset_vector1, 4); + AE_LP16X2F_IU(vector2_24x2, offset_vector2, 4); + AE_MULAAP24S_HH_LL(sum_56, vector1_24x2, vector2_24x2); + } + // Truncate directly since values are already 32bit aligned: + *scratch_ptr_batch = AE_TRUNCA32Q48(sum_56); + scratch_ptr_batch++; + } + } + } + + // Reduce, add bias, rescale, activation. + { + // Add bias. + if (bias_tensor) { + // Vector batch assign: + const int32_t* bias_data = + tflite::micro::GetTensorData(bias_tensor); + for (int i = 0; i < n_batch; ++i) { + int32_t* output_ptr = scratch_output_tensor + i * n_unit; + const int32_t* bias_ptr = bias_data; + for (int j = 0; j < n_unit; ++j) { + *output_ptr++ = *bias_ptr++; + } + } + } else { + int32_t* output_ptr = scratch_output_tensor; + for (int i = 0; i < n_batch * n_unit; ++i) { + *output_ptr++ = 0; + } + } + + // Reduce. + for (int b = 0; b < n_batch; ++b) { + int32_t* output_temp_ptr = scratch_output_tensor + b * n_unit; + int32_t* scratch_ptr_batch = scratch_tensor + b * n_filter; + + // Reduction sum vector + for (int i = 0; i < n_unit; ++i) { + for (int j = 0; j < n_rank; ++j) { + output_temp_ptr[i] += *scratch_ptr_batch++; + } + } + } + + // Rescale. + ae_q56s output_int8_max_56 = AE_CVTQ48A32S(INT8_MAX); + ae_q56s output_int8_min_56 = AE_CVTQ48A32S(INT8_MIN); + ae_q56s output_zp_56 = AE_CVTQ48A32S(data.output_zero_point); + for (int i = 0; i < n_batch * n_unit; ++i) { + ae_q56s x_56 = MultiplyByQuantizedMultiplierResult48Bit( + scratch_output_tensor[i], data.effective_scale_2_a, + data.effective_scale_2_b); + // Add output adjustment: + x_56 = AE_ADDQ56(x_56, output_zp_56); + // Cap min/max and convert to int32_t (already aligned to 32bit): + x_56 = AE_MAXQ56S(x_56, output_int8_min_56); + x_56 = AE_MINQ56S(x_56, output_int8_max_56); + tflite::micro::GetTensorData(output_tensor)[i] = + static_cast(AE_TRUNCA32Q48(x_56)); + } + } + return kTfLiteOk; +} +} // namespace tflite +#endif // defined(HIFIMINI) diff --git a/tensorflow/lite/micro/kernels/xtensa/quantize.cc b/tensorflow/lite/micro/kernels/xtensa/quantize.cc index 06d4fbbff19..feb9064cce0 100644 --- a/tensorflow/lite/micro/kernels/xtensa/quantize.cc +++ b/tensorflow/lite/micro/kernels/xtensa/quantize.cc @@ -29,7 +29,7 @@ limitations under the License. namespace tflite { namespace { -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { TFLITE_DCHECK(node->user_data != nullptr); auto* op_data = static_cast(node->user_data); @@ -52,10 +52,7 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { } default: - MicroPrintf("Input %s, output %s not supported.", - TfLiteTypeGetName(input->type), - TfLiteTypeGetName(output->type)); - return kTfLiteError; + return EvalQuantizeReference(context, node); } break; } @@ -64,17 +61,34 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { switch (output->type) { case kTfLiteUInt8: { int size = ElementCount(*input->dims); +#if (defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) + int32_t zero_point = op_data->quantization_params.zero_point; + const int8_t* input_data_ptr; + uint8_t* output_data_ptr; + input_data_ptr = tflite::micro::GetTensorData(input); + output_data_ptr = tflite::micro::GetTensorData(output); + + TF_LITE_ENSURE_EQ( + context, + xa_nn_elm_requantize_asym8s_asym8u( + output_data_ptr, input_data_ptr, op_data->input_zero_point, + zero_point, op_data->requantize_output_shift, + op_data->requantize_output_multiplier, size), + 0); +#else // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) reference_ops::Requantize( tflite::micro::GetTensorData(input), size, op_data->requantize_output_multiplier, op_data->requantize_output_shift, op_data->input_zero_point, op_data->quantization_params.zero_point, tflite::micro::GetTensorData(output)); +#endif // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) break; } case kTfLiteInt8: { int size = ElementCount(*input->dims); +#if (defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) int32_t zero_point = op_data->quantization_params.zero_point; const int8_t* input_data_ptr; int8_t* output_data_ptr; @@ -88,23 +102,47 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { zero_point, op_data->requantize_output_shift, op_data->requantize_output_multiplier, size), 0); +#else // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + op_data->requantize_output_multiplier, + op_data->requantize_output_shift, op_data->input_zero_point, + op_data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); +#endif // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) break; } case kTfLiteInt16: { int size = ElementCount(*input->dims); +#if (defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) int32_t zero_point = op_data->quantization_params.zero_point; + const int8_t* input_data_ptr; + int16_t* output_data_ptr; + input_data_ptr = tflite::micro::GetTensorData(input); + output_data_ptr = tflite::micro::GetTensorData(output); + + TF_LITE_ENSURE_EQ( + context, + xa_nn_elm_requantize_asym8s_asym16s( + output_data_ptr, input_data_ptr, op_data->input_zero_point, + zero_point, op_data->requantize_output_shift, + op_data->requantize_output_multiplier, size), + 0); +#else // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) reference_ops::Requantize( tflite::micro::GetTensorData(input), size, op_data->requantize_output_multiplier, op_data->requantize_output_shift, op_data->input_zero_point, - zero_point, tflite::micro::GetTensorData(output)); + op_data->quantization_params.zero_point, tflite::micro::GetTensorData(output)); +#endif // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) break; } case kTfLiteInt32: { int size = ElementCount(*input->dims); int32_t zero_point = op_data->quantization_params.zero_point; +#if (defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) const int8_t* input_data_ptr; int32_t* output_data_ptr; input_data_ptr = tflite::micro::GetTensorData(input); @@ -117,6 +155,13 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { zero_point, op_data->requantize_output_shift, op_data->requantize_output_multiplier, size), 0); +#else // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + op_data->requantize_output_multiplier, + op_data->requantize_output_shift, op_data->input_zero_point, + zero_point, tflite::micro::GetTensorData(output)); +#endif // #if defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) break; } @@ -134,20 +179,30 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { switch (output->type) { case kTfLiteInt8: { int size = ElementCount(*input->dims); - TF_LITE_ENSURE_EQ(context, - xa_nn_elm_requantize_asym16s_asym8s( - tflite::micro::GetTensorData(output), - tflite::micro::GetTensorData(input), - op_data->input_zero_point, - op_data->quantization_params.zero_point, - op_data->requantize_output_shift, - op_data->requantize_output_multiplier, size), - 0); +#if (defined(HIFI4) || defined(HIFI4_INTERNAL) || defined(HIFI5) || defined(HIFI_IQ)) + TF_LITE_ENSURE_EQ( + context, + xa_nn_elm_requantize_asym16s_asym8s( + tflite::micro::GetTensorData(output), + tflite::micro::GetTensorData(input), + op_data->input_zero_point, op_data->quantization_params.zero_point, + op_data->requantize_output_shift, + op_data->requantize_output_multiplier, size), + 0); +#else + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + op_data->requantize_output_multiplier, + op_data->requantize_output_shift, op_data->input_zero_point, + op_data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); +#endif break; } case kTfLiteInt16: { int size = ElementCount(*input->dims); +#if (defined(HIFI4) || defined(HIFI4_INTERNAL) || defined(HIFI5) || defined(HIFI_IQ)) TF_LITE_ENSURE_EQ(context, xa_nn_elm_requantize_asym16s_asym16s( tflite::micro::GetTensorData(output), @@ -157,11 +212,20 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { op_data->requantize_output_shift, op_data->requantize_output_multiplier, size), 0); +#else + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + op_data->requantize_output_multiplier, + op_data->requantize_output_shift, op_data->input_zero_point, + op_data->quantization_params.zero_point, + tflite::micro::GetTensorData(output)); +#endif break; } case kTfLiteInt32: { int size = ElementCount(*input->dims); +#if (defined(HIFI4) || defined(HIFI4_INTERNAL) || defined(HIFI5) || defined(HIFI_IQ)) TF_LITE_ENSURE_EQ(context, xa_nn_elm_requantize_asym16s_asym32s( tflite::micro::GetTensorData(output), @@ -171,6 +235,14 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { op_data->requantize_output_shift, op_data->requantize_output_multiplier, size), 0); +#else + int32_t zero_point = op_data->quantization_params.zero_point; + reference_ops::Requantize( + tflite::micro::GetTensorData(input), size, + op_data->requantize_output_multiplier, + op_data->requantize_output_shift, op_data->input_zero_point, + zero_point, tflite::micro::GetTensorData(output)); +#endif // defined(HIFI5) break; } @@ -221,7 +293,7 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { case kTfLiteFloat32: { switch (output->type) { case kTfLiteInt8: { -#if HIFI_VFPU +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) int size = ElementCount(*input->dims); int32_t zero_point = op_data->quantization_params.zero_point; const float* input_data_ptr; @@ -236,19 +308,18 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { static_cast(op_data->quantization_params.scale), zero_point, size), 0); -#else // #if HIFI_VFPU +#else // #if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI4) || defined(HIFI5)) reference_ops::AffineQuantize( op_data->quantization_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); -#endif // #if HIFI_VFPU +#endif // #if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI4) || defined(HIFI5)) break; } - case kTfLiteInt16: { -#if HIFI_VFPU +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) int size = ElementCount(*input->dims); int32_t zero_point = op_data->quantization_params.zero_point; const float* input_data_ptr; @@ -263,14 +334,14 @@ TfLiteStatus EvalXtensa(TfLiteContext* context, TfLiteNode* node) { static_cast(op_data->quantization_params.scale), zero_point, size), 0); -#else // #if HIFI_VFPU +#else // #if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI4) || defined(HIFI5)) && !defined(HIFI_IQ) reference_ops::AffineQuantize( op_data->quantization_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); -#endif // #if HIFI_VFPU +#endif // #if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI4) || defined(HIFI5)) && !defined(HIFI_IQ) break; } @@ -328,7 +399,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) return EvalXtensa(context, node); #else return EvalQuantizeReference(context, node); diff --git a/tensorflow/lite/micro/kernels/xtensa/strided_slice.cc b/tensorflow/lite/micro/kernels/xtensa/strided_slice.cc index 8ebf7247a23..92209bc60b1 100644 --- a/tensorflow/lite/micro/kernels/xtensa/strided_slice.cc +++ b/tensorflow/lite/micro/kernels/xtensa/strided_slice.cc @@ -30,12 +30,59 @@ limitations under the License. namespace tflite { namespace { -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) -void StridedSlice_int16_hifi4opt(const tflite::StridedSliceParams& op_params, +#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) +void StridedSlice_int16_hifi(const tflite::StridedSliceParams& op_params, + const RuntimeShape& unextended_input_shape, + const int16_t* input_data, + const RuntimeShape& unextended_output_shape, + int16_t* output_data) { + using ::tflite::strided_slice::StartForAxis; + using ::tflite::strided_slice::StopForAxis; + + ruy::profiler::ScopeLabel label("StridedSlice"); + + // Note that the output_shape is not used herein. + tflite::StridedSliceParams params_copy = op_params; + + TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 5); + TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 5); + const RuntimeShape input_shape = + RuntimeShape::ExtendedShape(5, unextended_input_shape); + const RuntimeShape output_shape = + RuntimeShape::ExtendedShape(5, unextended_output_shape); + + // Reverse and pad to 5 dimensions because that is what the runtime code + // requires (ie. all shapes must be 5D and are given backwards). + ::tflite::strided_slice::StridedSlicePadIndices(¶ms_copy, 5); + + const int start_0 = StartForAxis(params_copy, input_shape, 0); + const int stop_0 = StopForAxis(params_copy, input_shape, 0, start_0); + const int start_1 = StartForAxis(params_copy, input_shape, 1); + const int stop_1 = StopForAxis(params_copy, input_shape, 1, start_1); + const int start_2 = StartForAxis(params_copy, input_shape, 2); + const int stop_2 = StopForAxis(params_copy, input_shape, 2, start_2); + const int start_3 = StartForAxis(params_copy, input_shape, 3); + const int stop_3 = StopForAxis(params_copy, input_shape, 3, start_3); + const int start_4 = StartForAxis(params_copy, input_shape, 4); + const int stop_4 = StopForAxis(params_copy, input_shape, 4, start_4); + + xa_nn_strided_slice_int16(output_data, input_data, static_cast(start_0), + static_cast(stop_0), static_cast(start_1), + static_cast(stop_1), static_cast(start_2), + static_cast(stop_2), static_cast(start_3), + static_cast(stop_3), static_cast(start_4), + static_cast(stop_4), params_copy.strides[0], + params_copy.strides[1], params_copy.strides[2], + params_copy.strides[3], params_copy.strides[4], + input_shape.Dims(1), input_shape.Dims(2), + input_shape.Dims(3), input_shape.Dims(4)); +} + +void StridedSlice_int32_hifi(const tflite::StridedSliceParams& op_params, const RuntimeShape& unextended_input_shape, - const int16_t* input_data, + const int32_t* input_data, const RuntimeShape& unextended_output_shape, - int16_t* output_data) { + int32_t* output_data) { using ::tflite::strided_slice::StartForAxis; using ::tflite::strided_slice::StopForAxis; @@ -66,7 +113,7 @@ void StridedSlice_int16_hifi4opt(const tflite::StridedSliceParams& op_params, const int start_4 = StartForAxis(params_copy, input_shape, 4); const int stop_4 = StopForAxis(params_copy, input_shape, 4, start_4); - xa_nn_strided_slice_int16(output_data, input_data, static_cast(start_0), + xa_nn_strided_slice_int32(output_data, input_data, static_cast(start_0), static_cast(stop_0), static_cast(start_1), static_cast(stop_1), static_cast(start_2), static_cast(stop_2), static_cast(start_3), @@ -77,7 +124,51 @@ void StridedSlice_int16_hifi4opt(const tflite::StridedSliceParams& op_params, input_shape.Dims(1), input_shape.Dims(2), input_shape.Dims(3), input_shape.Dims(4)); } -#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) + +void StridedSlice_int8_hifi(const tflite::StridedSliceParams& op_params, + const RuntimeShape& unextended_input_shape, + const int8_t* input_data, + const RuntimeShape& unextended_output_shape, + int8_t* output_data) { + using ::tflite::strided_slice::StartForAxis; + using ::tflite::strided_slice::StopForAxis; + + ruy::profiler::ScopeLabel label("StridedSlice"); + + // Note that the output_shape is not used herein. + tflite::StridedSliceParams params_copy = op_params; + + TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 5); + TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 5); + const RuntimeShape input_shape = + RuntimeShape::ExtendedShape(5, unextended_input_shape); + const RuntimeShape output_shape = + RuntimeShape::ExtendedShape(5, unextended_output_shape); + + // Reverse and pad to 5 dimensions because that is what the runtime code + // requires (ie. all shapes must be 5D and are given backwards). + ::tflite::strided_slice::StridedSlicePadIndices(¶ms_copy, 5); + + const int start_0 = StartForAxis(params_copy, input_shape, 0); + const int stop_0 = StopForAxis(params_copy, input_shape, 0, start_0); + const int start_1 = StartForAxis(params_copy, input_shape, 1); + const int stop_1 = StopForAxis(params_copy, input_shape, 1, start_1); + const int start_2 = StartForAxis(params_copy, input_shape, 2); + const int stop_2 = StopForAxis(params_copy, input_shape, 2, start_2); + const int start_3 = StartForAxis(params_copy, input_shape, 3); + const int stop_3 = StopForAxis(params_copy, input_shape, 3, start_3); + const int start_4 = StartForAxis(params_copy, input_shape, 4); + const int stop_4 = StopForAxis(params_copy, input_shape, 4, start_4); + + xa_nn_strided_slice_int8( + output_data, input_data, (int)start_0, (int)stop_0, (int)start_1, + (int)stop_1, (int)start_2, (int)stop_2, (int)start_3, (int)stop_3, + (int)start_4, (int)stop_4, params_copy.strides[0], params_copy.strides[1], + params_copy.strides[2], params_copy.strides[3], params_copy.strides[4], + input_shape.Dims(1), input_shape.Dims(2), input_shape.Dims(3), + input_shape.Dims(4)); +} +#endif TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TFLITE_DCHECK(node->user_data != nullptr); @@ -90,40 +181,63 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { tflite::micro::GetEvalOutput(context, node, kStridedSliceOutputTensor); switch (output->type) { case kTfLiteFloat32: +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) + StridedSlice_int32_hifi(op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +#else reference_ops::StridedSlice(op_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); +#endif break; case kTfLiteInt8: +#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) + StridedSlice_int8_hifi( + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) reference_ops::StridedSlice(op_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); +#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) break; case kTfLiteInt16: -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) - StridedSlice_int16_hifi4opt( +#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) + StridedSlice_int16_hifi( op_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); -#else +#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) reference_ops::StridedSlice( op_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); -#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) break; case kTfLiteInt32: +#if (defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) + StridedSlice_int32_hifi( + op_params, tflite::micro::GetTensorShape(input), + tflite::micro::GetTensorData(input), + tflite::micro::GetTensorShape(output), + tflite::micro::GetTensorData(output)); +#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) reference_ops::StridedSlice( op_params, tflite::micro::GetTensorShape(input), tflite::micro::GetTensorData(input), tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); +#endif //defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) break; case kTfLiteBool: reference_ops::StridedSlice(op_params, diff --git a/tensorflow/lite/micro/kernels/xtensa/sub.cc b/tensorflow/lite/micro/kernels/xtensa/sub.cc index b8308c93eaa..e8bb0df2780 100644 --- a/tensorflow/lite/micro/kernels/xtensa/sub.cc +++ b/tensorflow/lite/micro/kernels/xtensa/sub.cc @@ -36,7 +36,7 @@ void* SubInit(TfLiteContext* context, const char* buffer, size_t length) { return context->AllocatePersistentBuffer(context, sizeof(OpDataSub)); } -void EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, +TfLiteStatus EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, const OpDataSub* data, const TfLiteEvalTensor* input1, const TfLiteEvalTensor* input2, TfLiteEvalTensor* output) { float output_activation_min, output_activation_max; @@ -44,6 +44,48 @@ void EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, &output_activation_max); tflite::ArithmeticParams op_params; SetActivationParams(output_activation_min, output_activation_max, &op_params); + +#if defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) + const RuntimeShape extended_input1_shape = + RuntimeShape::ExtendedShape(5, tflite::micro::GetTensorShape(input1)); + const RuntimeShape extended_input2_shape = + RuntimeShape::ExtendedShape(5, tflite::micro::GetTensorShape(input2)); + const RuntimeShape extended_output_shape = + RuntimeShape::ExtendedShape(5, tflite::micro::GetTensorShape(output)); + const int* input1_dims = extended_input1_shape.DimsData(); + const int* input2_dims = extended_input2_shape.DimsData(); + const int* output_dims = extended_output_shape.DimsData(); + + int b; + int inp1_off = 0; + int inp2_off = 0; + int out_off; + out_off = output_dims[1] * output_dims[2] * output_dims[3] * output_dims[4]; + if (input1_dims[0] > 1) { + inp1_off = + input1_dims[1] * input1_dims[2] * input1_dims[3] * input1_dims[4]; + } + if (input2_dims[0] > 1) { + inp2_off = + input2_dims[1] * input2_dims[2] * input2_dims[3] * input2_dims[4]; + } + + for (b = 0; b < output_dims[0]; b++) { + int err = xa_nn_elm_sub_broadcast_4D_f32xf32_f32( + tflite::micro::GetTensorData(output) + b * out_off, + output_dims + 1, + tflite::micro::GetTensorData(input1) + b * inp1_off, + input1_dims + 1, + tflite::micro::GetTensorData(input2) + b * inp2_off, + input2_dims + 1); + TF_LITE_ENSURE(context, err == 0); + } + + float* output_data = tflite::micro::GetTensorData(output); + xa_nn_vec_activation_min_max_f32_f32( + output_data, output_data, op_params.float_activation_min, + op_params.float_activation_max, (output_dims[0] * out_off)); +#else // defined(INCLUDE_FLOAT_OPT) && (defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) if (data->requires_broadcast) { tflite::reference_ops::BroadcastSubSlow( op_params, tflite::micro::GetTensorShape(input1), @@ -61,6 +103,8 @@ void EvalSub(TfLiteContext* context, TfLiteNode* node, TfLiteSubParams* params, tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); } +#endif + return kTfLiteOk; } TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, @@ -83,15 +127,15 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, &op_params); // TODO(b/259724572): vision_p6 and hifi code path is getting very confusing. // Let's separate them into two different files. -#if !(defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) +#if !(defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ)) bool need_broadcast = reference_ops::ProcessBroadcastShapes( tflite::micro::GetTensorShape(input1), tflite::micro::GetTensorShape(input2), &op_params); -#endif // !(defined(HIFI3) || defined(HIFI4)) +#endif // !(defined(HIFI3) || defined(HIFI4) || defined(HIFI5)) switch (output->type) { case kTfLiteInt8: { -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) int err; const RuntimeShape extended_input1_shape = RuntimeShape::ExtendedShape(5, tflite::micro::GetTensorShape(input1)); @@ -105,6 +149,7 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, // TODO(b/259724572): Refactor the following block of code. int b; int inp1_off = 0; + int inp2_off = 0; int out_off; out_off = output_dims[1] * output_dims[2] * output_dims[3] * output_dims[4]; @@ -112,6 +157,10 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, inp1_off = input1_dims[1] * input1_dims[2] * input1_dims[3] * input1_dims[4]; } + if (input2_dims[0] > 1) { + inp2_off = + input2_dims[1] * input2_dims[2] * input2_dims[3] * input2_dims[4]; + } for (b = 0; b < output_dims[0]; b++) { err = xa_nn_elm_sub_broadcast_4D_asym8sxasym8s_asym8s( @@ -122,13 +171,13 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, tflite::micro::GetTensorData(input1) + b * inp1_off, input1_dims + 1, op_params.input1_offset, op_params.input1_shift, op_params.input1_multiplier, - tflite::micro::GetTensorData(input2), input2_dims + 1, - op_params.input2_offset, op_params.input2_shift, + tflite::micro::GetTensorData(input2) + b * inp2_off, + input2_dims + 1, op_params.input2_offset, op_params.input2_shift, op_params.input2_multiplier, op_params.left_shift); TF_LITE_ENSURE(context, err == 0); } -#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) if (need_broadcast) { tflite::reference_ops::BroadcastQuantSubSlow( op_params, tflite::micro::GetTensorShape(input1), @@ -150,7 +199,7 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, break; } case kTfLiteInt16: { -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) int err; const RuntimeShape extended_input1_shape = RuntimeShape::ExtendedShape(5, tflite::micro::GetTensorShape(input1)); @@ -163,6 +212,7 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, const int* output_dims = extended_output_shape.DimsData(); int b; int inp1_off = 0; + int inp2_off = 0; int out_off; out_off = output_dims[1] * output_dims[2] * output_dims[3] * output_dims[4]; @@ -170,6 +220,10 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, inp1_off = input1_dims[1] * input1_dims[2] * input1_dims[3] * input1_dims[4]; } + if (input2_dims[0] > 1) { + inp2_off = + input2_dims[1] * input2_dims[2] * input2_dims[3] * input2_dims[4]; + } for (b = 0; b < output_dims[0]; b++) { err = xa_nn_elm_sub_broadcast_4D_asym16sxasym16s_asym16s( @@ -180,13 +234,13 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, tflite::micro::GetTensorData(input1) + b * inp1_off, input1_dims + 1, op_params.input1_offset, op_params.input1_shift, op_params.input1_multiplier, - tflite::micro::GetTensorData(input2), input2_dims + 1, - op_params.input2_offset, op_params.input2_shift, + tflite::micro::GetTensorData(input2) + b * inp2_off, + input2_dims + 1, op_params.input2_offset, op_params.input2_shift, op_params.input2_multiplier, op_params.left_shift); TF_LITE_ENSURE(context, err == 0); } -#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#else // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) if (need_broadcast) { tflite::reference_ops::BroadcastQuantSubSlow( op_params, tflite::micro::GetTensorShape(input1), @@ -204,7 +258,7 @@ TfLiteStatus EvalSubQuantized(TfLiteContext* context, TfLiteNode* node, tflite::micro::GetTensorShape(output), tflite::micro::GetTensorData(output)); } -#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#endif // defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) break; } default: diff --git a/tensorflow/lite/micro/kernels/xtensa/xtensa.h b/tensorflow/lite/micro/kernels/xtensa/xtensa.h index 0e7e51b0cb6..0c42ee0e775 100644 --- a/tensorflow/lite/micro/kernels/xtensa/xtensa.h +++ b/tensorflow/lite/micro/kernels/xtensa/xtensa.h @@ -1,4 +1,4 @@ -/* Copyright 2025 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ limitations under the License. #if defined(HIFIMINI) #include -#include "tensorflow/lite/micro/kernels/xtensa/fixedpoint_utils_hifimini.h" +#include "tensorflow/lite/micro/kernels/xtensa/hifimini/fixedpoint_utils.h" #endif // defined(HIFMINI) -#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) +#if defined(HIFI3) || defined(HIFI4) || defined(HIFI5) || defined(HIFI_IQ) #include "include/nnlib/xa_nnlib_api.h" #include "include/nnlib/xa_nnlib_standards.h" @@ -35,4 +35,8 @@ limitations under the License. #include "vision_api.h" #endif // defined(VISION_P6) +#if (( XCHAL_HAVE_HIFI5_VFPU || XCHAL_HAVE_HIFI4_VFPU || XCHAL_HAVE_HIFI3Z_VFPU || XCHAL_HAVE_HIFI3_VFPU || XCHAL_HAVE_HIFI1_VFPU || XCHAL_HAVE_HIFIN_SP_VFPU ) && FLOAT_OPT_FLAG ) +#define INCLUDE_FLOAT_OPT +#endif + #endif // TENSORFLOW_LITE_MICRO_KERNELS_XTENSA_XTENSA_H_ diff --git a/tensorflow/lite/micro/tools/make/ext_libs/xtensa.inc b/tensorflow/lite/micro/tools/make/ext_libs/xtensa.inc index c3237cf8253..83cb487d46b 100644 --- a/tensorflow/lite/micro/tools/make/ext_libs/xtensa.inc +++ b/tensorflow/lite/micro/tools/make/ext_libs/xtensa.inc @@ -1,5 +1,43 @@ +# Explicitly add kernel sources specific to the Xtensa optimized +# implementations. +MICROLITE_CC_KERNEL_SRCS += \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/add_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_common_xtensa.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_common_xtensa.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_hifi.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_int16_reference.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_float32_reference.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_int8_int16_float32.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_int8_reference.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_hifi.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_int8_int16_float32.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_int8_reference.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_int16_reference.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_float32_reference.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_common_xtensa.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_int8.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_int16.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_float32.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pad_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pooling_int8.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pooling_int16.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pooling_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/reduce_hifi.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/reduce_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/reshape_vision.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/softmax_int8_int16.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/softmax_vision.cc + ifeq ($(TARGET_ARCH), hifimini) + # hifimini optimizations are implemented in the TFLM repository itself. + THIRD_PARTY_KERNEL_CC_SRCS += \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/hifimini/svdf.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/hifimini/fully_connected.cc + FFT_PATH := $(MAKEFILE_DIR)/downloads/hifi_fft INCLUDES += -I$(FFT_PATH)/ @@ -8,6 +46,36 @@ ifeq ($(TARGET_ARCH), hifimini) THIRD_PARTY_CC_HDRS += \ $(shell find $(FFT_PATH)/hifi2_fft -name "*.h") +else ifeq ($(TARGET_ARCH), hifi_iq) + + PLATFORM_FLAGS = \ + -Wno-shadow\ + + CCFLAGS += $(PLATFORM_FLAGS) + CXXFLAGS += $(PLATFORM_FLAGS) + + NNLIB_PATH := $(MAKEFILE_DIR)/downloads/xa_nnlib_hifi_iq + + THIRD_PARTY_KERNEL_CC_SRCS += \ + $(shell find $(NNLIB_PATH) -name "*.c") + + EXCLUDED_NNLIB_SRCS = \ + $(NNLIB_PATH)/algo/layers/cnn/src/xa_nn_cnn_api.c \ + $(NNLIB_PATH)/algo/layers/gru/src/xa_nn_gru_api.c \ + $(NNLIB_PATH)/algo/layers/lstm/src/xa_nn_lstm_api.c \ + + THIRD_PARTY_KERNEL_CC_SRCS := $(filter-out $(EXCLUDED_NNLIB_SRCS), $(THIRD_PARTY_KERNEL_CC_SRCS)) + + THIRD_PARTY_CC_HDRS += \ + $(shell find $(NNLIB_PATH) -name "*.h") \ + + INCLUDES += \ + -I$(NNLIB_PATH)/ \ + -I$(NNLIB_PATH)/algo/kernels/ \ + -I$(NNLIB_PATH)/include/nnlib/ \ + -I$(NNLIB_PATH)/include/ \ + -I$(NNLIB_PATH)/algo/common/include/ \ + else ifeq ($(TARGET_ARCH), hifi5) DOWNLOAD_RESULT := $(shell $(MAKEFILE_DIR)/ext_libs/xtensa_download.sh ${DOWNLOADS_DIR} hifi5 $(TENSORFLOW_ROOT)) ifneq ($(DOWNLOAD_RESULT), SUCCESS) @@ -23,8 +91,12 @@ else ifeq ($(TARGET_ARCH), hifi5) # not have separate cflags (or the concept of modular build targets) with the # Makefile, -Wno-shadow will be used for everything. + # TODO: Adding HIFI_SIMD_WIDTH here to avoid error in ndsp lib compilation, + # once ndsp headers are removed from NNLib, this won't be needed + PLATFORM_FLAGS = \ -DNNLIB_HIFI5 \ + -DHIFI_SIMD_WIDTH=16 \ -Wno-shadow CCFLAGS += $(PLATFORM_FLAGS) @@ -61,9 +133,11 @@ else ifeq ($(TARGET_ARCH), hifi5) -I$(NNLIB_PATH)/include/nnlib/ \ -I$(NNLIB_PATH)/include/ \ -I$(NNLIB_PATH)/algo/common/include/ \ + -I$(NNLIB_PATH)/algo/ndsp/hifi5/include/ \ -I$(NDSPLIB_PATH)/library/include/ \ -I$(NDSPLIB_PATH)/library/include_private/ -else ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi4)) + +else ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi4 hifi4_internal hifi3 hifi3z fusion_f1 hifi1)) # NNLib hifi4 also supports hifi3 DOWNLOAD_RESULT := $(shell $(MAKEFILE_DIR)/ext_libs/xtensa_download.sh ${DOWNLOADS_DIR} hifi4 $(TENSORFLOW_ROOT)) ifneq ($(DOWNLOAD_RESULT), SUCCESS) @@ -78,9 +152,13 @@ else ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi4)) # TODO(b/161489252): -Wno-shadow is only needed for xannlib. But since we do # not have separate cflags (or the concept of modular build targets) with the # Makefile, -Wno-shadow will be used for everything. + + # TODO: Adding HIFI_SIMD_WIDTH here to avoid error in ndsp lib compilation, + # once ndsp headers are removed from NNLib, this won't be needed PLATFORM_FLAGS = \ -DNNLIB_V2 \ + -DHIFI_SIMD_WIDTH=8 \ -Wno-shadow CCFLAGS += $(PLATFORM_FLAGS) @@ -104,7 +182,7 @@ else ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi4)) EXCLUDED_NNLIB_SRCS = \ $(NNLIB_PATH)/algo/layers/cnn/src/xa_nn_cnn_api.c \ $(NNLIB_PATH)/algo/layers/gru/src/xa_nn_gru_api.c \ - $(NNLIB_PATH)/algo/layers/lstm/src/xa_nn_lstm_api.c + $(NNLIB_PATH)/algo/layers/lstm/src/xa_nn_lstm_api.c ifeq ($(TARGET_ARCH), hifi3) EXCLUDED_NNLIB_SRCS += \ @@ -116,7 +194,6 @@ else ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi4)) ifeq ($(TARGET_ARCH), hifi4) EXCLUDED_NNLIB_SRCS += \ - $(NNLIB_PATH)/algo/kernels/activations/hifi4/xa_nn_activations_asym8_asym8.c \ $(NNLIB_PATH)/algo/kernels/norm/hifi4/xa_nn_norm3D_16.c endif @@ -132,6 +209,7 @@ else ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi4)) -I$(NNLIB_PATH)/include/nnlib/ \ -I$(NNLIB_PATH)/include/ \ -I$(NNLIB_PATH)/algo/common/include/ \ + -I$(NNLIB_PATH)/algo/ndsp/hifi4/include/ \ -I$(NDSPLIB_PATH)/library/include/ \ -I$(NDSPLIB_PATH)/library/include_private/ @@ -168,3 +246,21 @@ else ifeq ($(TARGET_ARCH), vision_p6) else $(error Unsupported TARGET_ARCH=$(TARGET_ARCH)) endif + +FFT_PATH := $(MAKEFILE_DIR)/downloads/hifi_fft + +INCLUDES += -I$(FFT_PATH)/ + +ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi3z hifi4 hifi4_internal hifi5 hifi1)) +THIRD_PARTY_KERNEL_CC_SRCS += \ + $(shell find $(FFT_PATH)/hifi3_fft -name "*.c") + +THIRD_PARTY_CC_HDRS += \ + $(shell find $(FFT_PATH)/hifi3_fft -name "*.h") +else ifeq ($(TARGET_ARCH), hifimini) +THIRD_PARTY_KERNEL_CC_SRCS += \ + $(shell find $(FFT_PATH)/hifi2_fft -name "*.c") + +THIRD_PARTY_CC_HDRS += \ + $(shell find $(FFT_PATH)/hifi2_fft -name "*.h") +endif diff --git a/tensorflow/lite/micro/tools/make/ext_libs/xtensa_download.sh b/tensorflow/lite/micro/tools/make/ext_libs/xtensa_download.sh index 391eaf12f4c..0cebb3b06c9 100755 --- a/tensorflow/lite/micro/tools/make/ext_libs/xtensa_download.sh +++ b/tensorflow/lite/micro/tools/make/ext_libs/xtensa_download.sh @@ -44,14 +44,17 @@ if [[ ${2} == "hifi3" ]]; then LIBRARY_URL="http://github.com/foss-xtensa/nnlib-hifi4/raw/master/archive/xa_nnlib_hifi4_09_30_2025.zip" LIBRARY_DIRNAME="xa_nnlib_hifi4" LIBRARY_MD5="a1cba57501a35d0bffdd39c536689366" + PATCH="../../ext_libs/xa_nnlib_${2}.patch" elif [[ ${2} == "hifi4" ]]; then LIBRARY_URL="http://github.com/foss-xtensa/nnlib-hifi4/raw/master/archive/xa_nnlib_hifi4_09_30_2025.zip" LIBRARY_DIRNAME="xa_nnlib_hifi4" LIBRARY_MD5="a1cba57501a35d0bffdd39c536689366" + PATCH="../../ext_libs/xa_nnlib_${2}.patch" elif [[ ${2} == "hifi5" ]]; then LIBRARY_URL="http://github.com/foss-xtensa/nnlib-hifi5/raw/master/archive/xa_nnlib_hifi5_09_30_2025.zip" LIBRARY_DIRNAME="xa_nnlib_hifi5" LIBRARY_MD5="f3ac445ae47143b1fddf26b85d763697" + PATCH="../../ext_libs/xa_nnlib_${2}.patch" elif [[ ${2} == "vision_p6" ]]; then LIBRARY_URL="https://github.com/foss-xtensa/tflmlib_vision/raw/main/archive/xi_tflmlib_vision_p6_22_06_29.zip" LIBRARY_DIRNAME="xi_tflmlib_vision_p6" @@ -85,7 +88,7 @@ else pushd "${LIBRARY_INSTALL_PATH}" > /dev/null chmod -R +w ./ - if [ "${PATCH}" ]; then + if [ -f "${PATCH}" ]; then create_git_repo ./ apply_patch_to_folder ./ ${PATCH} "TFLM patch" fi diff --git a/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc b/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc index 6001a90067b..bfb94732f63 100644 --- a/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc +++ b/tensorflow/lite/micro/tools/make/targets/xtensa_makefile.inc @@ -10,6 +10,7 @@ TARGET_ARCH := XTENSA_USE_LIBC := +XTENSA_INCLUDE_FLOAT_OPT := 1 # Allow additional flags on the command line for debugging. XTENSA_EXTRA_CFLAGS := @@ -30,11 +31,28 @@ ifeq ($(TARGET_ARCH), ) $(error TARGET_ARCH must be specified on the command line) endif +ifeq ($(XTENSA_INCLUDE_FLOAT_OPT), 1) + CXXFLAGS += -DFLOAT_OPT_FLAG=1 + CCFLAGS += -DFLOAT_OPT_FLAG=1 +endif + # Create a cflag based on the specified TARGET_ARCH. For example: # TARGET_ARCH=hifi4 --> -DHIFI4 -TARGET_ARCH_DEFINES := -D$(shell echo $(TARGET_ARCH) | tr [a-z] [A-Z]) +# For TARGET_ARCH=hifi3/3z or fusion_f1, cflag should be -DHIFI4. +ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi3 hifi3z fusion_f1 hifi1)) + TARGET_ARCH_DEFINES := -DHIFI4 +else + TARGET_ARCH_DEFINES := -D$(shell echo $(TARGET_ARCH) | tr [a-z] [A-Z]) +endif + +ifeq ($(BUILD_TYPE), debug) + THIRD_PARTY_KERNEL_OPTIMIZATION_LEVEL := -O0 +else + THIRD_PARTY_KERNEL_OPTIMIZATION_LEVEL := -O3 -LNO:simd +endif PLATFORM_FLAGS = \ + -stdlib=libc++ \ -DTF_LITE_MCU_DEBUG_LOG \ -DTF_LITE_USE_CTIME \ --xtensa-core=$(XTENSA_CORE) \ @@ -47,11 +65,6 @@ TARGET_TOOLCHAIN_PREFIX := xt- CXX_TOOL := clang++ CC_TOOL := clang -# Building with C++17 requires libc++ -ifneq ($(CC_VER11), true) - PLATFORM_FLAGS += -stdlib=libc++ -endif - # Unused exception related symbols make their way into a binary that links # against TFLM as described in https://github.com/tensorflow/tensorflow/issues/47575. # We have two options to avoid this. The first involves using -stdlib=libc++ and @@ -91,40 +104,15 @@ EXCLUDED_EXAMPLE_TESTS := \ MICRO_LITE_EXAMPLE_TESTS := $(filter-out $(EXCLUDED_EXAMPLE_TESTS), $(MICRO_LITE_EXAMPLE_TESTS)) MICRO_LITE_EXAMPLE_TESTS += $(shell find $(TENSORFLOW_ROOT)third_party/xtensa/examples/ -name Makefile.inc) -ifeq ($(OPTIMIZED_KERNEL_DIR), xtensa) - # Explicitly add kernel sources specific to the Xtensa optimized - # implementations. - # - # Do not include overlays for reference kernel files. - MICROLITE_CC_KERNEL_SRCS += \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/add_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_common_xtensa.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_hifi.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_int16_reference.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_int8_int16.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_int8_reference.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/conv_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_hifi.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/depthwise_conv_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_common_xtensa.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_hifimini.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_int8.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/fully_connected_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/lstm_eval_hifi.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pad_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pooling_int8.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/pooling_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/reduce_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/reshape_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/softmax_int8_int16.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/softmax_vision.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/svdf_hifimini.cc - - # Needed for LSTM support. - MICROLITE_CC_KERNEL_SRCS += \ - $(TENSORFLOW_ROOT)tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc \ - $(TENSORFLOW_ROOT)tensorflow/lite/kernels/kernel_util.cc +# Needed for LSTM support. +MICROLITE_CC_KERNEL_SRCS := $(MICROLITE_CC_KERNEL_SRCS) \ +$(TENSORFLOW_ROOT)tensorflow/lite/kernels/internal/reference/portable_tensor_utils.cc \ +$(TENSORFLOW_ROOT)tensorflow/lite/kernels/kernel_util.cc +ifeq ($(OPTIMIZED_KERNEL_DIR), xtensa) + MICROLITE_CC_KERNEL_SRCS := $(MICROLITE_CC_KERNEL_SRCS) \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/lstm_eval.cc \ + $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/unidirectional_sequence_lstm.cc # Additional kernel sources for DECODE operator support ifeq ($(TARGET_ARCH), $(filter $(TARGET_ARCH), hifi5)) MICROLITE_CC_KERNEL_SRCS += \ @@ -134,12 +122,12 @@ ifeq ($(OPTIMIZED_KERNEL_DIR), xtensa) endif endif -# override KERNEL_OPTIMIZATION_LEVEL to enable higher performance -# Xtensa intrinsics. + # override KERNEL_OPTIMIZATION_LEVEL to enable higher performance + # Xtensa intrinsics. $(KERNEL_OBJDIR)$(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/decompress.o: $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/decompress.cc @mkdir -p $(dir $@) $(CXX) $(CXXFLAGS) -O3 -LNO:simd $(INCLUDES) -c $< -o $@ - + $(KERNEL_OBJDIR)$(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/xtensa_decode_state_huffman.o: $(TENSORFLOW_ROOT)tensorflow/lite/micro/kernels/xtensa/xtensa_decode_state_huffman.cc @mkdir -p $(dir $@) $(CXX) $(CXXFLAGS) -O3 -LNO:simd $(INCLUDES) -c $< -o $@