runtime.py

# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Python package for TFLM Python Interpreter"""

import enum
import os
from tflite_micro.python.tflite_micro import _runtime
from tflite_micro.tensorflow.lite.python import schema_py_generated as schema_fb


def convert_bytearray_to_object(model_bytearray):
  """Converts a tflite model from a bytearray to an object for parsing."""
  model_object = schema_fb.Model.GetRootAsModel(model_bytearray, 0)
  return schema_fb.ModelT.InitFromObj(model_object)


def get_builtin_code_from_operator_code(opcode):
  """Return the builtin code of the given operator code."""
  if hasattr(opcode, 'BuiltinCode') and callable(opcode.BuiltinCode):
    return max(opcode.BuiltinCode(), opcode.DeprecatedBuiltinCode())
  return max(opcode.builtinCode, opcode.deprecatedBuiltinCode)


def count_resource_variables(model):
  """Calculates the number of unique resource variables in a model."""
  if not isinstance(model, schema_fb.ModelT):
    model = convert_bytearray_to_object(model)
  unique_shared_names = set()
  for subgraph in model.subgraphs:
    if subgraph.operators is None:
      continue
    for op in subgraph.operators:
      builtin_code = get_builtin_code_from_operator_code(
          model.operatorCodes[op.opcodeIndex])
      if builtin_code == schema_fb.BuiltinOperator.VAR_HANDLE:
        unique_shared_names.add(op.builtinOptions.sharedName)
  return len(unique_shared_names)


class InterpreterConfig(enum.Enum):
  """There are two mutually exclusive types of way you could use the TFLM python

  interpreter, this enum is made so that users can clearly choose between the
  two
  different usage method for the interpreter.

  The first default way is kRecordingAllocation where all memory usage by the
  interpreter is recorded on inference. When using this config the GetTensor()
  api is disabled by the interpreter since this interpreter configuration
  doesn’t
  guarantee that the valid data for all tensors is available post inference.

  The second way is kPreserveAllTensors where the GetTensor() api is disabled by
  the interpreter since this interpreter configuration doesn’t guarantee that
  the
  valid data for all tensors is available post inference. But the memory usage
  by
  the interpreter won’t be recorded on inference.

  Usage:

  default_interpreter = Interpreter(…
        intrepreter_config=InterpreterConfig.kAllocationRecording)

  preserve_interpreter = Interpreter(…
        intrepreter_config=InterpreterConfig.kPreserveAllTensors)
  """

  kAllocationRecording = 0
  kPreserveAllTensors = 1


#TODO(b/297118768): Once Korko Docker container for ubuntu x86 has imutabledict
# added to it, this should be turned into an immutabledict.
_ENUM_TRANSLATOR = {
    InterpreterConfig.kAllocationRecording:
    (_runtime.PythonInterpreterConfig.kAllocationRecording),
    InterpreterConfig.kPreserveAllTensors:
    (_runtime.PythonInterpreterConfig.kPreserveAllTensors),
}


class Interpreter(object):

  def __init__(
      self,
      model_data,
      custom_op_registerers,
      arena_size,
      intrepreter_config=InterpreterConfig.kAllocationRecording,
      alt_decompression_memory_size=0,
  ):
    if model_data is None:
      raise ValueError("Model must not be None")

    if not isinstance(custom_op_registerers, list) or not all(
        isinstance(s, str) for s in custom_op_registerers):
      raise ValueError("Custom ops registerers must be a list of strings")

    # This is a heuristic to ensure that the arena is sufficiently sized.
    if arena_size is None:
      arena_size = len(model_data) * 10
    # Some models make use of resource variables ops, get the count here
    num_resource_variables = count_resource_variables(model_data)
    print("Number of resource variables the model uses = ",
          num_resource_variables)

    self._interpreter = _runtime.InterpreterWrapper(
        model_data,
        custom_op_registerers,
        arena_size,
        num_resource_variables,
        _ENUM_TRANSLATOR[intrepreter_config],
        alt_decompression_memory_size,
    )

  @classmethod
  def from_file(
      self,
      model_path,
      custom_op_registerers=[],
      arena_size=None,
      intrepreter_config=InterpreterConfig.kAllocationRecording,
      alt_decompression_memory_size=0,
  ):
    """Instantiates a TFLM interpreter from a model .tflite filepath.

    Args:
      model_path: Filepath to the .tflite model
      custom_op_registerers: List of strings, each of which is the name of a
        custom OP registerer
      arena_size: Tensor arena size in bytes. If unused, tensor arena size will
        default to 10 times the model size.
      alt_decompression_memory_size: Size in bytes of alternate decompression
        memory. If non-zero, DECODE operators will use this memory instead of
        the main arena for decompressed tensor outputs.

    Returns:
      An Interpreter instance
    """
    if model_path is None or not os.path.isfile(model_path):
      raise ValueError("Invalid model file path")

    with open(model_path, "rb") as f:
      model_data = f.read()

    return Interpreter(
        model_data,
        custom_op_registerers,
        arena_size,
        intrepreter_config,
        alt_decompression_memory_size,
    )

  @classmethod
  def from_bytes(
      self,
      model_data,
      custom_op_registerers=[],
      arena_size=None,
      intrepreter_config=InterpreterConfig.kAllocationRecording,
      alt_decompression_memory_size=0,
  ):
    """Instantiates a TFLM interpreter from a model in byte array.

    Args:
      model_data: Model in byte array format
      custom_op_registerers: List of strings, each of which is the name of a
        custom OP registerer
      arena_size: Tensor arena size in bytes. If unused, tensor arena size will
        default to 10 times the model size.
      alt_decompression_memory_size: Size in bytes of alternate decompression
        memory. If non-zero, DECODE operators will use this memory instead of
        the main arena for decompressed tensor outputs.

    Returns:
      An Interpreter instance
    """

    return Interpreter(
        model_data,
        custom_op_registerers,
        arena_size,
        intrepreter_config,
        alt_decompression_memory_size,
    )

  def print_allocations(self):
    """Invoke the RecordingMicroAllocator to print the arena usage.

    This should be called after `invoke()`.

    Returns:
      This method does not return anything, but It dumps the arena
      usage to stderr.
    """
    self._interpreter.PrintAllocations()

  def invoke(self):
    """Invoke the TFLM interpreter to run an inference.

    This should be called after `set_input()`.

    Returns:
      Status code of the C++ invoke function. A RuntimeError will be raised as
      well upon any error.
    """
    return self._interpreter.Invoke()

  def reset(self):
    """Reset the model state to be what you would expect when the interpreter is first

    created. i.e. after Init and Prepare is called for the very first time.

    This should be called after invoke stateful model like LSTM.

    Returns:
      Status code of the C++ invoke function. A RuntimeError will be raised as
      well upon any error.
    """
    return self._interpreter.Reset()

  def set_input(self, input_data, index):
    """Set input data into input tensor.

    This should be called before `invoke()`.

    Args:
      input_data: Input data in numpy array format. The numpy array format is
        chosen to be consistent with TFLite interpreter.
      index: An integer between 0 and the number of input tensors (exclusive)
        consistent with the order defined in the list of inputs in the .tflite
        model
    """
    if input_data is None:
      raise ValueError("Input data must not be None")
    if index is None or index < 0:
      raise ValueError("Index must be a non-negative integer")

    self._interpreter.SetInputTensor(input_data, index)

  def get_output(self, index):
    """Get data from output tensor.

    The output data correspond to the most recent `invoke()`.

    Args:
      index: An integer between 0 and the number of output tensors (exclusive)
        consistent with the order defined in the list of outputs in the .tflite
        model

    Returns:
      Output data in numpy array format. The numpy array format is chosen to
      be consistent with TFLite interpreter.
    """
    if index is None or index < 0:
      raise ValueError("Index must be a non-negative integer")

    return self._interpreter.GetOutputTensor(index)

  def GetTensor(self, tensor_index, subgraph_index):
    return self._interpreter.GetTensor(tensor_index, subgraph_index)

  def get_input_details(self, index):
    """Get input tensor information

    Args:
        index (int): An integer between 0 and the number of output tensors
          (exclusive) consistent with the order defined in the list of outputs
          in the .tflite model

    Returns:
        A dictionary from input index to tensor details where each item is a
        dictionary with details about an input tensor. Each dictionary contains
        the following fields that describe the tensor:
        + `shape`: The shape of the tensor.
        + `dtype`: The numpy data type (such as `np.int32` or `np.uint8`).
        + `quantization_parameters`: A dictionary of parameters used to quantize
          the tensor:
          ~ `scales`: List of scales (one if per-tensor quantization).
          ~ `zero_points`: List of zero_points (one if per-tensor quantization).
          ~ `quantized_dimension`: Specifies the dimension of per-axis
          quantization, in the case of multiple scales/zero_points.

    """
    if index is None or index < 0:
      raise ValueError("Index must be a non-negative integer")

    return self._interpreter.GetInputTensorDetails(index)

  def get_output_details(self, index):
    """Get output tensor information

    Args:
        index (int): An integer between 0 and the number of output tensors
          (exclusive) consistent with the order defined in the list of outputs
          in the .tflite model

    Returns:
        A dictionary from input index to tensor details where each item is a
        dictionary with details about an input tensor. Each dictionary contains
        the following fields that describe the tensor:
        + `shape`: The shape of the tensor.
        + `dtype`: The numpy data type (such as `np.int32` or `np.uint8`).
        + `quantization_parameters`: A dictionary of parameters used to quantize
          the tensor:
          ~ `scales`: List of scales (one if per-tensor quantization).
          ~ `zero_points`: List of zero_points (one if per-tensor quantization).
          ~ `quantized_dimension`: Specifies the dimension of per-axis
          quantization, in the case of multiple scales/zero_points.

    """
    if index is None or index < 0:
      raise ValueError("Index must be a non-negative integer")

    return self._interpreter.GetOutputTensorDetails(index)