Add a graph rewrite to fallback to fp32 when fp16 isn't supported.

PiperOrigin-RevId: 917373652

Author: qukhan

BUILD.bazel

@@ -904,6 +904,7 @@ xnnpack_cc_library(
         ":xnnpack_h",
         "//src/configs:config_hdrs",
         "//src/configs:hardware_config",
+        "//src/subgraph/rewrites:fp16_to_fp32",
         "@pthreadpool",
     ],
 )

BUILD.gn

@@ -381,6 +381,10 @@ xnnpack_source_set("subgraph") {
 
   # From build_srcs.bzl
   sources = SUBGRAPH_SRCS
+  sources += [
+    "src/subgraph/rewrites/fp16_to_fp32.cc",
+    "src/subgraph/rewrites/fp16_to_fp32.h",
+  ]
 }
 
 xnnpack_source_set("table") {

CMakeLists.txt

@@ -565,6 +565,7 @@ SET(SUBGRAPH_SRCS
   src/subgraph/max-pooling-2d.c
   src/subgraph/pack-lh.c
   src/subgraph/reshape-helpers.c
+  src/subgraph/rewrites/fp16_to_fp32.cc
   src/subgraph/rope.c
   src/subgraph/softmax.c
   src/subgraph/space-to-depth-2d.c
@@ -996,7 +997,7 @@ IF(XNNPACK_BUILD_LIBRARY)
   TARGET_LINK_LIBRARIES(xnnpack-operator-run PRIVATE xnnpack-base xnnpack-logging)
   TARGET_LINK_LIBRARIES(xnnpack-operator-utils PRIVATE xnnpack-base xnnpack-logging)
   TARGET_LINK_LIBRARIES(xnnpack-reference-ukernels PRIVATE xnnpack-base xnnpack-datatype)
-  TARGET_LINK_LIBRARIES(xnnpack-subgraph PRIVATE xnnpack-base xnnpack-allocator xnnpack-logging xnnpack-memory xnnpack-mutex xnnpack-operators xnnpack-operator-run xnnpack-datatype)
+  TARGET_LINK_LIBRARIES(xnnpack-subgraph PRIVATE xnnpack-base xnnpack-allocator xnnpack-cache xnnpack-logging xnnpack-memory xnnpack-mutex xnnpack-operators xnnpack-operator-run xnnpack-datatype)
   TARGET_LINK_LIBRARIES(XNNPACK PRIVATE xnnpack-base xnnpack-allocator xnnpack-cache
       xnnpack-hardware-config xnnpack-indirection xnnpack-memory xnnpack-microkernel-utils xnnpack-microparams-init
       xnnpack-mutex xnnpack-normalization xnnpack-operators xnnpack-operator-run

src/runtime.c

@@ -22,6 +22,7 @@
 
 #include "include/experimental.h"
 #include "include/xnnpack.h"
+#include "src/subgraph/rewrites/fp16_to_fp32.h"
 #include "src/xnnpack/allocation-type.h"
 #include "src/xnnpack/allocator.h"
 #include "src/xnnpack/cache.h"
@@ -598,6 +599,9 @@ enum xnn_status xnn_create_runtime_v4(
     goto error;
   }
 
+  XNN_IF_ERROR_GOTO(error, xnn_subgraph_alias_fp16_fp32_fallback_data(
+                               subgraph, weights_cache));
+
   status = xnn_status_out_of_memory;
 
   runtime = xnn_allocate_zero_memory(sizeof(struct xnn_runtime));
@@ -1194,7 +1198,7 @@ enum xnn_status xnn_delete_runtime(
       xnn_release_memory(runtime->opdata);
 
       if (runtime->values != NULL) {
-        // Release the buffers created during FP16 rewrite.
+        // Release buffers created during rewrites.
         for (size_t i = 0; i < runtime->num_values; i++) {
           struct xnn_runtime_value* value = &runtime->values[i];
           if (value->allocation_type == xnn_allocation_type_dynamic ||

src/subgraph.c

@@ -15,6 +15,7 @@
 
 #include "include/experimental.h"
 #include "include/xnnpack.h"
+#include "src/subgraph/rewrites/fp16_to_fp32.h"
 #include "src/xnnpack/allocation-type.h"
 #include "src/xnnpack/allocator.h"
 #include "src/xnnpack/common.h"
@@ -4590,6 +4591,10 @@ enum xnn_status xnn_subgraph_optimize(xnn_subgraph_t subgraph,
   XNN_RETURN_IF_ERROR(
       xnn_subgraph_optimize_packed_lhs(subgraph, optimization_flags));
 
+  if (!xnn_is_f16_supported_natively(hardware_config) && !force_fp16) {
+    XNN_RETURN_IF_ERROR(xnn_subgraph_fallback_from_fp16_to_fp32(subgraph, optimization_flags));
+  }
+
   return xnn_status_success;
 }
 

src/subgraph/rewrites/BUILD

@@ -0,0 +1,23 @@
+load(
+    "//:build_defs.bzl",
+    "xnnpack_cxx_library",
+)
+
+package(default_visibility = ["//:__subpackages__"])
+
+xnnpack_cxx_library(
+    name = "fp16_to_fp32",
+    srcs = ["fp16_to_fp32.cc"],
+    hdrs = ["fp16_to_fp32.h"],
+    deps = [
+        "//:allocation_type",
+        "//:allocator",
+        "//:cache",
+        "//:fp16",
+        "//:internal",
+        "//:logging",
+        "//:node_type",
+        "//:subgraph_h",
+        "//:xnnpack_h",
+    ],
+)