FINALLY GOT IT WORKING WITH FRAMES WITHOUT FACES

Author: swittk

mediapipe/examples/ios/facemeshioslib/BUILD

@@ -123,6 +123,7 @@ objc_library(
     deps = [
         "//mediapipe/objc:mediapipe_framework_ios",
         "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/calculators/core:packet_presence_calculator",
 #        "//mediapipe/objc:mediapipe_layer_renderer", # no need for layer renderer since I don't render
     ] + select({
 #        "//mediapipe:ios_i386": [],

mediapipe/examples/ios/facemeshioslib/FaceMeshIOSLib.h

@@ -19,20 +19,14 @@
 
 @protocol FaceMeshIOSLibDelegate <NSObject>
 @optional
-/** Array of faces, with faces represented by arrays of face landmarks 
- * This does not always get called. If there are no faces detected by the Face Detector (Short Range) model,
- * then this does not get called
+/** 
+ * Array of faces, with faces represented by arrays of face landmarks 
 */
 - (void)didReceiveFaces:(NSArray <NSArray<FaceMeshIOSLibFaceLandmarkPoint *>*>*)faces;
-/** Array of faces, with faces represented by arrays of face landmarks 
- * This does not always get called. If there are no faces detected by the Face Detector (Short Range) model,
- * then this does not get called
+/** 
+ * Array of faces, with faces represented by arrays of face landmarks 
 */
 - (void)didReceiveFaceBoxes:(NSArray <FaceMeshIOSLibNormalizedRect *>*)faces;
-/** Array of faces, with faces represented by arrays of face landmarks 
- * This is the result called by the Face Detection (Short Range) model (a.k.a. BlazeFace)
-*/
-- (void)didReceiveFaceDetections:(NSArray <FaceMeshIOSLibNormalizedRect *>*)faces;
 @end
 
 @interface FaceMeshIOSLib : NSObject

mediapipe/examples/ios/facemeshioslib/FaceMeshIOSLib.mm

@@ -7,6 +7,7 @@
 // NormalizedLandmarkList won't be defined unless we import this header (following FaceMeshGpuViewController's imports)
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
 
 //#import "mediapipe/objc/MPPLayerRenderer.h"
 
@@ -20,7 +21,9 @@
 static const char* kNumFacesInputSidePacket = "num_faces";
 static const char* kLandmarksOutputStream = "multi_face_landmarks";
 static const char* kFaceRectsOutputStream = "face_rects_from_landmarks";
-static const char* kFaceDetectionRectsOutputStream = "face_rects_from_detections";
+static const char* kLandmarkPresenceOutputStream = "landmark_presence";
+// static const char* kFaceDetectionRectsOutputStream = "face_rects_from_detections";
+// static const char* kFaceDetectionsRawDetectionsOutputStream = "face_detections";
 
 // Max number of faces to detect/process.
 static const int kNumFaces = 1;
@@ -78,8 +81,14 @@ + (MPPGraph*)loadGraphFromResource:(NSString*)resource {
                           outputPacketType:MPPPacketTypeRaw];
   // The face detections rect output stream
   // This is kind of almost direct from blazeface I think, so it's likely out every frame.
+  // Turns out this doesn't come out at all... what the heck
   // [newGraph addFrameOutputStream:kFaceDetectionRectsOutputStream
   //                         outputPacketType:MPPPacketTypeRaw];
+
+  // The Presence Detection stream
+  // This is with much much many many thanks to @homuler here: https://github.com/google/mediapipe/issues/850#issuecomment-683268033
+  [newGraph addFrameOutputStream:kLandmarkPresenceOutputStream
+                           outputPacketType:MPPPacketTypeRaw];
   return newGraph;
 }
 
@@ -149,12 +158,12 @@ - (void)mediapipeGraph:(MPPGraph*)graph
      didOutputPacket:(const ::mediapipe::Packet&)packet
           fromStream:(const std::string&)streamName {
   if (streamName == kLandmarksOutputStream) {
-    if (packet.IsEmpty()) {
+    if (packet.IsEmpty()) { // This condition never gets called because FaceLandmarkFrontGpu does not process when there are no detections
       return;
     }
     const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
-    NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
-          multi_face_landmarks.size());
+    // NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
+          // multi_face_landmarks.size());
     NSMutableArray <NSArray <FaceMeshIOSLibFaceLandmarkPoint *>*>*faceLandmarks = [NSMutableArray new];
 
     for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
@@ -176,10 +185,11 @@ - (void)mediapipeGraph:(MPPGraph*)graph
       [self.delegate didReceiveFaces:faceLandmarks];
     }
   }
-  else if (streamName == kFaceDetectionRectsOutputStream) {
-    if (packet.IsEmpty()) {
-      NSLog(@"[TS:%lld] No face detections", packet.Timestamp().Value());
-      if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
+
+  else if (streamName == kFaceRectsOutputStream) {
+    if (packet.IsEmpty()) { // This condition never gets called because FaceLandmarkFrontGpu does not process when there are no detections
+      // NSLog(@"[TS:%lld] No face rects", packet.Timestamp().Value());
+      if([self.delegate respondsToSelector:@selector(didReceiveFaceBoxes:)]) {
         [self.delegate didReceiveFaceBoxes:@[]];
       }
       return;
@@ -197,35 +207,36 @@ - (void)mediapipeGraph:(MPPGraph*)graph
       rect.centerX = centerX; rect.centerY = centerY; rect.height = height; rect.width = width; rect.rotation = rotation;
       [outRects addObject:rect];
     }
-    if([self.delegate respondsToSelector:@selector(didReceiveFaceDetections:)]) {
-      [self.delegate didReceiveFaceDetections:outRects];
+    if([self.delegate respondsToSelector:@selector(didReceiveFaceBoxes:)]) {
+      [self.delegate didReceiveFaceBoxes:outRects];
     }
   }
-  else if (streamName == kFaceRectsOutputStream) {
+  else if (streamName == kLandmarkPresenceOutputStream) {
+    bool is_landmark_present = true;
     if (packet.IsEmpty()) {
-      NSLog(@"[TS:%lld] No face rects", packet.Timestamp().Value());
-      if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
-        [self.delegate didReceiveFaceBoxes:@[]];
-      }
-      return;
+      is_landmark_present = false;
     }
-    const auto& face_rects_from_landmarks = packet.Get<std::vector<::mediapipe::NormalizedRect>>();
-    NSMutableArray <FaceMeshIOSLibNormalizedRect *>*outRects = [NSMutableArray new];
-    for (int face_index = 0; face_index < face_rects_from_landmarks.size(); ++face_index) {
-      const auto& face = face_rects_from_landmarks[face_index];
-      float centerX = face.x_center();
-      float centerY = face.y_center();
-      float height = face.height();
-      float width = face.width();
-      float rotation = face.rotation();
-      FaceMeshIOSLibNormalizedRect *rect = [FaceMeshIOSLibNormalizedRect new];
-      rect.centerX = centerX; rect.centerY = centerY; rect.height = height; rect.width = width; rect.rotation = rotation;
-      [outRects addObject:rect];
+    else {
+      is_landmark_present = packet.Get<bool>();
     }
-    if([self.delegate respondsToSelector:@selector(didReceiveFaceBoxes:)]) {
-      [self.delegate didReceiveFaceBoxes:outRects];
+    if (is_landmark_present) {
+      // NSLog(@"Landmarks present");
+      // Landmarks are present; no need to do anything (the rest of the callbacks will get called on their own)
+    }
+    else {
+      // NSLog(@"Landmarks not present");
+      // No landmarks are present, we call our delegate with empty faces to make our protocol consistent with number of frames
+      if([self.delegate respondsToSelector:@selector(didReceiveFaceBoxes:)]) {
+        [self.delegate didReceiveFaceBoxes:@[]];
+      }
+      if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
+        [self.delegate didReceiveFaces:@[]];
+      }
     }
   }
+  else {
+    NSLog(@"Unknown %@ packet with stream name %s", packet.IsEmpty() ? @"EMPTY" : @"NON-EMPTY",streamName.c_str());
+  }
 }
 
 
@@ -235,14 +246,14 @@ - (void)processVideoFrame:(CVPixelBufferRef)imageBuffer {
   const auto ts =
       mediapipe::Timestamp(self.timestamp++ * mediapipe::Timestamp::kTimestampUnitsPerSecond);
   NSError* err = nil;
-  NSLog(@"sending imageBuffer @%@ to %s", @(ts.DebugString().c_str()), kInputStream);
+  // NSLog(@"sending imageBuffer @%@ to %s", @(ts.DebugString().c_str()), kInputStream);
   auto sent = [self.mediapipeGraph sendPixelBuffer:imageBuffer
                                         intoStream:kInputStream
                                         packetType:MPPPacketTypePixelBuffer
                                          timestamp:ts
                                     allowOverwrite:NO
                                              error:&err];
-  NSLog(@"imageBuffer %s", sent ? "sent!" : "not sent.");
+  // NSLog(@"imageBuffer %s", sent ? "sent!" : "not sent.");
   if (err) {
     NSLog(@"sendPixelBuffer error: %@", err);
   }

mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt

@@ -25,10 +25,15 @@ output_stream: "face_rects_from_landmarks"
 # see detection.proto
 # Regions of interest calculated based on face detections.
 # (std::vector<NormalizedRect>)
-output_stream: "face_rects_from_detections"
+# output_stream: "face_rects_from_detections"
 
-# There's also face_rects_from_detections, but that likely isn't as accurate (it's from BlazeFace)
+# Extra outputs (for debugging, for instance).
+# Detected faces. (std::vector<Detection>)
+# (std::vector<Detections>)
+# output_stream: "face_detections"
 
+# Landmark presence (needed because whole graph won't emit anything if no faces are detected)
+output_stream: "landmark_presence"
 
 # screw the throttling, we do that ourselves.
 # *throttling node code was deleted from here*
@@ -44,9 +49,17 @@ node {
   # face_detections is the stream that comes out from face_detection_short_range_common
   # output_stream: "DETECTIONS:face_detections"
   
-  # we need to output this too because otherwise the graph outputs nothing
-  # for frames without faces, and that makes it pretty hard to use
-  output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+  # output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+}
+
+# See this thread here https://github.com/google/mediapipe/issues/850#issuecomment-683268033
+# "if there are no packets in the corresponding output stream, it is designed to wait until the packet comes in"
+# That means that we'd get absolutely nothing to work with and won't know if our frame had anythin!
+# So we add PacketPresenceCalculator
+node {
+  calculator: "PacketPresenceCalculator"
+  input_stream: "PACKET:multi_face_landmarks"
+  output_stream: "PRESENCE:landmark_presence"
 }
 
 # nope not rendering.