it seems the detections do not come through even for BlazeFace if there are no detections

Author: swittk

mediapipe/examples/ios/facemeshioslib/FaceMeshIOSLib.h

@@ -9,10 +9,30 @@
 @property (nonatomic) float z;
 @end
 
+@interface FaceMeshIOSLibNormalizedRect : NSObject
+@property (nonatomic) float centerX;
+@property (nonatomic) float centerY;
+@property (nonatomic) float height;
+@property (nonatomic) float width;
+@property (nonatomic) float rotation;
+@end
+
 @protocol FaceMeshIOSLibDelegate <NSObject>
 @optional
-/** Array of faces, with faces represented by arrays of face landmarks */
+/** Array of faces, with faces represented by arrays of face landmarks 
+ * This does not always get called. If there are no faces detected by the Face Detector (Short Range) model,
+ * then this does not get called
+*/
 - (void)didReceiveFaces:(NSArray <NSArray<FaceMeshIOSLibFaceLandmarkPoint *>*>*)faces;
+/** Array of faces, with faces represented by arrays of face landmarks 
+ * This does not always get called. If there are no faces detected by the Face Detector (Short Range) model,
+ * then this does not get called
+*/
+- (void)didReceiveFaceBoxes:(NSArray <FaceMeshIOSLibNormalizedRect *>*)faces;
+/** Array of faces, with faces represented by arrays of face landmarks 
+ * This is the result called by the Face Detection (Short Range) model (a.k.a. BlazeFace)
+*/
+- (void)didReceiveFaceDetections:(NSArray <FaceMeshIOSLibNormalizedRect *>*)faces;
 @end
 
 @interface FaceMeshIOSLib : NSObject

mediapipe/examples/ios/facemeshioslib/FaceMeshIOSLib.mm

@@ -20,6 +20,7 @@
 static const char* kNumFacesInputSidePacket = "num_faces";
 static const char* kLandmarksOutputStream = "multi_face_landmarks";
 static const char* kFaceRectsOutputStream = "face_rects_from_landmarks";
+static const char* kFaceDetectionRectsOutputStream = "face_rects_from_detections";
 
 // Max number of faces to detect/process.
 static const int kNumFaces = 1;
@@ -68,8 +69,17 @@ + (MPPGraph*)loadGraphFromResource:(NSString*)resource {
   // Set graph configurations
   [newGraph setSidePacket:(mediapipe::MakePacket<int>(kNumFaces))
                               named:kNumFacesInputSidePacket];
+  // The landmarks output stream.
+  // !! This output does *not* give out any output when there are no faces detected by the face detector!
   [newGraph addFrameOutputStream:kLandmarksOutputStream
                           outputPacketType:MPPPacketTypeRaw];
+
+  [newGraph addFrameOutputStream:kFaceRectsOutputStream
+                          outputPacketType:MPPPacketTypeRaw];
+  // The face detections rect output stream
+  // This is kind of almost direct from blazeface I think, so it's likely out every frame.
+  // [newGraph addFrameOutputStream:kFaceDetectionRectsOutputStream
+  //                         outputPacketType:MPPPacketTypeRaw];
   return newGraph;
 }
 
@@ -140,10 +150,6 @@ - (void)mediapipeGraph:(MPPGraph*)graph
           fromStream:(const std::string&)streamName {
   if (streamName == kLandmarksOutputStream) {
     if (packet.IsEmpty()) {
-      NSLog(@"[TS:%lld] No face landmarks", packet.Timestamp().Value());
-      if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
-        [self.delegate didReceiveFaces:@[]];
-      }
       return;
     }
     const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
@@ -170,20 +176,55 @@ - (void)mediapipeGraph:(MPPGraph*)graph
       [self.delegate didReceiveFaces:faceLandmarks];
     }
   }
+  else if (streamName == kFaceDetectionRectsOutputStream) {
+    if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face detections", packet.Timestamp().Value());
+      if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
+        [self.delegate didReceiveFaceBoxes:@[]];
+      }
+      return;
+    }
+    const auto& face_rects_from_landmarks = packet.Get<std::vector<::mediapipe::NormalizedRect>>();
+    NSMutableArray <FaceMeshIOSLibNormalizedRect *>*outRects = [NSMutableArray new];
+    for (int face_index = 0; face_index < face_rects_from_landmarks.size(); ++face_index) {
+      const auto& face = face_rects_from_landmarks[face_index];
+      float centerX = face.x_center();
+      float centerY = face.y_center();
+      float height = face.height();
+      float width = face.width();
+      float rotation = face.rotation();
+      FaceMeshIOSLibNormalizedRect *rect = [FaceMeshIOSLibNormalizedRect new];
+      rect.centerX = centerX; rect.centerY = centerY; rect.height = height; rect.width = width; rect.rotation = rotation;
+      [outRects addObject:rect];
+    }
+    if([self.delegate respondsToSelector:@selector(didReceiveFaceDetections:)]) {
+      [self.delegate didReceiveFaceDetections:outRects];
+    }
+  }
   else if (streamName == kFaceRectsOutputStream) {
     if (packet.IsEmpty()) {
+      NSLog(@"[TS:%lld] No face rects", packet.Timestamp().Value());
+      if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
+        [self.delegate didReceiveFaceBoxes:@[]];
+      }
       return;
     }
     const auto& face_rects_from_landmarks = packet.Get<std::vector<::mediapipe::NormalizedRect>>();
+    NSMutableArray <FaceMeshIOSLibNormalizedRect *>*outRects = [NSMutableArray new];
     for (int face_index = 0; face_index < face_rects_from_landmarks.size(); ++face_index) {
       const auto& face = face_rects_from_landmarks[face_index];
       float centerX = face.x_center();
       float centerY = face.y_center();
       float height = face.height();
       float width = face.width();
       float rotation = face.rotation();
+      FaceMeshIOSLibNormalizedRect *rect = [FaceMeshIOSLibNormalizedRect new];
+      rect.centerX = centerX; rect.centerY = centerY; rect.height = height; rect.width = width; rect.rotation = rotation;
+      [outRects addObject:rect];
+    }
+    if([self.delegate respondsToSelector:@selector(didReceiveFaceBoxes:)]) {
+      [self.delegate didReceiveFaceBoxes:outRects];
     }
-    // TODO: MAYBE MAKE A DELEGATE METHOD HERE.
   }
 }
 
@@ -212,3 +253,6 @@ - (void)processVideoFrame:(CVPixelBufferRef)imageBuffer {
 
 @implementation FaceMeshIOSLibFaceLandmarkPoint
 @end
+
+@implementation FaceMeshIOSLibNormalizedRect
+@end

mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt

@@ -21,6 +21,12 @@ output_stream: "multi_face_landmarks"
 # For typings see "mediapipe/framework/formats/rect.pb.h"
 output_stream: "face_rects_from_landmarks"
 
+# The detections from the box model
+# see detection.proto
+# Regions of interest calculated based on face detections.
+# (std::vector<NormalizedRect>)
+output_stream: "face_rects_from_detections"
+
 # There's also face_rects_from_detections, but that likely isn't as accurate (it's from BlazeFace)
 
 
@@ -35,8 +41,12 @@ node {
   input_side_packet: "NUM_FACES:num_faces"
   output_stream: "LANDMARKS:multi_face_landmarks"
   output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
-#  output_stream: "DETECTIONS:face_detections"
-#  output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+  # face_detections is the stream that comes out from face_detection_short_range_common
+  # output_stream: "DETECTIONS:face_detections"
+  
+  # we need to output this too because otherwise the graph outputs nothing
+  # for frames without faces, and that makes it pretty hard to use
+  output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
 }
 
 # nope not rendering.