Landmarker Detection Confidence

Dockerfile

@@ -68,13 +68,22 @@ RUN ln -s /usr/bin/python3 /usr/bin/python
 
 # Install bazel
 ARG BAZEL_VERSION=6.5.0
+ARG TARGET_ARCH=x86_64
 RUN mkdir /bazel && \
-    wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
-azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
-    wget --no-check-certificate -O  /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
-    chmod +x /bazel/installer.sh && \
-    /bazel/installer.sh  && \
-    rm -f /bazel/installer.sh
+    if [ "$TARGET_ARCH" = "arm64" ]; then \
+        wget --no-check-certificate -O /bazel/bazel \
+            "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-${TARGET_ARCH}" && \
+        chmod +x /bazel/bazel && \
+        ln -s /bazel/bazel /usr/bin/bazel; \
+    else \
+        wget --no-check-certificate -O /bazel/installer.sh \
+            "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-${TARGET_ARCH}.sh" && \
+        chmod +x /bazel/installer.sh && \
+        /bazel/installer.sh && \
+        rm -f /bazel/installer.sh; \
+    fi && \
+    wget --no-check-certificate -O /bazel/LICENSE.txt \
+        "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE"
 
 COPY . /mediapipe/
 

mediapipe/calculators/util/BUILD

@@ -1851,3 +1851,14 @@ cc_test(
         "@com_google_absl//absl/status",
     ],
 )
+
+cc_library(
+    name = "confidence_landmark_merger_calculator",
+    srcs = ["confidence_landmark_merger_calculator.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/api2:node",
+    ],
+    alwayslink=1
+)

mediapipe/calculators/util/confidence_landmark_merger_calculator.cc

@@ -0,0 +1,56 @@
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "absl/status/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+template <typename LandmarkType>
+class ConfidenceMergerCalculator : public CalculatorBase {
+ public:
+  static constexpr Input<LandmarkType> kInLandmarks{"LANDMARKS"};
+  static constexpr Input<float> kInConfidence{"CONFIDENCE"};
+  static constexpr Output<LandmarkType> kOutLandmarks{"LANDMARKS"};
+
+  static absl::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Tag(kInLandmarks.Tag()).template Set<LandmarkType>();
+    cc->Inputs().Tag(kInConfidence.Tag()).template Set<float>();
+    cc->Outputs().Tag(kOutLandmarks.Tag()).template Set<LandmarkType>();
+    return absl::OkStatus();
+  }
+  absl::Status Open(CalculatorContext* cc) final {
+    cc->SetOffset(TimestampDiff(0));
+    return absl::OkStatus();
+  }
+  absl::Status Process(CalculatorContext* cc) override {
+    if (cc -> Inputs().Tag(kInLandmarks.Tag()).IsEmpty()|| cc -> Inputs().Tag(kInConfidence.Tag()).IsEmpty()) {
+    } else {
+      const auto& input_landmarks = 
+          cc->Inputs().Tag(kInLandmarks.Tag()).template Get<LandmarkType>();
+      const float confidence = 
+          cc->Inputs().Tag(kInConfidence.Tag()).template Get<float>();
+
+      auto output_landmarks = 
+          absl::make_unique<LandmarkType>(input_landmarks);
+
+      output_landmarks->set_detection_confidence(confidence);
+      output_landmarks->set_has_detection_confidence_set(true);
+
+      cc->Outputs()
+          .Tag(kOutLandmarks.Tag())
+          .Add(output_landmarks.release(), cc->InputTimestamp()); 
+    }
+    return absl::OkStatus();
+
+  }
+};
+
+using ConfidenceNormalizedLandmarkMergerCalculator = ConfidenceMergerCalculator<NormalizedLandmarkList>;
+using ConfidenceLandmarkMergerCalculator = ConfidenceMergerCalculator<LandmarkList>;
+
+REGISTER_CALCULATOR(ConfidenceNormalizedLandmarkMergerCalculator);
+REGISTER_CALCULATOR(ConfidenceLandmarkMergerCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/framework/deps/file_helpers.h

@@ -20,6 +20,8 @@
 #include "absl/strings/match.h"
 #include "mediapipe/framework/deps/mmapped_file.h"
 
+#include <sys/stat.h>
+
 namespace mediapipe {
 namespace file {
 absl::Status GetContents(absl::string_view file_name, std::string* output,
@@ -51,6 +53,24 @@ absl::Status IsDirectory(absl::string_view file_name);
 
 absl::Status RecursivelyCreateDir(absl::string_view path);
 
+
+// moved from android files since it seems like tests also require these options 
+// - not sure why
+// This exposes it to all platforms not just android files
+class Options {
+    public:
+     Options() = default;
+
+     void set_permissions(mode_t permissions) { permissions_ = permissions; }
+
+     mode_t permissions() const { return permissions_; }
+
+    private:
+     mode_t permissions_ = S_IRWXU | S_IRWXG | S_IRWXO;
+   };
+
+   inline Options Defaults() { return Options(); }
+
 }  // namespace file
 }  // namespace mediapipe
 

mediapipe/framework/formats/landmark.proto

@@ -45,6 +45,8 @@ message Landmark {
 // Group of Landmark protos.
 message LandmarkList {
   repeated Landmark landmark = 1;
+  optional float detection_confidence = 2;
+  optional bool has_detection_confidence_set = 3;
 }
 
 // Group of LandmarkList protos.
@@ -65,6 +67,8 @@ message NormalizedLandmark {
 // Group of NormalizedLandmark protos.
 message NormalizedLandmarkList {
   repeated NormalizedLandmark landmark = 1;
+  optional float detection_confidence = 2;
+  optional bool has_detection_confidence_set = 3;
 }
 
 // Group of NormalizedLandmarkList protos.

mediapipe/tasks/cc/components/containers/landmark.cc

@@ -49,6 +49,8 @@ Landmarks ConvertToLandmarks(const mediapipe::LandmarkList& proto) {
   for (const auto& landmark : proto.landmark()) {
     landmarks.landmarks.push_back(ConvertToLandmark(landmark));
   }
+  landmarks.detection_confidence = proto.detection_confidence();
+  landmarks.has_detection_confidence_set = proto.has_detection_confidence_set();
   return landmarks;
 }
 
@@ -59,6 +61,8 @@ NormalizedLandmarks ConvertToNormalizedLandmarks(
   for (const auto& landmark : proto.landmark()) {
     landmarks.landmarks.push_back(ConvertToNormalizedLandmark(landmark));
   }
+  landmarks.detection_confidence = proto.detection_confidence();
+  landmarks.has_detection_confidence_set = proto.has_detection_confidence_set();
   return landmarks;
 }
 

mediapipe/tasks/cc/components/containers/landmark.h

@@ -76,11 +76,15 @@ inline bool operator==(const NormalizedLandmark& lhs,
 // A list of Landmarks.
 struct Landmarks {
   std::vector<Landmark> landmarks;
+  std::optional<float> detection_confidence;
+  std::optional<bool> has_detection_confidence_set;
 };
 
 // A list of NormalizedLandmarks.
 struct NormalizedLandmarks {
   std::vector<NormalizedLandmark> landmarks;
+  std::optional<float> detection_confidence;
+  std::optional<bool> has_detection_confidence_set;
 };
 
 // Utility function to convert from Landmark proto to Landmark struct.

mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc

@@ -52,7 +52,7 @@ namespace vision {
 namespace face_detector {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::file::GetTextProto;
 using ::mediapipe::NormalizedRect;
 using ::mediapipe::api2::Input;

mediapipe/tasks/cc/vision/face_detector/face_detector_test.cc

@@ -36,7 +36,7 @@ namespace vision {
 namespace face_detector {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::mediapipe::file::JoinPath;
 using ::mediapipe::tasks::components::containers::NormalizedKeypoint;
 using ::mediapipe::tasks::vision::core::ImageProcessingOptions;

mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph_test.cc

@@ -42,7 +42,7 @@ namespace vision {
 namespace face_geometry {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::mediapipe::tasks::vision::face_geometry::proto::Environment;
 // using ::mediapipe::face_geometry::Environment;
 using ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry;

mediapipe/tasks/cc/vision/face_landmarker/BUILD

@@ -95,6 +95,7 @@ cc_library(
         "//mediapipe/calculators/util:rect_transformation_calculator_cc_proto",
         "//mediapipe/calculators/util:thresholding_calculator",
         "//mediapipe/calculators/util:thresholding_calculator_cc_proto",
+        "//mediapipe/calculators/util:confidence_landmark_merger_calculator",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/api2:builder",
@@ -223,3 +224,33 @@ cc_library(
     name = "face_landmarks_connections",
     hdrs = ["face_landmarks_connections.h"],
 )
+
+
+cc_test(
+    name = "face_landmarker_test",
+    srcs = ["face_landmarker_test.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "face_landmarker",
+        "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto",
+        "//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_cc_proto",
+        "//mediapipe/tasks/cc/vision/utils:image_utils",
+        "@org_tensorflow//tensorflow/lite:test_util",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:status_matchers",
+        "@com_google_googletest//:gtest_main"
+    ],
+    data = [
+        "//mediapipe/tasks/testdata/vision:test_protos",
+        "//mediapipe/tasks/testdata/vision:test_models",
+        "//mediapipe/tasks/testdata/vision:test_images"
+    ]
+)

mediapipe/tasks/cc/vision/face_landmarker/face_blendshapes_graph_test.cc

@@ -39,7 +39,7 @@ namespace vision {
 namespace face_landmarker {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::mediapipe::api2::Input;
 using ::mediapipe::api2::Output;
 using ::mediapipe::api2::builder::Graph;

mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph_test.cc

@@ -50,7 +50,7 @@ namespace vision {
 namespace face_landmarker {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::file::GetTextProto;
 using ::mediapipe::api2::Input;
 using ::mediapipe::api2::Output;

mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_test.cc

@@ -47,13 +47,17 @@ limitations under the License.
 #include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h"
 #include "mediapipe/tasks/cc/vision/utils/image_utils.h"
 
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_macros.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
 namespace mediapipe {
 namespace tasks {
 namespace vision {
 namespace face_landmarker {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::mediapipe::tasks::vision::core::ImageProcessingOptions;
 using ::testing::TestParamInfo;
 using ::testing::TestWithParam;
@@ -74,10 +78,23 @@ constexpr float kLandmarksDiffMargin = 0.03;
 constexpr float kBlendshapesDiffMargin = 0.12;
 constexpr float kFacialTransformationMatrixDiffMargin = 0.02;
 
+template <typename ProtoT>
+absl::Status GetTextProto(absl::string_view file_path,
+  ProtoT* result,
+  const mediapipe::file::Options& options) {
+  std::string proto_text;
+  MP_RETURN_IF_ERROR(mediapipe::file::GetContents(file_path, &proto_text));
+  if(!proto_ns::TextFormat::ParseFromString(proto_text, result)) {
+    return absl::InvalidArgumentError("Invalid Text Proto");
+  }
+  return absl::OkStatus();
+}
+
+
 template <typename ProtoT>
 ProtoT GetExpectedProto(absl::string_view filename) {
   ProtoT expected_proto;
-  MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
+  MP_EXPECT_OK(GetTextProto<ProtoT>(file::JoinPath("./", kTestDataDirectory, filename),
                             &expected_proto, Defaults()));
   return expected_proto;
 }
@@ -126,6 +143,9 @@ void ExpectLandmarksCorrect(
       EXPECT_THAT(actual_landmarks[i].landmarks[j],
                   LandmarkIs(expected_landmarks[i].landmarks[j]));
     }
+    if (actual_landmarks[i].has_detection_confidence_set) {
+      ASSERT_GE(actual_landmarks[i].detection_confidence, 0.5);
+    }
   }
 }
 

mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc

@@ -367,8 +367,13 @@ class SingleFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
         AllowIf(face_rect_transformation.Out("").Cast<NormalizedRect>(),
                 presence, graph);
 
+    auto& landmark_merger = graph.AddNode("ConfidenceNormalizedLandmarkMergerCalculator");
+    projected_landmarks >> landmark_merger.In("LANDMARKS");
+    presence_score >> landmark_merger.In("CONFIDENCE");
+    auto merged_landmarks = landmark_merger[Output<NormalizedLandmarkList>("LANDMARKS")];
+
     return {{
-        /* landmarks= */ projected_landmarks,
+        /* landmarks= */ merged_landmarks,
         /* rect_next_frame= */ face_rect_next_frame,
         /* presence= */ presence,
         /* presence_score= */ presence_score,

mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph_test.cc

@@ -46,7 +46,7 @@ namespace vision {
 namespace face_landmarker {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::file::GetTextProto;
 using ::mediapipe::NormalizedRect;
 using ::mediapipe::api2::Input;

mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph_test.cc

@@ -52,7 +52,7 @@ namespace vision {
 namespace hand_detector {
 namespace {
 
-using ::file::Defaults;
+using mediapipe::file::Defaults;
 using ::file::GetTextProto;
 using ::mediapipe::NormalizedRect;
 using ::mediapipe::api2::Input;