Port LandmarksSmoothingCalculator to api2

PiperOrigin-RevId: 536474279

mediapipe/calculators/util/BUILD

@@ -899,16 +899,32 @@ mediapipe_proto_library(
 cc_library(
     name = "landmarks_smoothing_calculator",
     srcs = ["landmarks_smoothing_calculator.cc"],
+    hdrs = ["landmarks_smoothing_calculator.h"],
     deps = [
         ":landmarks_smoothing_calculator_cc_proto",
+        ":landmarks_smoothing_calculator_utils",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "landmarks_smoothing_calculator_utils",
+    srcs = ["landmarks_smoothing_calculator_utils.cc"],
+    hdrs = ["landmarks_smoothing_calculator_utils.h"],
+    deps = [
+        ":landmarks_smoothing_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/util/filtering:one_euro_filter",
         "//mediapipe/util/filtering:relative_velocity_filter",
-        "@com_google_absl//absl/algorithm:container",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/util/landmarks_smoothing_calculator.cc

@@ -1,4 +1,4 @@
-// Copyright 2020 The MediaPipe Authors.
+// Copyright 2023 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,412 +12,49 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.h"
+
 #include <memory>
 
-#include "absl/algorithm/container.h"
 #include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+#include "mediapipe/framework/api2/node.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/timestamp.h"
-#include "mediapipe/util/filtering/one_euro_filter.h"
-#include "mediapipe/util/filtering/relative_velocity_filter.h"
 
 namespace mediapipe {
+namespace api2 {
 
 namespace {
 
-constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
-constexpr char kLandmarksTag[] = "LANDMARKS";
-constexpr char kImageSizeTag[] = "IMAGE_SIZE";
-constexpr char kObjectScaleRoiTag[] = "OBJECT_SCALE_ROI";
-constexpr char kNormalizedFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
-constexpr char kFilteredLandmarksTag[] = "FILTERED_LANDMARKS";
-
 using ::mediapipe::NormalizedRect;
-using mediapipe::OneEuroFilter;
 using ::mediapipe::Rect;
-using mediapipe::RelativeVelocityFilter;
+using ::mediapipe::landmarks_smoothing::GetObjectScale;
-
+using ::mediapipe::landmarks_smoothing::InitializeLandmarksFilter;
-void NormalizedLandmarksToLandmarks(
+using ::mediapipe::landmarks_smoothing::LandmarksFilter;
-    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+using ::mediapipe::landmarks_smoothing::LandmarksToNormalizedLandmarks;
-    const int image_height, LandmarkList* landmarks) {
+using ::mediapipe::landmarks_smoothing::NormalizedLandmarksToLandmarks;
-  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
-    const auto& norm_landmark = norm_landmarks.landmark(i);
-
-    auto* landmark = landmarks->add_landmark();
-    landmark->set_x(norm_landmark.x() * image_width);
-    landmark->set_y(norm_landmark.y() * image_height);
-    // Scale Z the same way as X (using image width).
-    landmark->set_z(norm_landmark.z() * image_width);
-    landmark->set_visibility(norm_landmark.visibility());
-    landmark->set_presence(norm_landmark.presence());
-  }
-}
-
-void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
-                                    const int image_width,
-                                    const int image_height,
-                                    NormalizedLandmarkList* norm_landmarks) {
-  for (int i = 0; i < landmarks.landmark_size(); ++i) {
-    const auto& landmark = landmarks.landmark(i);
-
-    auto* norm_landmark = norm_landmarks->add_landmark();
-    norm_landmark->set_x(landmark.x() / image_width);
-    norm_landmark->set_y(landmark.y() / image_height);
-    // Scale Z the same way as X (using image width).
-    norm_landmark->set_z(landmark.z() / image_width);
-    norm_landmark->set_visibility(landmark.visibility());
-    norm_landmark->set_presence(landmark.presence());
-  }
-}
-
-// Estimate object scale to use its inverse value as velocity scale for
-// RelativeVelocityFilter. If value will be too small (less than
-// `options_.min_allowed_object_scale`) smoothing will be disabled and
-// landmarks will be returned as is.
-// Object scale is calculated as average between bounding box width and height
-// with sides parallel to axis.
-float GetObjectScale(const LandmarkList& landmarks) {
-  const auto& lm_minmax_x = absl::c_minmax_element(
-      landmarks.landmark(),
-      [](const auto& a, const auto& b) { return a.x() < b.x(); });
-  const float x_min = lm_minmax_x.first->x();
-  const float x_max = lm_minmax_x.second->x();
-
-  const auto& lm_minmax_y = absl::c_minmax_element(
-      landmarks.landmark(),
-      [](const auto& a, const auto& b) { return a.y() < b.y(); });
-  const float y_min = lm_minmax_y.first->y();
-  const float y_max = lm_minmax_y.second->y();
-
-  const float object_width = x_max - x_min;
-  const float object_height = y_max - y_min;
-
-  return (object_width + object_height) / 2.0f;
-}
-
-float GetObjectScale(const NormalizedRect& roi, const int image_width,
-                     const int image_height) {
-  const float object_width = roi.width() * image_width;
-  const float object_height = roi.height() * image_height;
-
-  return (object_width + object_height) / 2.0f;
-}
-
-float GetObjectScale(const Rect& roi) {
-  return (roi.width() + roi.height()) / 2.0f;
-}
-
-// Abstract class for various landmarks filters.
-class LandmarksFilter {
- public:
-  virtual ~LandmarksFilter() = default;
-
-  virtual absl::Status Reset() { return absl::OkStatus(); }
-
-  virtual absl::Status Apply(const LandmarkList& in_landmarks,
-                             const absl::Duration& timestamp,
-                             const absl::optional<float> object_scale_opt,
-                             LandmarkList* out_landmarks) = 0;
-};
-
-// Returns landmarks as is without smoothing.
-class NoFilter : public LandmarksFilter {
- public:
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    *out_landmarks = in_landmarks;
-    return absl::OkStatus();
-  }
-};
-
-// Please check RelativeVelocityFilter documentation for details.
-class VelocityFilter : public LandmarksFilter {
- public:
-  VelocityFilter(int window_size, float velocity_scale,
-                 float min_allowed_object_scale, bool disable_value_scaling)
-      : window_size_(window_size),
-        velocity_scale_(velocity_scale),
-        min_allowed_object_scale_(min_allowed_object_scale),
-        disable_value_scaling_(disable_value_scaling) {}
-
-  absl::Status Reset() override {
-    x_filters_.clear();
-    y_filters_.clear();
-    z_filters_.clear();
-    return absl::OkStatus();
-  }
-
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    // Get value scale as inverse value of the object scale.
-    // If value is too small smoothing will be disabled and landmarks will be
-    // returned as is.
-    float value_scale = 1.0f;
-    if (!disable_value_scaling_) {
-      const float object_scale =
-          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
-      if (object_scale < min_allowed_object_scale_) {
-        *out_landmarks = in_landmarks;
-        return absl::OkStatus();
-      }
-      value_scale = 1.0f / object_scale;
-    }
-
-    // Initialize filters once.
-    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
-
-    // Filter landmarks. Every axis of every landmark is filtered separately.
-    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const auto& in_landmark = in_landmarks.landmark(i);
-
-      auto* out_landmark = out_landmarks->add_landmark();
-      *out_landmark = in_landmark;
-      out_landmark->set_x(
-          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-      out_landmark->set_y(
-          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-      out_landmark->set_z(
-          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-    }
-
-    return absl::OkStatus();
-  }
-
- private:
-  // Initializes filters for the first time or after Reset. If initialized then
-  // check the size.
-  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
-    if (!x_filters_.empty()) {
-      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
-      return absl::OkStatus();
-    }
-
-    x_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-    y_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-    z_filters_.resize(n_landmarks,
-                      RelativeVelocityFilter(window_size_, velocity_scale_));
-
-    return absl::OkStatus();
-  }
-
-  int window_size_;
-  float velocity_scale_;
-  float min_allowed_object_scale_;
-  bool disable_value_scaling_;
-
-  std::vector<RelativeVelocityFilter> x_filters_;
-  std::vector<RelativeVelocityFilter> y_filters_;
-  std::vector<RelativeVelocityFilter> z_filters_;
-};
-
-// Please check OneEuroFilter documentation for details.
-class OneEuroFilterImpl : public LandmarksFilter {
- public:
-  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
-                    double derivate_cutoff, float min_allowed_object_scale,
-                    bool disable_value_scaling)
-      : frequency_(frequency),
-        min_cutoff_(min_cutoff),
-        beta_(beta),
-        derivate_cutoff_(derivate_cutoff),
-        min_allowed_object_scale_(min_allowed_object_scale),
-        disable_value_scaling_(disable_value_scaling) {}
-
-  absl::Status Reset() override {
-    x_filters_.clear();
-    y_filters_.clear();
-    z_filters_.clear();
-    return absl::OkStatus();
-  }
-
-  absl::Status Apply(const LandmarkList& in_landmarks,
-                     const absl::Duration& timestamp,
-                     const absl::optional<float> object_scale_opt,
-                     LandmarkList* out_landmarks) override {
-    // Initialize filters once.
-    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
-
-    // Get value scale as inverse value of the object scale.
-    // If value is too small smoothing will be disabled and landmarks will be
-    // returned as is.
-    float value_scale = 1.0f;
-    if (!disable_value_scaling_) {
-      const float object_scale =
-          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
-      if (object_scale < min_allowed_object_scale_) {
-        *out_landmarks = in_landmarks;
-        return absl::OkStatus();
-      }
-      value_scale = 1.0f / object_scale;
-    }
-
-    // Filter landmarks. Every axis of every landmark is filtered separately.
-    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
-      const auto& in_landmark = in_landmarks.landmark(i);
-
-      auto* out_landmark = out_landmarks->add_landmark();
-      *out_landmark = in_landmark;
-      out_landmark->set_x(
-          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
-      out_landmark->set_y(
-          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
-      out_landmark->set_z(
-          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
-    }
-
-    return absl::OkStatus();
-  }
-
- private:
-  // Initializes filters for the first time or after Reset. If initialized then
-  // check the size.
-  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
-    if (!x_filters_.empty()) {
-      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
-      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
-      return absl::OkStatus();
-    }
-
-    for (int i = 0; i < n_landmarks; ++i) {
-      x_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-      y_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-      z_filters_.push_back(
-          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
-    }
-
-    return absl::OkStatus();
-  }
-
-  double frequency_;
-  double min_cutoff_;
-  double beta_;
-  double derivate_cutoff_;
-  double min_allowed_object_scale_;
-  bool disable_value_scaling_;
-
-  std::vector<OneEuroFilter> x_filters_;
-  std::vector<OneEuroFilter> y_filters_;
-  std::vector<OneEuroFilter> z_filters_;
-};
 
 }  // namespace
 
-// A calculator to smooth landmarks over time.
+class LandmarksSmoothingCalculatorImpl
-//
+    : public NodeImpl<LandmarksSmoothingCalculator> {
-// Inputs:
-//   NORM_LANDMARKS: A NormalizedLandmarkList of landmarks you want to smooth.
-//   IMAGE_SIZE: A std::pair<int, int> represention of image width and height.
-//     Required to perform all computations in absolute coordinates to avoid any
-//     influence of normalized values.
-//   OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
-//     input landmarks) used to determine the object scale for some of the
-//     filters. If not provided - object scale will be calculated from
-//     landmarks.
-//
-// Outputs:
-//   NORM_FILTERED_LANDMARKS: A NormalizedLandmarkList of smoothed landmarks.
-//
-// Example config:
-//   node {
-//     calculator: "LandmarksSmoothingCalculator"
-//     input_stream: "NORM_LANDMARKS:pose_landmarks"
-//     input_stream: "IMAGE_SIZE:image_size"
-//     input_stream: "OBJECT_SCALE_ROI:roi"
-//     output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_filtered"
-//     options: {
-//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
-//         velocity_filter: {
-//           window_size: 5
-//           velocity_scale: 10.0
-//         }
-//       }
-//     }
-//   }
-//
-class LandmarksSmoothingCalculator : public CalculatorBase {
  public:
-  static absl::Status GetContract(CalculatorContract* cc);
+  absl::Status Open(CalculatorContext* cc) override {
-  absl::Status Open(CalculatorContext* cc) override;
+    ASSIGN_OR_RETURN(landmarks_filter_,
-  absl::Status Process(CalculatorContext* cc) override;
+                     InitializeLandmarksFilter(
-
+                         cc->Options<LandmarksSmoothingCalculatorOptions>()));
- private:
-  std::unique_ptr<LandmarksFilter> landmarks_filter_;
-};
-REGISTER_CALCULATOR(LandmarksSmoothingCalculator);
-
-absl::Status LandmarksSmoothingCalculator::GetContract(CalculatorContract* cc) {
-  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
-    cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
-    cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
-    cc->Outputs()
-        .Tag(kNormalizedFilteredLandmarksTag)
-        .Set<NormalizedLandmarkList>();
-
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
-      cc->Inputs().Tag(kObjectScaleRoiTag).Set<NormalizedRect>();
-    }
-  } else {
-    cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
-    cc->Outputs().Tag(kFilteredLandmarksTag).Set<LandmarkList>();
-
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag)) {
-      cc->Inputs().Tag(kObjectScaleRoiTag).Set<Rect>();
-    }
-  }
-
     return absl::OkStatus();
-}
-
-absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) {
-  cc->SetOffset(TimestampDiff(0));
-
-  // Pick landmarks filter.
-  const auto& options = cc->Options<LandmarksSmoothingCalculatorOptions>();
-  if (options.has_no_filter()) {
-    landmarks_filter_ = absl::make_unique<NoFilter>();
-  } else if (options.has_velocity_filter()) {
-    landmarks_filter_ = absl::make_unique<VelocityFilter>(
-        options.velocity_filter().window_size(),
-        options.velocity_filter().velocity_scale(),
-        options.velocity_filter().min_allowed_object_scale(),
-        options.velocity_filter().disable_value_scaling());
-  } else if (options.has_one_euro_filter()) {
-    landmarks_filter_ = absl::make_unique<OneEuroFilterImpl>(
-        options.one_euro_filter().frequency(),
-        options.one_euro_filter().min_cutoff(),
-        options.one_euro_filter().beta(),
-        options.one_euro_filter().derivate_cutoff(),
-        options.one_euro_filter().min_allowed_object_scale(),
-        options.one_euro_filter().disable_value_scaling());
-  } else {
-    RET_CHECK_FAIL()
-        << "Landmarks filter is either not specified or not supported";
   }
 
-  return absl::OkStatus();
+  absl::Status Process(CalculatorContext* cc) override {
-}
-
-absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
     // Check that landmarks are not empty and reset the filter if so.
     // Don't emit an empty packet for this timestamp.
-  if ((cc->Inputs().HasTag(kNormalizedLandmarksTag) &&
+    if ((kInNormLandmarks(cc).IsConnected() &&
-       cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) ||
+         kInNormLandmarks(cc).IsEmpty()) ||
-      (cc->Inputs().HasTag(kLandmarksTag) &&
+        (kInLandmarks(cc).IsConnected() && kInLandmarks(cc).IsEmpty())) {
-       cc->Inputs().Tag(kLandmarksTag).IsEmpty())) {
       MP_RETURN_IF_ERROR(landmarks_filter_->Reset());
       return absl::OkStatus();
     }
@@ -425,58 +62,55 @@ absl::Status LandmarksSmoothingCalculator::Process(CalculatorContext* cc) {
     const auto& timestamp =
         absl::Microseconds(cc->InputTimestamp().Microseconds());
 
-  if (cc->Inputs().HasTag(kNormalizedLandmarksTag)) {
+    if (kInNormLandmarks(cc).IsConnected()) {
-    const auto& in_norm_landmarks =
+      const auto& in_norm_landmarks = kInNormLandmarks(cc).Get();
-        cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
 
       int image_width;
       int image_height;
-    std::tie(image_width, image_height) =
+      std::tie(image_width, image_height) = kImageSize(cc).Get();
-        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
 
       absl::optional<float> object_scale;
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
+      if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
-        !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
+        auto& roi = kObjectScaleRoi(cc).Get<NormalizedRect>();
-      auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<NormalizedRect>();
         object_scale = GetObjectScale(roi, image_width, image_height);
       }
 
       auto in_landmarks = absl::make_unique<LandmarkList>();
-    NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width, image_height,
+      NormalizedLandmarksToLandmarks(in_norm_landmarks, image_width,
-                                   in_landmarks.get());
+                                     image_height, *in_landmarks.get());
 
       auto out_landmarks = absl::make_unique<LandmarkList>();
       MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
-        *in_landmarks, timestamp, object_scale, out_landmarks.get()));
+          *in_landmarks, timestamp, object_scale, *out_landmarks));
 
       auto out_norm_landmarks = absl::make_unique<NormalizedLandmarkList>();
       LandmarksToNormalizedLandmarks(*out_landmarks, image_width, image_height,
-                                   out_norm_landmarks.get());
+                                     *out_norm_landmarks.get());
 
-    cc->Outputs()
+      kOutNormLandmarks(cc).Send(std::move(out_norm_landmarks));
-        .Tag(kNormalizedFilteredLandmarksTag)
-        .Add(out_norm_landmarks.release(), cc->InputTimestamp());
     } else {
-    const auto& in_landmarks =
+      const auto& in_landmarks = kInLandmarks(cc).Get();
-        cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
 
       absl::optional<float> object_scale;
-    if (cc->Inputs().HasTag(kObjectScaleRoiTag) &&
+      if (kObjectScaleRoi(cc).IsConnected() && !kObjectScaleRoi(cc).IsEmpty()) {
-        !cc->Inputs().Tag(kObjectScaleRoiTag).IsEmpty()) {
+        auto& roi = kObjectScaleRoi(cc).Get<Rect>();
-      auto& roi = cc->Inputs().Tag(kObjectScaleRoiTag).Get<Rect>();
         object_scale = GetObjectScale(roi);
       }
 
       auto out_landmarks = absl::make_unique<LandmarkList>();
       MP_RETURN_IF_ERROR(landmarks_filter_->Apply(
-        in_landmarks, timestamp, object_scale, out_landmarks.get()));
+          in_landmarks, timestamp, object_scale, *out_landmarks));
 
-    cc->Outputs()
+      kOutLandmarks(cc).Send(std::move(out_landmarks));
-        .Tag(kFilteredLandmarksTag)
-        .Add(out_landmarks.release(), cc->InputTimestamp());
     }
 
     return absl::OkStatus();
-}
+  }
 
+ private:
+  std::unique_ptr<LandmarksFilter> landmarks_filter_;
+};
+MEDIAPIPE_NODE_IMPLEMENTATION(LandmarksSmoothingCalculatorImpl);
+
+}  // namespace api2
 }  // namespace mediapipe

mediapipe/calculators/util/landmarks_smoothing_calculator.h

@@ -0,0 +1,106 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_
+
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// A calculator to smooth landmarks over time.
+//
+// Inputs:
+//   NORM_LANDMARKS (optional): A NormalizedLandmarkList of landmarks you want
+//     to smooth.
+//   LANDMARKS (optional): A LandmarkList of landmarks you want to smooth.
+//   IMAGE_SIZE (optional): A std::pair<int, int> represention of image width
+//     and height. Required to perform all computations in absolute coordinates
+//     when smoothing NORM_LANDMARKS to avoid any influence of normalized
+//     values.
+//   OBJECT_SCALE_ROI (optional): A NormRect or Rect (depending on the format of
+//     input landmarks) used to determine the object scale for some of the
+//     filters. If not provided - object scale will be calculated from
+//     landmarks.
+//
+// Outputs:
+//   NORM_FILTERED_LANDMARKS (optional): A NormalizedLandmarkList of smoothed
+//     landmarks.
+//   FILTERED_LANDMARKS (optional): A LandmarkList of smoothed landmarks.
+//
+// Example config:
+//   node {
+//     calculator: "LandmarksSmoothingCalculator"
+//     input_stream: "NORM_LANDMARKS:landmarks"
+//     input_stream: "IMAGE_SIZE:image_size"
+//     input_stream: "OBJECT_SCALE_ROI:roi"
+//     output_stream: "NORM_FILTERED_LANDMARKS:landmarks_filtered"
+//     options: {
+//       [mediapipe.LandmarksSmoothingCalculatorOptions.ext] {
+//         velocity_filter: {
+//           window_size: 5
+//           velocity_scale: 10.0
+//         }
+//       }
+//     }
+//   }
+//
+class LandmarksSmoothingCalculator : public NodeIntf {
+ public:
+  static constexpr Input<mediapipe::NormalizedLandmarkList>::Optional
+      kInNormLandmarks{"NORM_LANDMARKS"};
+  static constexpr Input<mediapipe::LandmarkList>::Optional kInLandmarks{
+      "LANDMARKS"};
+  static constexpr Input<std::pair<int, int>>::Optional kImageSize{
+      "IMAGE_SIZE"};
+  static constexpr Input<OneOf<NormalizedRect, Rect>>::Optional kObjectScaleRoi{
+      "OBJECT_SCALE_ROI"};
+  static constexpr Output<mediapipe::NormalizedLandmarkList>::Optional
+      kOutNormLandmarks{"NORM_FILTERED_LANDMARKS"};
+  static constexpr Output<mediapipe::LandmarkList>::Optional kOutLandmarks{
+      "FILTERED_LANDMARKS"};
+  MEDIAPIPE_NODE_INTERFACE(LandmarksSmoothingCalculator, kInNormLandmarks,
+                           kInLandmarks, kImageSize, kObjectScaleRoi,
+                           kOutNormLandmarks, kOutLandmarks);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+    RET_CHECK(kInNormLandmarks(cc).IsConnected() ^
+              kInLandmarks(cc).IsConnected())
+        << "One and only one of NORM_LANDMARKS and LANDMARKS input is allowed";
+
+    // TODO: Verify scale ROI is of the same type as landmarks
+    // that are being smoothed.
+
+    if (kInNormLandmarks(cc).IsConnected()) {
+      RET_CHECK(kImageSize(cc).IsConnected());
+      RET_CHECK(kOutNormLandmarks(cc).IsConnected());
+      RET_CHECK(!kOutLandmarks(cc).IsConnected());
+    } else {
+      RET_CHECK(!kImageSize(cc).IsConnected());
+      RET_CHECK(kOutLandmarks(cc).IsConnected());
+      RET_CHECK(!kOutNormLandmarks(cc).IsConnected());
+    }
+
+    return absl::OkStatus();
+  }
+};
+
+}  // namespace api2
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_H_

mediapipe/calculators/util/landmarks_smoothing_calculator_utils.cc

@@ -0,0 +1,322 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h"
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+
+namespace {
+
+using ::mediapipe::NormalizedRect;
+using ::mediapipe::OneEuroFilter;
+using ::mediapipe::Rect;
+using ::mediapipe::RelativeVelocityFilter;
+
+// Estimate object scale to use its inverse value as velocity scale for
+// RelativeVelocityFilter. If value will be too small (less than
+// `options_.min_allowed_object_scale`) smoothing will be disabled and
+// landmarks will be returned as is.
+// Object scale is calculated as average between bounding box width and height
+// with sides parallel to axis.
+float GetObjectScale(const LandmarkList& landmarks) {
+  const auto& lm_minmax_x = absl::c_minmax_element(
+      landmarks.landmark(),
+      [](const auto& a, const auto& b) { return a.x() < b.x(); });
+  const float x_min = lm_minmax_x.first->x();
+  const float x_max = lm_minmax_x.second->x();
+
+  const auto& lm_minmax_y = absl::c_minmax_element(
+      landmarks.landmark(),
+      [](const auto& a, const auto& b) { return a.y() < b.y(); });
+  const float y_min = lm_minmax_y.first->y();
+  const float y_max = lm_minmax_y.second->y();
+
+  const float object_width = x_max - x_min;
+  const float object_height = y_max - y_min;
+
+  return (object_width + object_height) / 2.0f;
+}
+
+// Returns landmarks as is without smoothing.
+class NoFilter : public LandmarksFilter {
+ public:
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    out_landmarks = in_landmarks;
+    return absl::OkStatus();
+  }
+};
+
+// Please check RelativeVelocityFilter documentation for details.
+class VelocityFilter : public LandmarksFilter {
+ public:
+  VelocityFilter(int window_size, float velocity_scale,
+                 float min_allowed_object_scale, bool disable_value_scaling)
+      : window_size_(window_size),
+        velocity_scale_(velocity_scale),
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    // Get value scale as inverse value of the object scale.
+    // If value is too small smoothing will be disabled and landmarks will be
+    // returned as is.
+    float value_scale = 1.0f;
+    if (!disable_value_scaling_) {
+      const float object_scale =
+          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
+      if (object_scale < min_allowed_object_scale_) {
+        out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
+    }
+
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    x_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+    y_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+    z_filters_.resize(n_landmarks,
+                      RelativeVelocityFilter(window_size_, velocity_scale_));
+
+    return absl::OkStatus();
+  }
+
+  int window_size_;
+  float velocity_scale_;
+  float min_allowed_object_scale_;
+  bool disable_value_scaling_;
+
+  std::vector<RelativeVelocityFilter> x_filters_;
+  std::vector<RelativeVelocityFilter> y_filters_;
+  std::vector<RelativeVelocityFilter> z_filters_;
+};
+
+// Please check OneEuroFilter documentation for details.
+class OneEuroFilterImpl : public LandmarksFilter {
+ public:
+  OneEuroFilterImpl(double frequency, double min_cutoff, double beta,
+                    double derivate_cutoff, float min_allowed_object_scale,
+                    bool disable_value_scaling)
+      : frequency_(frequency),
+        min_cutoff_(min_cutoff),
+        beta_(beta),
+        derivate_cutoff_(derivate_cutoff),
+        min_allowed_object_scale_(min_allowed_object_scale),
+        disable_value_scaling_(disable_value_scaling) {}
+
+  absl::Status Reset() override {
+    x_filters_.clear();
+    y_filters_.clear();
+    z_filters_.clear();
+    return absl::OkStatus();
+  }
+
+  absl::Status Apply(const LandmarkList& in_landmarks,
+                     const absl::Duration& timestamp,
+                     const absl::optional<float> object_scale_opt,
+                     LandmarkList& out_landmarks) override {
+    // Initialize filters once.
+    MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size()));
+
+    // Get value scale as inverse value of the object scale.
+    // If value is too small smoothing will be disabled and landmarks will be
+    // returned as is.
+    float value_scale = 1.0f;
+    if (!disable_value_scaling_) {
+      const float object_scale =
+          object_scale_opt ? *object_scale_opt : GetObjectScale(in_landmarks);
+      if (object_scale < min_allowed_object_scale_) {
+        out_landmarks = in_landmarks;
+        return absl::OkStatus();
+      }
+      value_scale = 1.0f / object_scale;
+    }
+
+    // Filter landmarks. Every axis of every landmark is filtered separately.
+    for (int i = 0; i < in_landmarks.landmark_size(); ++i) {
+      const auto& in_landmark = in_landmarks.landmark(i);
+
+      auto* out_landmark = out_landmarks.add_landmark();
+      *out_landmark = in_landmark;
+      out_landmark->set_x(
+          x_filters_[i].Apply(timestamp, value_scale, in_landmark.x()));
+      out_landmark->set_y(
+          y_filters_[i].Apply(timestamp, value_scale, in_landmark.y()));
+      out_landmark->set_z(
+          z_filters_[i].Apply(timestamp, value_scale, in_landmark.z()));
+    }
+
+    return absl::OkStatus();
+  }
+
+ private:
+  // Initializes filters for the first time or after Reset. If initialized then
+  // check the size.
+  absl::Status InitializeFiltersIfEmpty(const int n_landmarks) {
+    if (!x_filters_.empty()) {
+      RET_CHECK_EQ(x_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(y_filters_.size(), n_landmarks);
+      RET_CHECK_EQ(z_filters_.size(), n_landmarks);
+      return absl::OkStatus();
+    }
+
+    for (int i = 0; i < n_landmarks; ++i) {
+      x_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      y_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+      z_filters_.push_back(
+          OneEuroFilter(frequency_, min_cutoff_, beta_, derivate_cutoff_));
+    }
+
+    return absl::OkStatus();
+  }
+
+  double frequency_;
+  double min_cutoff_;
+  double beta_;
+  double derivate_cutoff_;
+  double min_allowed_object_scale_;
+  bool disable_value_scaling_;
+
+  std::vector<OneEuroFilter> x_filters_;
+  std::vector<OneEuroFilter> y_filters_;
+  std::vector<OneEuroFilter> z_filters_;
+};
+
+}  // namespace
+
+void NormalizedLandmarksToLandmarks(
+    const NormalizedLandmarkList& norm_landmarks, const int image_width,
+    const int image_height, LandmarkList& landmarks) {
+  for (int i = 0; i < norm_landmarks.landmark_size(); ++i) {
+    const auto& norm_landmark = norm_landmarks.landmark(i);
+
+    auto* landmark = landmarks.add_landmark();
+    landmark->set_x(norm_landmark.x() * image_width);
+    landmark->set_y(norm_landmark.y() * image_height);
+    // Scale Z the same way as X (using image width).
+    landmark->set_z(norm_landmark.z() * image_width);
+    landmark->set_visibility(norm_landmark.visibility());
+    landmark->set_presence(norm_landmark.presence());
+  }
+}
+
+void LandmarksToNormalizedLandmarks(const LandmarkList& landmarks,
+                                    const int image_width,
+                                    const int image_height,
+                                    NormalizedLandmarkList& norm_landmarks) {
+  for (int i = 0; i < landmarks.landmark_size(); ++i) {
+    const auto& landmark = landmarks.landmark(i);
+
+    auto* norm_landmark = norm_landmarks.add_landmark();
+    norm_landmark->set_x(landmark.x() / image_width);
+    norm_landmark->set_y(landmark.y() / image_height);
+    // Scale Z the same way as X (using image width).
+    norm_landmark->set_z(landmark.z() / image_width);
+    norm_landmark->set_visibility(landmark.visibility());
+    norm_landmark->set_presence(landmark.presence());
+  }
+}
+
+float GetObjectScale(const NormalizedRect& roi, const int image_width,
+                     const int image_height) {
+  const float object_width = roi.width() * image_width;
+  const float object_height = roi.height() * image_height;
+
+  return (object_width + object_height) / 2.0f;
+}
+
+float GetObjectScale(const Rect& roi) {
+  return (roi.width() + roi.height()) / 2.0f;
+}
+
+absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
+    const LandmarksSmoothingCalculatorOptions& options) {
+  if (options.has_no_filter()) {
+    return absl::make_unique<NoFilter>();
+  } else if (options.has_velocity_filter()) {
+    return absl::make_unique<VelocityFilter>(
+        options.velocity_filter().window_size(),
+        options.velocity_filter().velocity_scale(),
+        options.velocity_filter().min_allowed_object_scale(),
+        options.velocity_filter().disable_value_scaling());
+  } else if (options.has_one_euro_filter()) {
+    return absl::make_unique<OneEuroFilterImpl>(
+        options.one_euro_filter().frequency(),
+        options.one_euro_filter().min_cutoff(),
+        options.one_euro_filter().beta(),
+        options.one_euro_filter().derivate_cutoff(),
+        options.one_euro_filter().min_allowed_object_scale(),
+        options.one_euro_filter().disable_value_scaling());
+  } else {
+    RET_CHECK_FAIL()
+        << "Landmarks filter is either not specified or not supported";
+  }
+}
+
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe

mediapipe/calculators/util/landmarks_smoothing_calculator_utils.h

@@ -0,0 +1,61 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_
+
+#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/util/filtering/one_euro_filter.h"
+#include "mediapipe/util/filtering/relative_velocity_filter.h"
+
+namespace mediapipe {
+namespace landmarks_smoothing {
+
+void NormalizedLandmarksToLandmarks(
+    const mediapipe::NormalizedLandmarkList& norm_landmarks,
+    const int image_width, const int image_height,
+    mediapipe::LandmarkList& landmarks);
+
+void LandmarksToNormalizedLandmarks(
+    const mediapipe::LandmarkList& landmarks, const int image_width,
+    const int image_height, mediapipe::NormalizedLandmarkList& norm_landmarks);
+
+float GetObjectScale(const NormalizedRect& roi, const int image_width,
+                     const int image_height);
+
+float GetObjectScale(const Rect& roi);
+
+// Abstract class for various landmarks filters.
+class LandmarksFilter {
+ public:
+  virtual ~LandmarksFilter() = default;
+
+  virtual absl::Status Reset() { return absl::OkStatus(); }
+
+  virtual absl::Status Apply(const mediapipe::LandmarkList& in_landmarks,
+                             const absl::Duration& timestamp,
+                             const absl::optional<float> object_scale_opt,
+                             mediapipe::LandmarkList& out_landmarks) = 0;
+};
+
+absl::StatusOr<std::unique_ptr<LandmarksFilter>> InitializeLandmarksFilter(
+    const mediapipe::LandmarksSmoothingCalculatorOptions& options);
+
+}  // namespace landmarks_smoothing
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_SMOOTHING_CALCULATOR_UTILS_H_