Internal change

PiperOrigin-RevId: 572125477
2023-10-09 21:26:33 -07:00 · 2023-10-09 21:26:33 -07:00 · ef6e712a88
commit ef6e712a88
parent f72542ae5d
7 changed files with 706 additions and 0 deletions
--- a/mediapipe/tasks/cc/vision/utils/ghum/BUILD
+++ b/mediapipe/tasks/cc/vision/utils/ghum/BUILD
@ -0,0 +1,61 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+cc_library(
+    name = "ghum_hand_topology",
+    hdrs = ["ghum_hand_topology.h"],
+)
+
+cc_library(
+    name = "ghum_hand_utils",
+    srcs = ["ghum_hand_utils.cc"],
+    hdrs = ["ghum_hand_utils.h"],
+    deps = [
+        ":ghum_hand_topology",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator_cc_proto",
+        "//mediapipe/framework/api2:builder",
+        "//mediapipe/framework/formats:body_rig_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/tasks/cc/vision/hand_landmarker:hand_topology",
+        "@com_google_absl//absl/log:absl_check",
+    ],
+)
+
+cc_library(
+    name = "ghum_topology",
+    hdrs = ["ghum_topology.h"],
+)
+
+cc_library(
+    name = "ghum_utils",
+    srcs = ["ghum_utils.cc"],
+    hdrs = ["ghum_utils.h"],
+    deps = [
+        ":ghum_topology",
+        "//mediapipe/calculators/util:combine_joints_calculator",
+        "//mediapipe/calculators/util:combine_joints_calculator_cc_proto",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator",
+        "//mediapipe/calculators/util:set_joints_visibility_calculator_cc_proto",
+        "//mediapipe/framework/api2:builder",
+        "//mediapipe/framework/formats:body_rig_cc_proto",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/tasks/cc/vision/pose_landmarker:pose_topology",
+        "@com_google_absl//absl/types:span",
+    ],
+)
--- a/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h
+++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h
@ -0,0 +1,46 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Defines topology of the GHUM Hand model. This is related only to the 3D model of
+the hand, not the way NN model predicts joint rotations for it.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_
+#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+// GHUM hand 16 joint names in order they are produced by the HandRig pipeline.
+enum class GhumHandJointName {
+  kHand = 0,
+  kIndex01,
+  kIndex02,
+  kIndex03,
+  kMiddle01,
+  kMiddle02,
+  kMiddle03,
+  kRing01,
+  kRing02,
+  kRing03,
+  kPinky01,
+  kPinky02,
+  kPinky03,
+  kThumb01,
+  kThumb02,
+  kThumb03,
+};
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
+
+#endif  // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_TOPOLOGY_H_
--- a/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.cc
+++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.cc
@ -0,0 +1,80 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h"
+
+#include <array>
+#include <utility>
+
+#include "absl/log/absl_check.h"
+#include "mediapipe/calculators/util/set_joints_visibility_calculator.h"
+#include "mediapipe/calculators/util/set_joints_visibility_calculator.pb.h"
+#include "mediapipe/framework/api2/builder.h"
+#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_topology.h"
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_topology.h"
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+namespace {
+
+using ::mediapipe::api2::SetJointsVisibilityCalculator;
+using ::mediapipe::api2::builder::Stream;
+using ::mediapipe::tasks::vision::hand_landmarker::HandLandmarkName;
+
+// Includes mapping for all 16 GHUM Hand joints.
+constexpr std::array<std::pair<GhumHandJointName, HandLandmarkName>, 16>
+    kGhumHandJointsToHandLandmarksMapping = {{
+        {GhumHandJointName::kHand, HandLandmarkName::kWrist},
+        {GhumHandJointName::kIndex01, HandLandmarkName::kIndex1},
+        {GhumHandJointName::kIndex02, HandLandmarkName::kIndex2},
+        {GhumHandJointName::kIndex03, HandLandmarkName::kIndex3},
+        {GhumHandJointName::kMiddle01, HandLandmarkName::kMiddle1},
+        {GhumHandJointName::kMiddle02, HandLandmarkName::kMiddle2},
+        {GhumHandJointName::kMiddle03, HandLandmarkName::kMiddle3},
+        {GhumHandJointName::kRing01, HandLandmarkName::kRing1},
+        {GhumHandJointName::kRing02, HandLandmarkName::kRing2},
+        {GhumHandJointName::kRing03, HandLandmarkName::kRing3},
+        {GhumHandJointName::kPinky01, HandLandmarkName::kPinky1},
+        {GhumHandJointName::kPinky02, HandLandmarkName::kPinky2},
+        {GhumHandJointName::kPinky03, HandLandmarkName::kPinky3},
+        {GhumHandJointName::kThumb01, HandLandmarkName::kThumb1},
+        {GhumHandJointName::kThumb02, HandLandmarkName::kThumb2},
+        {GhumHandJointName::kThumb03, HandLandmarkName::kThumb3},
+    }};
+
+}  // namespace
+
+Stream<JointList> SetGhumHandJointsVisibilityFromWorldLandmarks(
+    Stream<JointList> ghum_hand_joints,
+    Stream<LandmarkList> hand_world_landmarks,
+    mediapipe::api2::builder::Graph& graph) {
+  auto& node = graph.AddNode("SetJointsVisibilityCalculator");
+  auto& opts = node.GetOptions<SetJointsVisibilityCalculatorOptions>();
+  for (const auto& pair : kGhumHandJointsToHandLandmarksMapping) {
+    // Sanity check to verify that all hand joints are set and are set in the
+    // right order.
+    ABSL_CHECK_EQ(static_cast<int>(pair.first), opts.mapping_size());
+    auto* mapping = opts.add_mapping();
+    mapping->mutable_copy()->set_idx(static_cast<int>(pair.second));
+  }
+
+  ghum_hand_joints.ConnectTo(node[SetJointsVisibilityCalculator::kInJoints]);
+  hand_world_landmarks.ConnectTo(
+      node[SetJointsVisibilityCalculator::kInLandmarks]);
+
+  return node[SetJointsVisibilityCalculator::kOutJoints];
+}
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
--- a/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h
+++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_hand_utils.h
@ -0,0 +1,38 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Utility methods for populating Ghum Hand joints from joints produced by Hund
+Hand model and hand landmarks.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_
+#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_
+
+#include "mediapipe/framework/api2/builder.h"
+#include "mediapipe/framework/formats/body_rig.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+// Sets visibility of 16 GHUM hand joints from 21 hand world landmarks.
+mediapipe::api2::builder::Stream<mediapipe::JointList>
+SetGhumHandJointsVisibilityFromWorldLandmarks(
+    mediapipe::api2::builder::Stream<mediapipe::JointList> ghum_hand_joints,
+    mediapipe::api2::builder::Stream<mediapipe::LandmarkList>
+        hand_world_landmarks,
+    mediapipe::api2::builder::Graph& graph);
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
+
+#endif  // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_HAND_UTILS_H_
--- a/mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h
+++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h
@ -0,0 +1,230 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Defines topology of the GHUM model. This is related only to the 3D model of the
+body, not the way NN models predict joint rotations for it.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_TOPOLOGY_H_
+#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_TOPOLOGY_H_
+
+#include <array>
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+// GHUM 63 joint names in order they are produced by the PoseRig and
+// XenomorphRig pipelines.
+//
+// Note that some joints will always remain in default rotation (i.e.
+// kGhumDefaultJointRotation) depending on what joints are supported by the
+// pipeline.
+enum class GhumJointName {
+  kPelvis = 0,
+  kSpine01,
+  kSpine02,
+  kSpine03,
+  kNeck01,
+  kHead01,
+  kJaw01,
+  kEye01L,
+  kEyelidup01L,
+  kEye01R,
+  kEyelidup01R,
+  kEyeliddown01L,
+  kEyeliddown01R,
+  kClavicleL,
+  kUpperarmL,
+  kLowerarmL,
+  kHandL,
+  kIndex01L,
+  kIndex02L,
+  kIndex03L,
+  kMiddle01L,
+  kMiddle02L,
+  kMiddle03L,
+  kRing01L,
+  kRing02L,
+  kRing03L,
+  kPinky01L,
+  kPinky02L,
+  kPinky03L,
+  kThumb01L,
+  kThumb02L,
+  kThumb03L,
+  kClavicleR,
+  kUpperarmR,
+  kLowerarmR,
+  kHandR,
+  kIndex01R,
+  kIndex02R,
+  kIndex03R,
+  kMiddle01R,
+  kMiddle02R,
+  kMiddle03R,
+  kRing01R,
+  kRing02R,
+  kRing03R,
+  kPinky01R,
+  kPinky02R,
+  kPinky03R,
+  kThumb01R,
+  kThumb02R,
+  kThumb03R,
+  kThighL,
+  kCalfL,
+  kAnkleL,
+  kFootL,
+  kBallL,
+  kToes01L,
+  kThighR,
+  kCalfR,
+  kAnkleR,
+  kFootR,
+  kBallR,
+  kToes01R,
+};
+
+// GHUM default joint rotation. When applied - won't change initial pose of 3D
+// model.
+constexpr std::array<int, 6> kGhumDefaultJointRotation = {{1, 0, 0, 1, 0, 0}};
+
+// GHUM resting joint positions defined in 6D rotation format. Joint rotations
+// produced by Hund models should be applied ON TOP OF THEM (i.e. multiplied).
+//
+constexpr std::array<std::array<float, 6>, 63> kGhumRestingJointRotations = {{
+    {3.3306690738754696e-16, -1.1102230246251565e-16, 0.9907665252685547,
+     0.1355789452791214, -0.1355789452791214, 0.9907665252685547},
+    {-2.220446049250313e-16, -2.220446049250313e-16, 0.9965463876724243,
+     -0.08303803205490112, 0.08303803205490112, 0.9965463876724243},
+    {2.220446049250313e-16, -1.1102230246251565e-16, 0.9830689430236816,
+     0.1832359880208969, -0.1832359880208969, 0.9830689430236816},
+    {0.0, -1.1102230246251565e-16, 0.9998827576637268, -0.015312896110117435,
+     0.015312896110117435, 0.9998827576637268},
+    {1.1102230246251565e-16, -1.1102230246251565e-16, 0.9737479090690613,
+     -0.22762899100780487, 0.22762899100780487, 0.9737479090690613},
+    {0.0, -2.220446049250313e-16, 0.9975937008857727, -0.06933078169822693,
+     0.06933078169822693, 0.9975937008857727},
+    {0.0, 0.0, -0.7646467089653015, 0.6444497108459473, 0.6444497108459473,
+     0.7646467089653015},
+    {1.0, 0.0, 0.0, 1.0, 0.0, 0.0},
+    {1.0, 0.0, 0.0, 1.0, 0.0, 0.0},
+    {1.0, 0.0, 0.0, -1.0, 0.0, -1.2246468525851679e-16},
+    {1.0, 0.0, 0.0, -1.0, 0.0, -1.2246468525851679e-16},
+    {1.0, 0.0, 0.0, 1.0, 0.0, 0.0},
+    {1.0, 0.0, 0.0, -1.0, 0.0, -1.2246468525851679e-16},
+    {0.8921475410461426, 0.2971198558807373, -0.17762203514575958,
+     0.9233117699623108, -0.41535913944244385, 0.24334177374839783},
+    {0.4769144654273987, 0.87869793176651, -0.878876805305481,
+     0.4764299690723419, -0.011322125792503357, 0.03007427789270878},
+    {0.5015625953674316, 0.865040123462677, -0.8650957345962524,
+     0.5016061067581177, 0.00665692426264286, 0.009843245148658752},
+    {0.5015625953674316, 0.865040123462677, -0.8650957345962524,
+     0.5016061067581177, 0.00665692426264286, 0.009843245148658752},
+    {0.37746769189834595, 0.8884291648864746, -0.9007542133331299,
+     0.4176981747150421, 0.21484869718551636, 0.1903204619884491},
+    {0.37291213870048523, 0.9067361354827881, -0.9177148342132568,
+     0.39173614978790283, 0.13687947392463684, 0.1561165302991867},
+    {0.37291213870048523, 0.9067361354827881, -0.9177148342132568,
+     0.39173614978790283, 0.13687947392463684, 0.1561165302991867},
+    {0.46410179138183594, 0.8850933313369751, -0.8857383131980896,
+     0.4641091823577881, -0.008784775622189045, -0.03482021763920784},
+    {0.4025099277496338, 0.9154127836227417, -0.9146613478660583,
+     0.4022779166698456, 0.03715258091688156, -0.013853170908987522},
+    {0.4025099277496338, 0.9154127836227417, -0.9146613478660583,
+     0.4022779166698456, 0.03715258091688156, -0.013853170908987522},
+    {0.34835490584373474, 0.8863212466239929, -0.9299189448356628,
+     0.36770808696746826, -0.11789664626121521, -0.28147009015083313},
+    {0.35047438740730286, 0.8873153328895569, -0.9331605434417725,
+     0.35812410712242126, -0.07986952364444733, -0.2905488610267639},
+    {0.35047438740730286, 0.8873153328895569, -0.9331605434417725,
+     0.35812410712242126, -0.07986952364444733, -0.2905488610267639},
+    {0.29830706119537354, 0.7789736986160278, -0.9084595441818237,
+     0.4089738428592682, -0.2927698493003845, -0.4753318727016449},
+    {0.2655753493309021, 0.7524029612541199, -0.9429818987846375,
+     0.33284398913383484, -0.2006361186504364, -0.5684230327606201},
+    {0.2655753493309021, 0.7524029612541199, -0.9429818987846375,
+     0.33284398913383484, -0.2006361186504364, -0.5684230327606201},
+    {-0.1977168470621109, -0.2275119423866272, -0.7548012137413025,
+     0.6559536457061768, 0.6254464387893677, 0.719696581363678},
+    {-0.07406296581029892, -0.2195579558610916, -0.947542130947113,
+     0.3196309804916382, 0.31093189120292664, 0.9217539429664612},
+    {-0.07406296581029892, -0.2195579558610916, -0.947542130947113,
+     0.3196309804916382, 0.31093189120292664, 0.9217539429664612},
+    {0.8921475410461426, 0.2971198558807373, 0.17762203514575958,
+     -0.9233117699623108, 0.41535913944244385, -0.24334177374839783},
+    {0.4769144654273987, 0.87869793176651, 0.878876805305481,
+     -0.4764299690723419, 0.011322125792503357, -0.03007427789270878},
+    {0.5015625953674316, 0.865040123462677, 0.8650957345962524,
+     -0.5016061067581177, -0.00665692426264286, -0.009843245148658752},
+    {0.5015625953674316, 0.865040123462677, 0.8650957345962524,
+     -0.5016061067581177, -0.00665692426264286, -0.009843245148658752},
+    {0.37746769189834595, 0.8884291648864746, 0.9007542133331299,
+     -0.4176981747150421, -0.21484869718551636, -0.1903204619884491},
+    {0.37291213870048523, 0.9067361354827881, 0.9177148342132568,
+     -0.39173614978790283, -0.13687947392463684, -0.1561165302991867},
+    {0.37291213870048523, 0.9067361354827881, 0.9177148342132568,
+     -0.39173614978790283, -0.13687947392463684, -0.1561165302991867},
+    {0.46410179138183594, 0.8850933313369751, 0.8857383131980896,
+     -0.4641091823577881, 0.008784775622189045, 0.03482021763920784},
+    {0.4025099277496338, 0.9154127836227417, 0.9146613478660583,
+     -0.4022779166698456, -0.03715258091688156, 0.013853170908987522},
+    {0.4025099277496338, 0.9154127836227417, 0.9146613478660583,
+     -0.4022779166698456, -0.03715258091688156, 0.013853170908987522},
+    {0.34835490584373474, 0.8863212466239929, 0.9299189448356628,
+     -0.36770808696746826, 0.11789664626121521, 0.28147009015083313},
+    {0.35047438740730286, 0.8873153328895569, 0.9331605434417725,
+     -0.35812410712242126, 0.07986952364444733, 0.2905488610267639},
+    {0.35047438740730286, 0.8873153328895569, 0.9331605434417725,
+     -0.35812410712242126, 0.07986952364444733, 0.2905488610267639},
+    {0.29830706119537354, 0.7789736986160278, 0.9084595441818237,
+     -0.4089738428592682, 0.2927698493003845, 0.4753318727016449},
+    {0.2655753493309021, 0.7524029612541199, 0.9429818987846375,
+     -0.33284398913383484, 0.2006361186504364, 0.5684230327606201},
+    {0.2655753493309021, 0.7524029612541199, 0.9429818987846375,
+     -0.33284398913383484, 0.2006361186504364, 0.5684230327606201},
+    {-0.1977168470621109, -0.2275119423866272, 0.7548012137413025,
+     -0.6559536457061768, -0.6254464387893677, -0.719696581363678},
+    {-0.07406296581029892, -0.2195579558610916, 0.947542130947113,
+     -0.3196309804916382, -0.31093189120292664, -0.9217539429664612},
+    {-0.07406295835971832, -0.2195579558610916, 0.947542130947113,
+     -0.3196309804916382, -0.31093189120292664, -0.9217539429664612},
+    {-0.01636510342359543, 0.01171059813350439, -0.9988228678703308,
+     0.045474592596292496, 0.04566308110952377, 0.9988968372344971},
+    {0.05019572749733925, 0.024085810407996178, -0.9970070123672485,
+     -0.05764812231063843, -0.05879946053028107, 0.9980463981628418},
+    {-0.012483594939112663, -0.038166593760252, -0.9944827556610107,
+     0.104560986161232, 0.10415486991405487, 0.9937858581542969},
+    {3.3306690738754696e-16, 2.7755575615628914e-17, -0.30688488483428955,
+     0.9517466425895691, 0.9517466425895691, 0.30688488483428955},
+    {-0.06284070760011673, -0.009795981459319592, -0.15403378009796143,
+     0.9880656003952026, 0.9860652089118958, 0.15372197329998016},
+    {2.220446049250313e-16, 6.938893903907228e-18, -0.0882575735449791,
+     0.9960976839065552, 0.9960976839065552, 0.0882575735449791},
+    {-0.01636510342359543, 0.01171059813350439, 0.9988228678703308,
+     -0.045474592596292496, -0.04566308110952377, -0.9988968372344971},
+    {0.05019572749733925, 0.024085810407996178, 0.9970070123672485,
+     0.05764812231063843, 0.05879946053028107, -0.9980463981628418},
+    {-0.012483594939112663, -0.038166593760252, 0.9944827556610107,
+     -0.104560986161232, -0.10415486991405487, -0.9937858581542969},
+    {3.3306690738754696e-16, -8.326672684688674e-17, 0.30688488483428955,
+     -0.9517466425895691, -0.9517466425895691, -0.30688488483428955},
+    {-0.06284070760011673, -0.009795981459319592, 0.15403378009796143,
+     -0.9880656003952026, -0.9860652089118958, -0.15372197329998016},
+    {2.220446049250313e-16, -1.1796119636642288e-16, 0.0882575735449791,
+     -0.9960976839065552, -0.9960976839065552, -0.0882575735449791},
+}};
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
+
+#endif  // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_TOPOLOGY_H_
--- a/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.cc
+++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.cc
@ -0,0 +1,184 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h"
+
+#include <array>
+#include <utility>
+#include <vector>
+
+#include "absl/types/span.h"
+#include "mediapipe/calculators/util/combine_joints_calculator.h"
+#include "mediapipe/calculators/util/combine_joints_calculator.pb.h"
+#include "mediapipe/calculators/util/set_joints_visibility_calculator.h"
+#include "mediapipe/calculators/util/set_joints_visibility_calculator.pb.h"
+#include "mediapipe/framework/api2/builder.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/tasks/cc/vision/pose_landmarker/pose_topology.h"
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h"
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+namespace {
+
+using ::mediapipe::api2::CombineJointsCalculator;
+using ::mediapipe::api2::SetJointsVisibilityCalculator;
+using ::mediapipe::api2::builder::Graph;
+using ::mediapipe::api2::builder::SidePacket;
+using ::mediapipe::api2::builder::Stream;
+using ::mediapipe::tasks::vision::pose_landmarker::PoseLandmarkName;
+
+}  // namespace
+
+Stream<JointList> SetGhumJointsFromHundJoints(
+    std::vector<HundToGhumJointsMapping>& mappings, Graph& graph) {
+  auto& to_ghum = graph.AddNode("CombineJointsCalculator");
+  auto& to_ghum_options = to_ghum.GetOptions<CombineJointsCalculatorOptions>();
+  to_ghum_options.set_num_joints(63);
+
+  // Default joint values for not mapped joints.
+  for (float v : kGhumDefaultJointRotation) {
+    to_ghum_options.mutable_default_joint()->add_rotation_6d(v);
+  }
+  to_ghum_options.mutable_default_joint()->set_visibility(1.0);
+
+  for (int i = 0; i < mappings.size(); ++i) {
+    auto* subset = to_ghum_options.add_joints_mapping();
+    for (const auto& joint_name : mappings[i].ghum_joints_order) {
+      subset->add_idx(static_cast<int>(joint_name));
+    }
+    mappings[i].hund_joints.ConnectTo(
+        to_ghum[CombineJointsCalculator::kInJoints][i]);
+  }
+
+  return to_ghum[CombineJointsCalculator::kOutJoints];
+}
+
+Stream<JointList> SetGhumJointsVisibilityFromWorldLandmarks(
+    Stream<JointList> ghum_joints, Stream<LandmarkList> pose_world_landmarks,
+    Graph& graph) {
+  auto& set_visibility = graph.AddNode("SetJointsVisibilityCalculator");
+  auto& set_visibility_options =
+      set_visibility.GetOptions<SetJointsVisibilityCalculatorOptions>();
+  std::vector<std::pair<GhumJointName, std::vector<PoseLandmarkName>>> mapping =
+      {
+          {GhumJointName::kPelvis,
+           {PoseLandmarkName::kLeftHip, PoseLandmarkName::kRightHip}},
+          {GhumJointName::kSpine01,
+           {PoseLandmarkName::kLeftHip, PoseLandmarkName::kRightHip}},
+          {GhumJointName::kSpine02,
+           {PoseLandmarkName::kLeftHip, PoseLandmarkName::kRightHip}},
+          {GhumJointName::kSpine03,
+           {PoseLandmarkName::kLeftShoulder, PoseLandmarkName::kRightShoulder}},
+          {GhumJointName::kNeck01,
+           {PoseLandmarkName::kLeftShoulder, PoseLandmarkName::kRightShoulder}},
+          {GhumJointName::kHead01,
+           {PoseLandmarkName::kLeftShoulder, PoseLandmarkName::kRightShoulder}},
+          {GhumJointName::kJaw01,
+           {PoseLandmarkName::kMouthLeft, PoseLandmarkName::kMouthRight}},
+          {GhumJointName::kEye01L, {PoseLandmarkName::kLeftEye}},
+          {GhumJointName::kEyelidup01L, {PoseLandmarkName::kLeftEye}},
+          {GhumJointName::kEye01R, {PoseLandmarkName::kRightEye}},
+          {GhumJointName::kEyelidup01R, {PoseLandmarkName::kRightEye}},
+          {GhumJointName::kEyeliddown01L, {PoseLandmarkName::kLeftEye}},
+          {GhumJointName::kEyeliddown01R, {PoseLandmarkName::kRightEye}},
+          {GhumJointName::kClavicleL, {PoseLandmarkName::kLeftShoulder}},
+          {GhumJointName::kUpperarmL, {PoseLandmarkName::kLeftShoulder}},
+          {GhumJointName::kLowerarmL, {PoseLandmarkName::kLeftElbow}},
+          {GhumJointName::kHandL, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kIndex01L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kIndex02L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kIndex03L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kMiddle01L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kMiddle02L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kMiddle03L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kRing01L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kRing02L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kRing03L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kPinky01L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kPinky02L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kPinky03L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kThumb01L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kThumb02L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kThumb03L, {PoseLandmarkName::kLeftWrist}},
+          {GhumJointName::kClavicleR, {PoseLandmarkName::kRightShoulder}},
+          {GhumJointName::kUpperarmR, {PoseLandmarkName::kRightShoulder}},
+          {GhumJointName::kLowerarmR, {PoseLandmarkName::kRightElbow}},
+          {GhumJointName::kHandR, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kIndex01R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kIndex02R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kIndex03R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kMiddle01R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kMiddle02R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kMiddle03R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kRing01R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kRing02R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kRing03R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kPinky01R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kPinky02R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kPinky03R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kThumb01R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kThumb02R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kThumb03R, {PoseLandmarkName::kRightWrist}},
+          {GhumJointName::kThighL, {PoseLandmarkName::kLeftHip}},
+          {GhumJointName::kCalfL, {PoseLandmarkName::kLeftKnee}},
+          {GhumJointName::kAnkleL, {PoseLandmarkName::kLeftAnkle}},
+          {GhumJointName::kFootL, {PoseLandmarkName::kLeftAnkle}},
+          {GhumJointName::kBallL, {PoseLandmarkName::kLeftHeel}},
+          {GhumJointName::kToes01L, {PoseLandmarkName::kLeftFootIndex}},
+          {GhumJointName::kThighR, {PoseLandmarkName::kRightHip}},
+          {GhumJointName::kCalfR, {PoseLandmarkName::kRightKnee}},
+          {GhumJointName::kAnkleR, {PoseLandmarkName::kRightAnkle}},
+          {GhumJointName::kFootR, {PoseLandmarkName::kRightAnkle}},
+          {GhumJointName::kBallR, {PoseLandmarkName::kRightHeel}},
+          {GhumJointName::kToes01R, {PoseLandmarkName::kRightFootIndex}},
+      };
+  for (const auto& pair : mapping) {
+    const auto& joint_name = pair.first;
+    const auto& landmark_names = pair.second;
+
+    CHECK_EQ(static_cast<int>(joint_name),
+             set_visibility_options.mapping_size());
+    auto* mapping = set_visibility_options.add_mapping();
+
+    if (landmark_names.size() == 1) {
+      mapping->mutable_copy()->set_idx(static_cast<int>(landmark_names[0]));
+    } else {
+      for (const auto& landmark_name : landmark_names) {
+        mapping->mutable_highest()->add_idx(static_cast<int>(landmark_name));
+      }
+    }
+  }
+
+  ghum_joints.ConnectTo(
+      set_visibility[SetJointsVisibilityCalculator::kInJoints]);
+  pose_world_landmarks.ConnectTo(
+      set_visibility[SetJointsVisibilityCalculator::kInLandmarks]);
+
+  return set_visibility[SetJointsVisibilityCalculator::kOutJoints];
+}
+
+std::vector<std::array<float, 6>> GetGhumRestingJointRotationsSubset(
+    absl::Span<const GhumJointName> ghum_joint_names) {
+  std::vector<std::array<float, 6>> res;
+  res.reserve(ghum_joint_names.size());
+  for (GhumJointName ghum_joint_name : ghum_joint_names) {
+    res.push_back(
+        kGhumRestingJointRotations[static_cast<int>(ghum_joint_name)]);
+  }
+  return res;
+}
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
--- a/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h
+++ b/mediapipe/tasks/cc/vision/utils/ghum/ghum_utils.h
@ -0,0 +1,67 @@
+/* Copyright 2023 The MediaPipe Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Utility methods for populating Ghum joints from joints produced by Hund Hand
+Pose and Hund Hand models and pose landmarks.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_UTILS_H_
+#define MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_UTILS_H_
+
+#include <array>
+#include <vector>
+
+#include "absl/types/span.h"
+#include "mediapipe/framework/api2/builder.h"
+#include "mediapipe/framework/formats/body_rig.pb.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/tasks/cc/vision/utils/ghum/ghum_topology.h"
+
+namespace mediapipe::tasks::vision::utils::ghum {
+
+struct HundToGhumJointsMapping {
+  // Joints produced by the HUND models.
+  mediapipe::api2::builder::Stream<mediapipe::JointList> hund_joints;
+  // Order of joints in GHUM topology.
+  absl::Span<const GhumJointName> ghum_joints_order;
+};
+
+// Sets GHUM joints from given HUND joints according to the mappings and in
+// specified order.
+//
+// All joints that are not defined will remain in kGhumDefaultJointRotation
+// and with 1.0 visibility.
+//
+// All joints specified later in order will override those that were specified
+// earlier.
+mediapipe::api2::builder::Stream<mediapipe::JointList>
+SetGhumJointsFromHundJoints(
+    std::vector<HundToGhumJointsMapping>& hund_to_ghum_joints_mappings,
+    mediapipe::api2::builder::Graph& graph);
+
+// Sets visibility of 63 GHUM joints from 33 pose world landmarks.
+mediapipe::api2::builder::Stream<mediapipe::JointList>
+SetGhumJointsVisibilityFromWorldLandmarks(
+    mediapipe::api2::builder::Stream<mediapipe::JointList> ghum_joints,
+    mediapipe::api2::builder::Stream<mediapipe::LandmarkList>
+        pose_world_landmarks,
+    mediapipe::api2::builder::Graph& graph);
+
+// Gets a subset of kGhumRestingJointRotations in 6D format.
+std::vector<std::array<float, 6>> GetGhumRestingJointRotationsSubset(
+    absl::Span<const GhumJointName> ghum_joint_names);
+
+}  // namespace mediapipe::tasks::vision::utils::ghum
+
+#endif  // MEDIAPIPE_TASKS_CC_VISION_UTILS_GHUM_GHUM_UTILS_H_