From 49d9440db9cae6d9a047f9408bbe237265f01e90 Mon Sep 17 00:00:00 2001 From: akash Date: Mon, 16 May 2022 15:47:31 +0530 Subject: [PATCH] initial commit - pose with flipped landmarks --- mediapipe/calculators/image/BUILD | 24 +++ .../image/image_flip_calculator.cc | 165 ++++++++++++++++++ mediapipe/modules/pose_landmark/BUILD | 2 + .../pose_landmark/pose_landmark_cpu.pbtxt | 10 +- 4 files changed, 200 insertions(+), 1 deletion(-) create mode 100755 mediapipe/calculators/image/image_flip_calculator.cc diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index 5428f98fd..812bdba89 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -691,6 +691,30 @@ cc_library( ], ) +# added by akash +cc_library( + name = "custom_calculator", + srcs = ["image_flip_calculator.cc"], + visibility = ["//visibility:public"], + + deps = [ + "//mediapipe/framework/api2:node", + "//mediapipe/framework/formats:image", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status:statusor", + "@eigen_archive//:eigen3", + ], + alwayslink = 1, + +) + + cc_library( name = "affine_transformation_runner_opencv", srcs = ["affine_transformation_runner_opencv.cc"], diff --git a/mediapipe/calculators/image/image_flip_calculator.cc b/mediapipe/calculators/image/image_flip_calculator.cc new file mode 100755 index 000000000..d48c52cc4 --- /dev/null +++ b/mediapipe/calculators/image/image_flip_calculator.cc @@ -0,0 +1,165 @@ +// Copyright 2021 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//#include "mediapipe/calculators/image/image_clone_calculator.pb.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/status.h" + +// akash + +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +//#include "mediapipe/framework/formats/video_stream_header.h" +// akash + +#if !MEDIAPIPE_DISABLE_GPU +#include "mediapipe/gpu/gl_calculator_helper.h" +#endif // !MEDIAPIPE_DISABLE_GPU + +namespace mediapipe { +namespace api2 { + +#if MEDIAPIPE_DISABLE_GPU +// Just a placeholder to not have to depend on mediapipe::GpuBuffer. +using GpuBuffer = AnyType; +#else +using GpuBuffer = mediapipe::GpuBuffer; +#endif // MEDIAPIPE_DISABLE_GPU + +// Clones an input image and makes sure in the output clone the pixel data are +// stored on the target storage (CPU vs GPU) specified in the calculator option. +// +// The clone shares ownership of the input pixel data on the existing storage. +// If the target storage is different from the existing one, then the data is +// further copied there. +// +// Example usage: +// node { +// calculator: "ImageCloneCalculator" +// input_stream: "input" +// output_stream: "output" +// options: { +// [mediapipe.ImageCloneCalculatorOptions.ext] { +// output_on_gpu: true +// } +// } +// } +class AkashImageFlipCalculator : public Node { + public: + static constexpr Input kIn{"IMAGE"}; + static constexpr Output kOut{"IMAGE"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + static absl::Status UpdateContract(CalculatorContract* cc) { +#if MEDIAPIPE_DISABLE_GPU + /*if (cc->Options().output_on_gpu()) { + return absl::UnimplementedError( + "GPU processing is disabled in build flags"); + }*/ + +#else + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); +#endif // MEDIAPIPE_DISABLE_GPU + return absl::OkStatus(); + } + + absl::Status Open(CalculatorContext* cc) override { + /* + const auto& options = cc->Options(); + output_on_gpu_ = options.output_on_gpu(); +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); +#endif // !MEDIAPIPE_DISABLE_GPU*/ + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) override { + std::cout<<"\n inside image_flip_calculator process"; + float scale_fact=1; + int output_width=0; + int output_height=0; + mediapipe::ImageFormat::Format format; + cv::Mat input_mat; + cv::Mat scaled_mat; + cv::Mat flipped_mat; + + std::unique_ptr output; + const auto& input = *kIn(cc); + if (FALSE) {// +#if !MEDIAPIPE_DISABLE_GPU + // Create an output Image that co-owns the underlying texture buffer as + // the input Image. + output = std::make_unique(input.GetGpuBuffer()); +#endif // !MEDIAPIPE_DISABLE_GPU + } else { + + // Make a copy of the input packet to co-own the input Image. + mediapipe::Packet* packet_copy_ptr = + new mediapipe::Packet(kIn(cc).packet()); + // Create an output Image that (co-)owns a new ImageFrame that points to + // the same pixel data as the input Image and also owns the packet + // copy. As a result, the output Image indirectly co-owns the input + // Image. This ensures a correct life span of the shared pixel data. + + // copde to flip + + + output_width=input.Width(); + output_height=input.Height(); + input_mat = formats::MatView(&input); + + format = input.Format(); + // code to flip + cv::flip(input_mat, flipped_mat, 1); +// Use Flip code 0 to flip vertically + + output = absl::make_unique( + mediapipe::ImageFormat::SRGB, + int(output_width), + int(output_height), + mediapipe::ImageFrame::kGlDefaultAlignmentBoundary + ); + flipped_mat.copyTo(formats::MatView(output.get())); + } + + if (output_on_gpu_) { +#if !MEDIAPIPE_DISABLE_GPU + gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); }); +#endif // !MEDIAPIPE_DISABLE_GPU + } else { + //output->ConvertToCpu(); + + } + //ASSIGN_OR_RETURN(output,); + + kOut(cc).Send(std::move(output)); + + return absl::OkStatus(); + } + + private: + bool output_on_gpu_; +#if !MEDIAPIPE_DISABLE_GPU + mediapipe::GlCalculatorHelper gpu_helper_; +#endif // !MEDIAPIPE_DISABLE_GPU +}; +MEDIAPIPE_REGISTER_NODE(AkashImageFlipCalculator); + +} // namespace api2 +} // namespace mediapipe diff --git a/mediapipe/modules/pose_landmark/BUILD b/mediapipe/modules/pose_landmark/BUILD index 424579a46..68fcd0116 100644 --- a/mediapipe/modules/pose_landmark/BUILD +++ b/mediapipe/modules/pose_landmark/BUILD @@ -156,6 +156,8 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:from_image_calculator", "//mediapipe/modules/pose_detection:pose_detection_cpu", + "//mediapipe/calculators/image:custom_calculator", + ], ) diff --git a/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt index 5faf08a76..69d9dbda2 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt @@ -30,7 +30,7 @@ type: "PoseLandmarkCpu" # CPU image. (ImageFrame) -input_stream: "IMAGE:image" +input_stream: "IMAGE:image1" # Whether to filter landmarks across different input images to reduce jitter. # If unspecified, functions as set to true. (bool) @@ -130,6 +130,14 @@ node { } } +# added by Akash + +node { + calculator: "AkashImageFlipCalculator" + input_stream: "IMAGE:image1" + output_stream: "IMAGE:image" +} + # Checks if there's previous pose rect calculated from landmarks. node: { calculator: "PacketPresenceCalculator"