Merge e610e80c9e into c450283715

README.md

@@ -170,3 +170,17 @@ We welcome contributions. Please follow these
 
 We use GitHub issues for tracking requests and bugs. Please post questions to
 the MediaPipe Stack Overflow with a `mediapipe` tag.
+
+
+## Steps to add an image_flip_calculator
+Objective of this exercise is to create a custom calculator which can do some custom prcessing of the image('flipping') and then insert this calculator in the mediapipe graph pipe line.
+Description: Calculators are the blocks of code which reperesent a node in a subgraph in mediapipe computational graphs.
+
+1. Goto /mediapipe/calculators/image directory , which contains several image manipulation calculators, place your custom calculator code in a .cc file right here.
+2. Edit the BUILD file in the same image folder by creating a <cc_library> entry for your calculator (the name is 'custom_calculator' in this case, mentioning the name the calculator  src file in last step in the <srcs> section).
+3. Edit the /mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt , creating a <node> entry for your calculator by giving it a name ('CustomImageFlipCalculator' in this case, same as that of the class defined by the calculator in step 1. ) whose input willbe  the inputstream  of the graph and output will connect to the node previously connected to the original input stream)
+4. The node created in the step 3 will connect to you custom_calculator by making an entry in the  BUILD file in /mediapipe/modules/pose_landmark/ ; edit the file by finding the entry which mentions name <PoseLandmarkCpu> in the <register_as> section. <PoseLandmarkCpu> is the name of the sub_graph described by the   pose_landmark_cpu.pbtxt.
+In the 'deps' section, add the calculator entry as dependency to the subgraph(in  this case "//mediapipe/calculators/image:custom_calculator").
+5. BUILD your custom calculator by running 
+"bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/calculators/image:custom_calculator" , and also for the pose_detection by running 
+"bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/pose_tracking:pose_tracking_cpu", and then run the pose detection desktop example.

mediapipe/calculators/image/BUILD

@@ -693,6 +693,30 @@ cc_library(
     ],
 )
 
+# added by akash
+cc_library(
+    name = "custom_calculator",
+    srcs = ["image_flip_calculator.cc"],
+    visibility = ["//visibility:public"],
+
+    deps = [
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status:statusor",
+        "@eigen_archive//:eigen3",
+    ],
+        alwayslink = 1,
+
+)
+
+
 cc_library(
     name = "affine_transformation_runner_opencv",
     srcs = ["affine_transformation_runner_opencv.cc"],

mediapipe/calculators/image/image_flip_calculator.cc

@@ -0,0 +1,179 @@
+// Copyright 2021 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//#include "mediapipe/calculators/image/image_clone_calculator.pb.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/port/status.h"
+
+// akash
+
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+//#include "mediapipe/framework/formats/video_stream_header.h"
+// akash
+
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gl_calculator_helper.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
+namespace mediapipe {
+namespace api2 {
+
+#if MEDIAPIPE_DISABLE_GPU
+// Just a placeholder to not have to depend on mediapipe::GpuBuffer.
+using GpuBuffer = AnyType;
+#else
+using GpuBuffer = mediapipe::GpuBuffer;
+#endif  // MEDIAPIPE_DISABLE_GPU
+
+// Clones an input image and makes sure in the output clone the pixel data are
+// stored on the target storage (CPU vs GPU) specified in the calculator option.
+//
+// The clone shares ownership of the input pixel data on the existing storage.
+// If the target storage is different from the existing one, then the data is
+// further copied there.
+//
+// Example usage:
+// node {
+//   calculator: "ImageCloneCalculator"
+//   input_stream: "input"
+//   output_stream: "output"
+//   options: {
+//     [mediapipe.ImageCloneCalculatorOptions.ext] {
+//       output_on_gpu: true
+//     }
+//   }
+// }
+class CustomImageFlipCalculator : public Node {
+ public:
+  static constexpr Input<ImageFrame> kIn{"IMAGE"};
+  static constexpr Output<ImageFrame> kOut{"IMAGE"};
+  
+  MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+  static absl::Status UpdateContract(CalculatorContract* cc) {
+#if MEDIAPIPE_DISABLE_GPU
+    /*if (cc->Options<mediapipe::ImageCloneCalculatorOptions>().output_on_gpu()) {
+      return absl::UnimplementedError(
+          "GPU processing is disabled in build flags");
+    }*/
+        
+#else
+    MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+#endif  // MEDIAPIPE_DISABLE_GPU
+    return absl::OkStatus();
+  }
+
+    absl::Status Open(CalculatorContext* cc) override {
+    /*
+    const auto& options = cc->Options<mediapipe::ImageCloneCalculatorOptions>();
+    output_on_gpu_ = options.output_on_gpu();
+#if !MEDIAPIPE_DISABLE_GPU
+    MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
+#endif  // !MEDIAPIPE_DISABLE_GPU*/
+    return absl::OkStatus();
+  }
+
+  absl::Status Process(CalculatorContext* cc) override {
+      std::cout<<"\n inside image_flip_calculator  process";
+      float scale_fact=1;
+      int output_width=0;
+      int output_height=0;
+      mediapipe::ImageFormat::Format format;
+      cv::Mat input_mat;
+      cv::Mat scaled_mat;
+      cv::Mat flipped_mat;
+
+    std::unique_ptr<ImageFrame> output;
+    const auto& input = *kIn(cc);
+    if (FALSE) {//
+#if !MEDIAPIPE_DISABLE_GPU
+      // Create an output Image that co-owns the underlying texture buffer as
+      // the input Image.
+      output = std::make_unique<Image>(input.GetGpuBuffer());
+#endif  // !MEDIAPIPE_DISABLE_GPU
+    } else {
+        
+      // Make a copy of the input packet to co-own the input Image.
+      mediapipe::Packet* packet_copy_ptr =
+          new mediapipe::Packet(kIn(cc).packet());
+      // Create an output Image that (co-)owns a new ImageFrame that points to
+      // the same pixel data as the input Image and also owns the packet
+      // copy. As a result, the output Image indirectly co-owns the input
+      // Image. This ensures a correct life span of the shared pixel data.
+        
+        // copde to flip
+        
+
+        output_width=input.Width();
+        output_height=input.Height();
+        input_mat = formats::MatView(&input);
+        
+        format = input.Format();
+        /*cv::Rect myROI(0, 0, output_width/2, output_height/2);
+        cv::Mat croppedRef(input_mat, myROI);
+        cv::Mat cropped;
+        croppedRef.copyTo(cropped);
+        cv::Mat imgPanel(480, 640, CV_8UC1, Scalar(0));
+        Mat imgPanelRoi(imgPanel, Rect(0, 0, imgSrc.cols, imgSrc.rows));
+        imgSrc.copyTo(imgPanelRoi);
+        cv::resize(cropped, input_mat, cv::Size(), 2, 2);
+        std::cout<<"size>>>>"<<input_mat.size;*/
+        
+
+
+
+            
+        // code to flip
+        cv::flip(input_mat, flipped_mat, 1);
+// Use Flip code 0 to flip vertically
+
+        output = absl::make_unique<mediapipe::ImageFrame>(
+         mediapipe::ImageFormat::SRGB,
+         int(output_width),
+         int(output_height),
+         mediapipe::ImageFrame::kGlDefaultAlignmentBoundary
+       );
+        flipped_mat.copyTo(formats::MatView(output.get()));
+    }
+
+    if (output_on_gpu_) {
+#if !MEDIAPIPE_DISABLE_GPU
+      gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); });
+#endif  // !MEDIAPIPE_DISABLE_GPU
+    } else {
+      //output->ConvertToCpu();
+        
+    }
+      //ASSIGN_OR_RETURN(output,);
+
+      kOut(cc).Send(std::move(output));
+
+    return absl::OkStatus();
+  }
+
+ private:
+  bool output_on_gpu_;
+#if !MEDIAPIPE_DISABLE_GPU
+  mediapipe::GlCalculatorHelper gpu_helper_;
+#endif  // !MEDIAPIPE_DISABLE_GPU
+};
+MEDIAPIPE_REGISTER_NODE(CustomImageFlipCalculator);
+
+}  // namespace api2
+}  // namespace mediapipe

mediapipe/modules/pose_landmark/BUILD

@@ -160,6 +160,8 @@ mediapipe_simple_subgraph(
         "//mediapipe/calculators/image:image_properties_calculator",
         "//mediapipe/calculators/util:from_image_calculator",
         "//mediapipe/modules/pose_detection:pose_detection_cpu",
+        "//mediapipe/calculators/image:custom_calculator",
+
     ],
 )
 

mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt

@@ -30,7 +30,7 @@
 type: "PoseLandmarkCpu"
 
 # CPU image. (ImageFrame)
-input_stream: "IMAGE:image"
+input_stream: "IMAGE:image1"
 
 # Whether to filter landmarks across different input images to reduce jitter.
 # If unspecified, functions as set to true. (bool)
@@ -130,6 +130,14 @@ node {
   }
 }
 
+# added by Akash for Image Flip Calculator
+
+node {
+  calculator: "CustomImageFlipCalculator"
+  input_stream: "IMAGE:image1"
+  output_stream: "IMAGE:image"
+} 
+
 # Checks if there's previous pose rect calculated from landmarks.
 node: {
   calculator: "PacketPresenceCalculator"