cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

No public description

Browse Source 
PiperOrigin-RevId: 559211117
This commit is contained in:
MediaPipe Team 2023-08-22 13:40:42 -07:00 committed by Copybara-Service
parent 8c4b971c14
commit 3443fe4c8e
4 changed files with 310 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mediapipe/tasks/cc/vision/utils/BUILD
View File

@ -111,3 +111,23 @@ cc_test(
"//mediapipe/tasks/cc/components/containers:rect", "//mediapipe/tasks/cc/components/containers:rect",
], ],
) )
cc_library(
name = "data_renderer",
srcs = ["data_renderer.cc"],
hdrs = ["data_renderer.h"],
deps = [
"//mediapipe/calculators/util:annotation_overlay_calculator",
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
"//mediapipe/calculators/util:landmarks_to_render_data_calculator_cc_proto",
"//mediapipe/calculators/util:rect_to_render_data_calculator_cc_proto",
"//mediapipe/calculators/util:rect_to_render_scale_calculator",
"//mediapipe/calculators/util:rect_to_render_scale_calculator_cc_proto",
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/util:render_data_cc_proto",
"@com_google_absl//absl/types:span",
],
)

88
mediapipe/tasks/cc/vision/utils/data_renderer.cc Normal file
View File

@ -0,0 +1,88 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/cc/vision/utils/data_renderer.h"
#include <optional>
#include <utility>
#include <vector>
#include "absl/types/span.h"
#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h"
#include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h"
#include "mediapipe/calculators/util/rect_to_render_scale_calculator.pb.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/util/render_data.pb.h"
namespace mediapipe::tasks::vision::utils {
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::Stream;
Stream<Image> Render(Stream<Image> image,
absl::Span<Stream<mediapipe::RenderData>> render_data_list,
Graph& graph) {
auto& annotation_overlay = graph.AddNode("AnnotationOverlayCalculator");
image >> annotation_overlay.In("UIMAGE");
for (int i = 0; i < render_data_list.size(); ++i) {
render_data_list[i] >> annotation_overlay.In(i);
}
return annotation_overlay.Out("UIMAGE").Cast<Image>();
}
Stream<mediapipe::RenderData> RenderLandmarks(
Stream<mediapipe::NormalizedLandmarkList> landmarks,
std::optional<api2::builder::Stream<float>> render_scale,
const mediapipe::LandmarksToRenderDataCalculatorOptions& renderer_options,
Graph& graph) {
auto& landmarks_render = graph.AddNode("LandmarksToRenderDataCalculator");
landmarks_render
.GetOptions<mediapipe::LandmarksToRenderDataCalculatorOptions>()
.CopyFrom(renderer_options);
landmarks >> landmarks_render.In("NORM_LANDMARKS");
if (render_scale.has_value()) {
*render_scale >> landmarks_render.In("RENDER_SCALE");
}
auto render_data = landmarks_render.Out("RENDER_DATA");
return render_data.Cast<mediapipe::RenderData>();
}
Stream<float> GetRenderScale(Stream<std::pair<int, int>> image_size,
Stream<NormalizedRect> roi, float multiplier,
Graph& graph) {
auto& to_render_scale = graph.AddNode("RectToRenderScaleCalculator");
to_render_scale.GetOptions<mediapipe::RectToRenderScaleCalculatorOptions>()
.set_multiplier(multiplier);
roi >> to_render_scale.In("NORM_RECT");
image_size >> to_render_scale.In("IMAGE_SIZE");
return to_render_scale.Out("RENDER_SCALE").Cast<float>();
}
Stream<mediapipe::RenderData> RenderRect(
Stream<NormalizedRect> rect,
const mediapipe::RectToRenderDataCalculatorOptions& renderer_options,
Graph& graph) {
auto& rect_render = graph.AddNode("RectToRenderDataCalculator");
rect_render.GetOptions<mediapipe::RectToRenderDataCalculatorOptions>()
.CopyFrom(renderer_options);
rect >> rect_render.In("NORM_RECT");
auto render_data = rect_render.Out("RENDER_DATA");
return render_data.Cast<mediapipe::RenderData>();
}
} // namespace mediapipe::tasks::vision::utils

69
mediapipe/tasks/cc/vision/utils/data_renderer.h Normal file
View File

@ -0,0 +1,69 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_DATA_RENDERER_H_
#define MEDIAPIPE_TASKS_CC_VISION_UTILS_DATA_RENDERER_H_
#include <optional>
#include <utility>
#include "absl/types/span.h"
#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h"
#include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/util/render_data.pb.h"
namespace mediapipe::tasks::vision::utils {
// Adds a node to the provided graph that renders the render_data_list on the
// given image, and returns the rendered image.
api2::builder::Stream<Image> Render(
api2::builder::Stream<Image> image,
absl::Span<api2::builder::Stream<mediapipe::RenderData>> render_data_list,
api2::builder::Graph& graph);
// Adds a node to the provided graph that infers the render scale from the image
// size and the object RoI. It will give you bigger rendered primitives for
// bigger/closer objects and smaller primitives for smaller/far objects. The
// primitives scale is proportional to `roi_size * multiplier`.
//
// See more details in
// mediapipe/calculators/util/rect_to_render_scale_calculator.cc
api2::builder::Stream<float> GetRenderScale(
api2::builder::Stream<std::pair<int, int>> image_size,
api2::builder::Stream<NormalizedRect> roi, float multiplier,
api2::builder::Graph& graph);
// Adds a node to the provided graph that gets the landmarks render data
// according to the renderer_options.
api2::builder::Stream<mediapipe::RenderData> RenderLandmarks(
api2::builder::Stream<mediapipe::NormalizedLandmarkList> landmarks,
std::optional<api2::builder::Stream<float>> render_scale,
const mediapipe::LandmarksToRenderDataCalculatorOptions& renderer_options,
api2::builder::Graph& graph);
// Adds a node to the provided graph that gets the rect render data according to
// the renderer_options.
api2::builder::Stream<mediapipe::RenderData> RenderRect(
api2::builder::Stream<NormalizedRect> rect,
const mediapipe::RectToRenderDataCalculatorOptions& renderer_options,
api2::builder::Graph& graph);
} // namespace mediapipe::tasks::vision::utils
#endif // MEDIAPIPE_TASKS_CC_VISION_UTILS_DATA_RENDERER_H_

133
mediapipe/tasks/cc/vision/utils/data_renderer_test.cc Normal file
View File

@ -0,0 +1,133 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/cc/vision/utils/data_renderer.h"
#include <utility>
#include <vector>
#include "absl/types/span.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/util/render_data.pb.h"
namespace mediapipe::tasks::vision::utils {
namespace {
using ::mediapipe::CalculatorGraphConfig;
using ::mediapipe::EqualsProto;
using ::mediapipe::NormalizedRect;
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::Stream;
TEST(DataRenderer, Render) {
Graph graph;
Stream<Image> image_in = graph.In("IMAGE").Cast<Image>();
Stream<RenderData> render_data_in =
graph.In("RENDER_DATA").Cast<RenderData>();
std::vector<Stream<RenderData>> render_data_list = {render_data_in};
Stream<Image> image_out =
Render(image_in, absl::Span<Stream<RenderData>>(render_data_list), graph);
image_out.SetName("image_out");
EXPECT_THAT(
graph.GetConfig(),
EqualsProto(mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "__stream_1"
input_stream: "UIMAGE:__stream_0"
output_stream: "UIMAGE:image_out"
}
input_stream: "IMAGE:__stream_0"
input_stream: "RENDER_DATA:__stream_1"
)pb")));
}
TEST(DataRenderer, RenderLandmarks) {
Graph graph;
Stream<NormalizedLandmarkList> rect =
graph.In("NORM_LANDMARKS").Cast<NormalizedLandmarkList>();
Stream<RenderData> render_data =
RenderLandmarks(rect, std::nullopt, {}, graph);
render_data.SetName("render_data");
EXPECT_THAT(
graph.GetConfig(),
EqualsProto(mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
node {
calculator: "LandmarksToRenderDataCalculator"
input_stream: "NORM_LANDMARKS:__stream_0"
output_stream: "RENDER_DATA:render_data"
options {
[mediapipe.LandmarksToRenderDataCalculatorOptions.ext] {}
}
}
input_stream: "NORM_LANDMARKS:__stream_0"
)pb")));
}
TEST(DataRenderer, GetRenderScale) {
Graph graph;
Stream<std::pair<int, int>> image_size =
graph.In("IMAGE_SIZE").Cast<std::pair<int, int>>();
Stream<NormalizedRect> roi = graph.In("ROI").Cast<NormalizedRect>();
Stream<float> render_scale = GetRenderScale(image_size, roi, 0.0001, graph);
render_scale.SetName("render_scale");
EXPECT_THAT(
graph.GetConfig(),
EqualsProto(mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
node {
calculator: "RectToRenderScaleCalculator"
input_stream: "IMAGE_SIZE:__stream_0"
input_stream: "NORM_RECT:__stream_1"
output_stream: "RENDER_SCALE:render_scale"
options {
[mediapipe.RectToRenderScaleCalculatorOptions.ext] {
multiplier: 0.0001
}
}
}
input_stream: "IMAGE_SIZE:__stream_0"
input_stream: "ROI:__stream_1"
)pb")));
}
TEST(DataRenderer, RenderRect) {
Graph graph;
Stream<NormalizedRect> rect = graph.In("NORM_RECT").Cast<NormalizedRect>();
Stream<RenderData> render_data = RenderRect(rect, {}, graph);
render_data.SetName("render_data");
EXPECT_THAT(
graph.GetConfig(),
EqualsProto(mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
node {
calculator: "RectToRenderDataCalculator"
input_stream: "NORM_RECT:__stream_0"
output_stream: "RENDER_DATA:render_data"
options {
[mediapipe.RectToRenderDataCalculatorOptions.ext] {}
}
}
input_stream: "NORM_RECT:__stream_0"
)pb")));
}
} // namespace
} // namespace mediapipe::tasks::vision::utils