cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix incorrect rotation handling in C++ vision tasks

Browse Source 
PiperOrigin-RevId: 520670536
This commit is contained in:
MediaPipe Team 2023-03-30 10:02:58 -07:00 committed by Copybara-Service
parent 0e951b8add
commit d7fd5b0cf5
15 changed files with 93 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mediapipe/tasks/cc/vision/core/BUILD
View File

@ -39,6 +39,7 @@ cc_library(
":running_mode",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/tasks/cc/components/containers:rect",
"//mediapipe/tasks/cc/core:base_task_api",

20
mediapipe/tasks/cc/vision/core/base_vision_task_api.h
View File

@ -17,6 +17,7 @@ limitations under the License.
#define MEDIAPIPE_TASKS_CC_VISION_CORE_BASE_VISION_TASK_API_H_
#include <cmath>
#include <cstdlib>
#include <memory>
#include <optional>
#include <string>
@ -26,6 +27,7 @@ limitations under the License.
#include "absl/status/statusor.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/tasks/cc/components/containers/rect.h"
#include "mediapipe/tasks/cc/core/base_task_api.h"
@ -136,7 +138,8 @@ class BaseVisionTaskApi : public tasks::core::BaseTaskApi {
// to 0. If 'roi_allowed' is false, an error will be returned if the input
// ImageProcessingOptions has its 'region_or_interest' field set.
static absl::StatusOr<mediapipe::NormalizedRect> ConvertToNormalizedRect(
std::optional<ImageProcessingOptions> options, bool roi_allowed = true) {
std::optional<ImageProcessingOptions> options,
const mediapipe::Image& image, bool roi_allowed = true) {
mediapipe::NormalizedRect normalized_rect;
normalized_rect.set_rotation(0);
normalized_rect.set_x_center(0.5);
@ -181,6 +184,21 @@ class BaseVisionTaskApi : public tasks::core::BaseTaskApi {
normalized_rect.set_width(roi.right - roi.left);
normalized_rect.set_height(roi.bottom - roi.top);
}
// For 90° and 270° rotations, we need to swap width and height.
// This is due to the internal behavior of ImageToTensorCalculator, which:
// - first denormalizes the provided rect by multiplying the rect width or
// height by the image width or height, repectively.
// - then rotates this by denormalized rect by the provided rotation, and
// uses this for cropping,
// - then finally rotates this back.
if (std::abs(options->rotation_degrees) % 180 != 0) {
float w = normalized_rect.height() * image.height() / image.width();
float h = normalized_rect.width() * image.width() / image.height();
normalized_rect.set_width(w);
normalized_rect.set_height(h);
}
return normalized_rect;
}

18
mediapipe/tasks/cc/vision/face_detector/face_detector.cc
View File

@ -136,9 +136,9 @@ absl::StatusOr<std::unique_ptr<FaceDetector>> FaceDetector::Create(
absl::StatusOr<FaceDetectorResult> FaceDetector::Detect(
mediapipe::Image image,
std::optional<core::ImageProcessingOptions> image_processing_options) {
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -156,9 +156,9 @@ absl::StatusOr<FaceDetectorResult> FaceDetector::Detect(
absl::StatusOr<FaceDetectorResult> FaceDetector::DetectForVideo(
mediapipe::Image image, uint64_t timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -179,9 +179,9 @@ absl::StatusOr<FaceDetectorResult> FaceDetector::DetectForVideo(
absl::Status FaceDetector::DetectAsync(
mediapipe::Image image, uint64_t timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

6
mediapipe/tasks/cc/vision/face_landmarker/face_landmarker.cc
View File

@ -194,7 +194,7 @@ absl::StatusOr<FaceLandmarkerResult> FaceLandmarker::Detect(
mediapipe::Image image,
std::optional<core::ImageProcessingOptions> image_processing_options) {
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
@ -212,7 +212,7 @@ absl::StatusOr<FaceLandmarkerResult> FaceLandmarker::DetectForVideo(
mediapipe::Image image, int64_t timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
@ -233,7 +233,7 @@ absl::Status FaceLandmarker::DetectAsync(
mediapipe::Image image, int64_t timestamp_ms,
std::optional<core::ImageProcessingOptions> image_processing_options) {
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,

6
mediapipe/tasks/cc/vision/face_stylizer/face_stylizer.cc
View File

@ -138,7 +138,7 @@ absl::StatusOr<Image> FaceStylizer::Stylize(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -157,7 +157,7 @@ absl::StatusOr<Image> FaceStylizer::StylizeForVideo(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -180,7 +180,7 @@ absl::Status FaceStylizer::StylizeAsync(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

18
mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.cc
View File

@ -222,9 +222,9 @@ absl::StatusOr<GestureRecognizerResult> GestureRecognizer::Recognize(
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -258,9 +258,9 @@ absl::StatusOr<GestureRecognizerResult> GestureRecognizer::RecognizeForVideo(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -297,9 +297,9 @@ absl::Status GestureRecognizer::RecognizeAsync(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

18
mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker.cc
View File

@ -185,9 +185,9 @@ absl::StatusOr<HandLandmarkerResult> HandLandmarker::Detect(
"GPU input images are currently not supported.",
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -223,9 +223,9 @@ absl::StatusOr<HandLandmarkerResult> HandLandmarker::DetectForVideo(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -264,9 +264,9 @@ absl::Status HandLandmarker::DetectAsync(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

6
mediapipe/tasks/cc/vision/image_classifier/image_classifier.cc
View File

@ -156,7 +156,7 @@ absl::StatusOr<ImageClassifierResult> ImageClassifier::Classify(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -176,7 +176,7 @@ absl::StatusOr<ImageClassifierResult> ImageClassifier::ClassifyForVideo(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -200,7 +200,7 @@ absl::Status ImageClassifier::ClassifyAsync(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

19
mediapipe/tasks/cc/vision/image_classifier/image_classifier_test.cc
View File

@ -505,11 +505,9 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
ImageClassifierResult expected;
expected.classifications.emplace_back(Classifications{
/*categories=*/{
{/*index=*/934, /*score=*/0.6371766,
/*category_name=*/"cheeseburger"},
{/*index=*/963, /*score=*/0.049443405, /*category_name=*/"meat loaf"},
{/*index=*/925, /*score=*/0.047918003,
/*category_name=*/"guacamole"}},
{/*index=*/934, /*score=*/0.754467, /*category_name=*/"cheeseburger"},
{/*index=*/925, /*score=*/0.0288028, /*category_name=*/"guacamole"},
{/*index=*/932, /*score=*/0.0286119, /*category_name=*/"bagel"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);
@ -525,9 +523,10 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterestAndRotation) {
options->classifier_options.max_results = 1;
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ImageClassifier> image_classifier,
ImageClassifier::Create(std::move(options)));
// Region-of-interest around the chair, with 90° anti-clockwise rotation.
RectF roi{/*left=*/0.006, /*top=*/0.1763, /*right=*/0.5702,
/*bottom=*/0.3049};
// Region-of-interest around the soccer ball, with 90° anti-clockwise
// rotation.
RectF roi{/*left=*/0.2655, /*top=*/0.45, /*right=*/0.6925,
/*bottom=*/0.614};
ImageProcessingOptions image_processing_options{roi,
/*rotation_degrees=*/-90};
@ -536,8 +535,8 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterestAndRotation) {
ImageClassifierResult expected;
expected.classifications.emplace_back(
Classifications{/*categories=*/{{/*index=*/560, /*score=*/0.6522213,
/*category_name=*/"folding chair"}},
Classifications{/*categories=*/{{/*index=*/806, /*score=*/0.997684,
/*category_name=*/"soccer ball"}},
/*head_index=*/0,
/*head_name=*/"probability"});
ExpectApproximatelyEqual(results, expected);

6
mediapipe/tasks/cc/vision/image_embedder/image_embedder.cc
View File

@ -151,7 +151,7 @@ absl::StatusOr<ImageEmbedderResult> ImageEmbedder::Embed(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -172,7 +172,7 @@ absl::StatusOr<ImageEmbedderResult> ImageEmbedder::EmbedForVideo(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -196,7 +196,7 @@ absl::Status ImageEmbedder::EmbedAsync(
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options));
ConvertToNormalizedRect(image_processing_options, image));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

4
mediapipe/tasks/cc/vision/image_embedder/image_embedder_test.cc
View File

@ -371,7 +371,7 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
MP_ASSERT_OK_AND_ASSIGN(double similarity, ImageEmbedder::CosineSimilarity(
image_result.embeddings[0],
rotated_result.embeddings[0]));
double expected_similarity = 0.572265;
double expected_similarity = 0.98223;
EXPECT_LE(abs(similarity - expected_similarity), kSimilarityTolerancy);
}
@ -406,7 +406,7 @@ TEST_F(ImageModeTest, SucceedsWithRegionOfInterestAndRotation) {
MP_ASSERT_OK_AND_ASSIGN(double similarity, ImageEmbedder::CosineSimilarity(
crop_result.embeddings[0],
rotated_result.embeddings[0]));
double expected_similarity = 0.62838;
double expected_similarity = 0.974683;
EXPECT_LE(abs(similarity - expected_similarity), kSimilarityTolerancy);
}

18
mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.cc
View File

@ -192,9 +192,9 @@ absl::StatusOr<std::vector<Image>> ImageSegmenter::Segment(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -213,9 +213,9 @@ absl::StatusOr<std::vector<Image>> ImageSegmenter::SegmentForVideo(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -237,9 +237,9 @@ absl::Status ImageSegmenter::SegmentAsync(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

6
mediapipe/tasks/cc/vision/interactive_segmenter/interactive_segmenter.cc
View File

@ -142,9 +142,9 @@ absl::StatusOr<std::vector<Image>> InteractiveSegmenter::Segment(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(RenderData roi_as_render_data, ConvertRoiToRenderData(roi));
ASSIGN_OR_RETURN(
auto output_packets,

18
mediapipe/tasks/cc/vision/object_detector/object_detector.cc
View File

@ -157,9 +157,9 @@ absl::StatusOr<ObjectDetectorResult> ObjectDetector::Detect(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessImageData(
@ -178,9 +178,9 @@ absl::StatusOr<ObjectDetectorResult> ObjectDetector::DetectForVideo(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
ASSIGN_OR_RETURN(
auto output_packets,
ProcessVideoData(
@ -203,9 +203,9 @@ absl::Status ObjectDetector::DetectAsync(
absl::StrCat("GPU input images are currently not supported."),
MediaPipeTasksStatus::kRunnerUnexpectedInputError);
}
ASSIGN_OR_RETURN(
NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, /*roi_allowed=*/false));
ASSIGN_OR_RETURN(NormalizedRect norm_rect,
ConvertToNormalizedRect(image_processing_options, image,
/*roi_allowed=*/false));
return SendLiveStreamData(
{{kImageInStreamName,
MakePacket<Image>(std::move(image))

5
mediapipe/tasks/cc/vision/object_detector/object_detector_test.cc
View File

@ -575,7 +575,6 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
"cats_and_dogs_rotated.jpg")));
auto options = std::make_unique<ObjectDetectorOptions>();
options->max_results = 1;
options->category_allowlist.push_back("cat");
options->base_options.model_asset_path =
JoinPath("./", kTestDataDirectory, kMobileSsdWithMetadata);
MP_ASSERT_OK_AND_ASSIGN(std::unique_ptr<ObjectDetector> object_detector,
@ -589,10 +588,10 @@ TEST_F(ImageModeTest, SucceedsWithRotation) {
results,
ConvertToDetectionResult({ParseTextProtoOrDie<DetectionProto>(R"pb(
label: "cat"
score: 0.7109375
score: 0.69921875
location_data {
format: BOUNDING_BOX
bounding_box { xmin: 0 ymin: 622 width: 436 height: 276 }
bounding_box { xmin: 0 ymin: 608 width: 439 height: 387 }
})pb")}));
}