diff --git a/mediapipe/model_maker/python/vision/face_stylizer/model_options.py b/mediapipe/model_maker/python/vision/face_stylizer/model_options.py index f63d4b962..54cb1a8cd 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/model_options.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/model_options.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/mediapipe/model_maker/python/vision/face_stylizer/model_spec_test.py b/mediapipe/model_maker/python/vision/face_stylizer/model_spec_test.py index e420adef4..aef684d5b 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/model_spec_test.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/model_spec_test.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. diff --git a/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc b/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc index d4057467b..7fab54834 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc +++ b/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc @@ -71,23 +71,21 @@ constexpr char kFaceLandmarksDetectorTFLiteName[] = "face_landmarks_detector.tflite"; constexpr char kFaceStylizerTFLiteName[] = "face_stylizer.tflite"; constexpr char kImageTag[] = "IMAGE"; -constexpr char kImageCpuTag[] = "IMAGE_CPU"; -constexpr char kImageGpuTag[] = "IMAGE_GPU"; constexpr char kImageSizeTag[] = "IMAGE_SIZE"; constexpr char kMatrixTag[] = "MATRIX"; constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; constexpr char kNormRectTag[] = "NORM_RECT"; -constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kSizeTag[] = "SIZE"; constexpr char kStylizedImageTag[] = "STYLIZED_IMAGE"; constexpr char kTensorsTag[] = "TENSORS"; -constexpr int kFaceAlignmentOutputSize = 256; +constexpr char kTransformationMatrixTag[] = "TRANSFORMATION_MATRIX"; // Struct holding the different output streams produced by the face stylizer // graph. struct FaceStylizerOutputStreams { std::optional> stylized_image; std::optional> face_alignment_image; + std::optional>> transformation_matrix; Source original_image; }; @@ -202,6 +200,10 @@ void ConfigureTensorsToImageCalculator( // The aligned face image that is fed to the face stylization model to // perform stylization. Also useful for preparing face stylization training // data. +// TRANSFORMATION_MATRIX - std::array +// An std::array representing a 4x4 row-major-order matrix that +// maps a point on the input image to a point on the output image, and +// can be used to reverse the mapping by inverting the matrix. // IMAGE - mediapipe::Image // The input image that the face landmarker runs on and has the pixel data // stored on the target storage (CPU vs GPU). @@ -280,6 +282,8 @@ class FaceStylizerGraph : public core::ModelTaskGraph { output_streams.face_alignment_image.value() >> graph[Output(kFaceAlignmentTag)]; } + output_streams.transformation_matrix.value() >> + graph[Output>(kTransformationMatrixTag)]; output_streams.original_image >> graph[Output(kImageTag)]; return graph.GetConfig(); } @@ -357,9 +361,10 @@ class FaceStylizerGraph : public core::ModelTaskGraph { image_to_tensor.GetOptions(); image_to_tensor_options.mutable_output_tensor_float_range()->set_min(0); image_to_tensor_options.mutable_output_tensor_float_range()->set_max(1); - image_to_tensor_options.set_output_tensor_width(kFaceAlignmentOutputSize); + image_to_tensor_options.set_output_tensor_width( + task_options.face_alignment_size()); image_to_tensor_options.set_output_tensor_height( - kFaceAlignmentOutputSize); + task_options.face_alignment_size()); image_to_tensor_options.set_keep_aspect_ratio(true); image_to_tensor_options.set_border_mode( mediapipe::ImageToTensorCalculatorOptions::BORDER_ZERO); @@ -378,6 +383,8 @@ class FaceStylizerGraph : public core::ModelTaskGraph { return {{/*stylized_image=*/std::nullopt, /*alignment_image=*/face_alignment, + /*transformation_matrix=*/ + image_to_tensor.Out(kMatrixTag).Cast>(), /*original_image=*/pass_through.Out("").Cast()}}; } @@ -439,6 +446,8 @@ class FaceStylizerGraph : public core::ModelTaskGraph { return {{/*stylized_image=*/stylized, /*alignment_image=*/face_alignment, + /*transformation_matrix=*/ + preprocessing.Out(kMatrixTag).Cast>(), /*original_image=*/preprocessing.Out(kImageTag).Cast()}}; } }; diff --git a/mediapipe/tasks/cc/vision/face_stylizer/proto/face_stylizer_graph_options.proto b/mediapipe/tasks/cc/vision/face_stylizer/proto/face_stylizer_graph_options.proto index 68abee43c..9528fab09 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/proto/face_stylizer_graph_options.proto +++ b/mediapipe/tasks/cc/vision/face_stylizer/proto/face_stylizer_graph_options.proto @@ -36,4 +36,7 @@ message FaceStylizerGraphOptions { // Options for face landmarker graph. optional vision.face_landmarker.proto.FaceLandmarkerGraphOptions face_landmarker_graph_options = 2; + + // The width and height of the output face alignment images. + optional int32 face_alignment_size = 3 [default = 256]; } diff --git a/mediapipe/tasks/examples/android/objectdetector/src/main/AndroidManifest.xml b/mediapipe/tasks/examples/android/objectdetector/src/main/AndroidManifest.xml deleted file mode 100644 index 5c53dc269..000000000 --- a/mediapipe/tasks/examples/android/objectdetector/src/main/AndroidManifest.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/tasks/examples/android/objectdetector/src/main/BUILD b/mediapipe/tasks/examples/android/objectdetector/src/main/BUILD deleted file mode 100644 index 28bd38f9f..000000000 --- a/mediapipe/tasks/examples/android/objectdetector/src/main/BUILD +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2022 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -licenses(["notice"]) - -package(default_visibility = ["//visibility:private"]) - -android_binary( - name = "objectdetector", - srcs = glob(["**/*.java"]), - assets = [ - "//mediapipe/tasks/testdata/vision:test_models", - ], - assets_dir = "", - custom_package = "com.google.mediapipe.tasks.examples.objectdetector", - manifest = "AndroidManifest.xml", - manifest_values = { - "applicationId": "com.google.mediapipe.tasks.examples.objectdetector", - }, - multidex = "native", - resource_files = ["//mediapipe/tasks/examples/android:resource_files"], - deps = [ - "//mediapipe/java/com/google/mediapipe/framework:android_framework", - "//mediapipe/java/com/google/mediapipe/framework/image", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:detection", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/vision:core", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/vision:objectdetector", - "//third_party:androidx_appcompat", - "//third_party:androidx_constraint_layout", - "//third_party:opencv", - "@maven//:androidx_activity_activity", - "@maven//:androidx_concurrent_concurrent_futures", - "@maven//:androidx_exifinterface_exifinterface", - "@maven//:androidx_fragment_fragment", - "@maven//:com_google_guava_guava", - ], -) diff --git a/mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/MainActivity.java b/mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/MainActivity.java deleted file mode 100644 index abfd7c7d1..000000000 --- a/mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/MainActivity.java +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2022 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.tasks.examples.objectdetector; - -import android.content.Intent; -import android.graphics.Bitmap; -import android.media.MediaMetadataRetriever; -import android.os.Bundle; -import android.provider.MediaStore; -import androidx.appcompat.app.AppCompatActivity; -import android.util.Log; -import android.view.View; -import android.widget.Button; -import android.widget.FrameLayout; -import androidx.activity.result.ActivityResultLauncher; -import androidx.activity.result.contract.ActivityResultContracts; -import androidx.exifinterface.media.ExifInterface; -// ContentResolver dependency -import com.google.mediapipe.framework.MediaPipeException; -import com.google.mediapipe.framework.image.BitmapImageBuilder; -import com.google.mediapipe.framework.image.MPImage; -import com.google.mediapipe.tasks.core.BaseOptions; -import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions; -import com.google.mediapipe.tasks.vision.core.RunningMode; -import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectionResult; -import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector; -import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetector.ObjectDetectorOptions; -import java.io.IOException; -import java.io.InputStream; - -/** Main activity of MediaPipe Task Object Detector reference app. */ -public class MainActivity extends AppCompatActivity { - private static final String TAG = "MainActivity"; - private static final String MODEL_FILE = "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite"; - - private ObjectDetector objectDetector; - - private enum InputSource { - UNKNOWN, - IMAGE, - VIDEO, - CAMERA, - } - - private InputSource inputSource = InputSource.UNKNOWN; - - // Image mode demo component. - private ActivityResultLauncher imageGetter; - // Video mode demo component. - private ActivityResultLauncher videoGetter; - private ObjectDetectionResultImageView imageView; - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - setContentView(R.layout.activity_main); - setupImageModeDemo(); - setupVideoModeDemo(); - // TODO: Adds live camera demo. - } - - /** Sets up the image mode demo. */ - private void setupImageModeDemo() { - imageView = new ObjectDetectionResultImageView(this); - // The Intent to access gallery and read images as bitmap. - imageGetter = - registerForActivityResult( - new ActivityResultContracts.StartActivityForResult(), - result -> { - Intent resultIntent = result.getData(); - if (resultIntent != null) { - if (result.getResultCode() == RESULT_OK) { - Bitmap bitmap = null; - int rotation = 0; - try { - bitmap = - downscaleBitmap( - MediaStore.Images.Media.getBitmap( - this.getContentResolver(), resultIntent.getData())); - } catch (IOException e) { - Log.e(TAG, "Bitmap reading error:" + e); - } - try { - InputStream imageData = - this.getContentResolver().openInputStream(resultIntent.getData()); - rotation = getImageRotation(imageData); - } catch (IOException | MediaPipeException e) { - Log.e(TAG, "Bitmap rotation error:" + e); - } - if (bitmap != null) { - MPImage image = new BitmapImageBuilder(bitmap).build(); - ObjectDetectionResult detectionResult = - objectDetector.detect( - image, - ImageProcessingOptions.builder().setRotationDegrees(rotation).build()); - imageView.setData(image, detectionResult); - runOnUiThread(() -> imageView.update()); - } - } - } - }); - Button loadImageButton = findViewById(R.id.button_load_picture); - loadImageButton.setOnClickListener( - v -> { - if (inputSource != InputSource.IMAGE) { - createObjectDetector(RunningMode.IMAGE); - this.inputSource = InputSource.IMAGE; - updateLayout(); - } - // Reads images from gallery. - Intent pickImageIntent = new Intent(Intent.ACTION_PICK); - pickImageIntent.setDataAndType(MediaStore.Images.Media.INTERNAL_CONTENT_URI, "image/*"); - imageGetter.launch(pickImageIntent); - }); - } - - /** Sets up the video mode demo. */ - private void setupVideoModeDemo() { - imageView = new ObjectDetectionResultImageView(this); - // The Intent to access gallery and read a video file. - videoGetter = - registerForActivityResult( - new ActivityResultContracts.StartActivityForResult(), - result -> { - Intent resultIntent = result.getData(); - if (resultIntent != null) { - if (result.getResultCode() == RESULT_OK) { - MediaMetadataRetriever metaRetriever = new MediaMetadataRetriever(); - metaRetriever.setDataSource(this, resultIntent.getData()); - long duration = - Long.parseLong( - metaRetriever.extractMetadata( - MediaMetadataRetriever.METADATA_KEY_DURATION)); - int numFrames = - Integer.parseInt( - metaRetriever.extractMetadata( - MediaMetadataRetriever.METADATA_KEY_VIDEO_FRAME_COUNT)); - long frameIntervalMs = duration / numFrames; - for (int i = 0; i < numFrames; ++i) { - MPImage image = - new BitmapImageBuilder(metaRetriever.getFrameAtIndex(i)).build(); - ObjectDetectionResult detectionResult = - objectDetector.detectForVideo(image, frameIntervalMs * i); - // Currently only annotates the detection result on the first video frame and - // display it to verify the correctness. - // TODO: Annotates the detection result on every frame, save the - // annotated frames as a video file, and play back the video afterwards. - if (i == 0) { - imageView.setData(image, detectionResult); - runOnUiThread(() -> imageView.update()); - } - } - } - } - }); - Button loadVideoButton = findViewById(R.id.button_load_video); - loadVideoButton.setOnClickListener( - v -> { - createObjectDetector(RunningMode.VIDEO); - updateLayout(); - this.inputSource = InputSource.VIDEO; - - // Reads a video from gallery. - Intent pickVideoIntent = new Intent(Intent.ACTION_PICK); - pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*"); - videoGetter.launch(pickVideoIntent); - }); - } - - private void createObjectDetector(RunningMode mode) { - if (objectDetector != null) { - objectDetector.close(); - } - // Initializes a new MediaPipe ObjectDetector instance - ObjectDetectorOptions options = - ObjectDetectorOptions.builder() - .setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build()) - .setScoreThreshold(0.5f) - .setMaxResults(5) - .setRunningMode(mode) - .build(); - objectDetector = ObjectDetector.createFromOptions(this, options); - } - - private void updateLayout() { - // Updates the preview layout. - FrameLayout frameLayout = findViewById(R.id.preview_display_layout); - frameLayout.removeAllViewsInLayout(); - imageView.setImageDrawable(null); - frameLayout.addView(imageView); - imageView.setVisibility(View.VISIBLE); - } - - private Bitmap downscaleBitmap(Bitmap originalBitmap) { - double aspectRatio = (double) originalBitmap.getWidth() / originalBitmap.getHeight(); - int width = imageView.getWidth(); - int height = imageView.getHeight(); - if (((double) imageView.getWidth() / imageView.getHeight()) > aspectRatio) { - width = (int) (height * aspectRatio); - } else { - height = (int) (width / aspectRatio); - } - return Bitmap.createScaledBitmap(originalBitmap, width, height, false); - } - - private int getImageRotation(InputStream imageData) throws IOException, MediaPipeException { - int orientation = - new ExifInterface(imageData) - .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); - switch (orientation) { - case ExifInterface.ORIENTATION_NORMAL: - return 0; - case ExifInterface.ORIENTATION_ROTATE_90: - return 90; - case ExifInterface.ORIENTATION_ROTATE_180: - return 180; - case ExifInterface.ORIENTATION_ROTATE_270: - return 270; - default: - // TODO: use getRotationDegrees() and isFlipped() instead of switch once flip - // is supported. - throw new MediaPipeException( - MediaPipeException.StatusCode.UNIMPLEMENTED.ordinal(), - "Flipped images are not supported yet."); - } - } -} diff --git a/mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/ObjectDetectionResultImageView.java b/mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/ObjectDetectionResultImageView.java deleted file mode 100644 index e37c64156..000000000 --- a/mediapipe/tasks/examples/android/objectdetector/src/main/java/com/google/mediapipe/tasks/examples/objectdetector/ObjectDetectionResultImageView.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2022 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.mediapipe.tasks.examples.objectdetector; - -import android.content.Context; -import android.graphics.Bitmap; -import android.graphics.Canvas; -import android.graphics.Color; -import android.graphics.Matrix; -import android.graphics.Paint; -import androidx.appcompat.widget.AppCompatImageView; -import com.google.mediapipe.framework.image.BitmapExtractor; -import com.google.mediapipe.framework.image.MPImage; -import com.google.mediapipe.tasks.components.containers.Detection; -import com.google.mediapipe.tasks.vision.objectdetector.ObjectDetectionResult; - -/** An ImageView implementation for displaying {@link ObjectDetectionResult}. */ -public class ObjectDetectionResultImageView extends AppCompatImageView { - private static final String TAG = "ObjectDetectionResultImageView"; - - private static final int BBOX_COLOR = Color.GREEN; - private static final int BBOX_THICKNESS = 5; // Pixels - private Bitmap latest; - - public ObjectDetectionResultImageView(Context context) { - super(context); - setScaleType(AppCompatImageView.ScaleType.FIT_CENTER); - } - - /** - * Sets a {@link MPImage} and an {@link ObjectDetectionResult} to render. - * - * @param image a {@link MPImage} object for annotation. - * @param result an {@link ObjectDetectionResult} object that contains the detection result. - */ - public void setData(MPImage image, ObjectDetectionResult result) { - if (image == null || result == null) { - return; - } - latest = BitmapExtractor.extract(image); - Canvas canvas = new Canvas(latest); - canvas.drawBitmap(latest, new Matrix(), null); - for (int i = 0; i < result.detections().size(); ++i) { - drawDetectionOnCanvas(result.detections().get(i), canvas); - } - } - - /** Updates the image view with the latest {@link ObjectDetectionResult}. */ - public void update() { - postInvalidate(); - if (latest != null) { - setImageBitmap(latest); - } - } - - private void drawDetectionOnCanvas(Detection detection, Canvas canvas) { - // TODO: Draws the category and the score per bounding box. - // Draws bounding box. - Paint bboxPaint = new Paint(); - bboxPaint.setColor(BBOX_COLOR); - bboxPaint.setStyle(Paint.Style.STROKE); - bboxPaint.setStrokeWidth(BBOX_THICKNESS); - canvas.drawRect(detection.boundingBox(), bboxPaint); - } -} diff --git a/mediapipe/tasks/examples/android/res/drawable-v24/ic_launcher_foreground.xml b/mediapipe/tasks/examples/android/res/drawable-v24/ic_launcher_foreground.xml deleted file mode 100644 index c7bd21dbd..000000000 --- a/mediapipe/tasks/examples/android/res/drawable-v24/ic_launcher_foreground.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - diff --git a/mediapipe/tasks/examples/android/res/drawable/ic_launcher_background.xml b/mediapipe/tasks/examples/android/res/drawable/ic_launcher_background.xml deleted file mode 100644 index 01f0af0ad..000000000 --- a/mediapipe/tasks/examples/android/res/drawable/ic_launcher_background.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mediapipe/tasks/examples/android/res/layout/activity_main.xml b/mediapipe/tasks/examples/android/res/layout/activity_main.xml deleted file mode 100644 index 834e9a3e6..000000000 --- a/mediapipe/tasks/examples/android/res/layout/activity_main.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - -