cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update ImageFrameToGpuBufferCalculator to use api2 and GpuBuffer conversions

Browse Source 
PiperOrigin-RevId: 515407159
This commit is contained in:
MediaPipe Team 2023-03-09 12:18:16 -08:00 committed by Copybara-Service
parent 5daf58009d
commit c2a69ab476
2 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mediapipe/gpu/BUILD
View File

@ -918,8 +918,6 @@ cc_library(
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
":gl_calculator_helper", ":gl_calculator_helper",
":gpu_buffer_storage_image_frame",
"//mediapipe/framework/api2:node",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",

62
mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc
View File

@ -12,63 +12,73 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_calculator_helper.h"
#ifdef __APPLE__
#include "mediapipe/objc/util.h"
#endif
namespace mediapipe { namespace mediapipe {
namespace api2 {
class ImageFrameToGpuBufferCalculator // Convert ImageFrame to GpuBuffer.
: public RegisteredNode<ImageFrameToGpuBufferCalculator> { class ImageFrameToGpuBufferCalculator : public CalculatorBase {
public: public:
static constexpr Input<ImageFrame> kIn{""}; ImageFrameToGpuBufferCalculator() {}
static constexpr Output<GpuBuffer> kOut{""};
MEDIAPIPE_NODE_INTERFACE(ImageFrameToGpuBufferCalculator, kIn, kOut); static absl::Status GetContract(CalculatorContract* cc);
static absl::Status UpdateContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override; absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override; absl::Status Process(CalculatorContext* cc) override;
private: private:
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlCalculatorHelper helper_; GlCalculatorHelper helper_;
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}; };
REGISTER_CALCULATOR(ImageFrameToGpuBufferCalculator);
// static // static
absl::Status ImageFrameToGpuBufferCalculator::UpdateContract( absl::Status ImageFrameToGpuBufferCalculator::GetContract(
CalculatorContract* cc) { CalculatorContract* cc) {
cc->Inputs().Index(0).Set<ImageFrame>();
cc->Outputs().Index(0).Set<GpuBuffer>();
// Note: we call this method even on platforms where we don't use the helper, // Note: we call this method even on platforms where we don't use the helper,
// to ensure the calculator's contract is the same. In particular, the helper // to ensure the calculator's contract is the same. In particular, the helper
// enables support for the legacy side packet, which several graphs still use. // enables support for the legacy side packet, which several graphs still use.
return GlCalculatorHelper::UpdateContract(cc); MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
return absl::OkStatus();
} }
absl::Status ImageFrameToGpuBufferCalculator::Open(CalculatorContext* cc) { absl::Status ImageFrameToGpuBufferCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(TimestampDiff(0));
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
MP_RETURN_IF_ERROR(helper_.Open(cc)); MP_RETURN_IF_ERROR(helper_.Open(cc));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) { absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) {
auto image_frame = std::const_pointer_cast<ImageFrame>( #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
mediapipe::SharedPtrWithPacket<ImageFrame>(kIn(cc).packet())); CFHolder<CVPixelBufferRef> buffer;
auto gpu_buffer = api2::MakePacket<GpuBuffer>( MP_RETURN_IF_ERROR(CreateCVPixelBufferForImageFramePacket(
std::make_shared<mediapipe::GpuBufferStorageImageFrame>( cc->Inputs().Index(0).Value(), &buffer));
std::move(image_frame))) cc->Outputs().Index(0).Add(new GpuBuffer(buffer), cc->InputTimestamp());
.At(cc->InputTimestamp()); #else
// This calculator's behavior has been to do the texture upload eagerly, and const auto& input = cc->Inputs().Index(0).Get<ImageFrame>();
// some graphs may rely on running this on a separate GL context to avoid helper_.RunInGlContext([this, &input, &cc]() {
// blocking another context with the read operation. So let's request GPU auto src = helper_.CreateSourceTexture(input);
// access here to ensure that the behavior stays the same. auto output = src.GetFrame<GpuBuffer>();
// TODO: have a better way to do this, or defer until later. glFlush();
helper_.RunInGlContext( cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
[&gpu_buffer] { auto view = gpu_buffer->GetReadView<GlTextureView>(0); }); src.Release();
kOut(cc).Send(std::move(gpu_buffer)); });
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return absl::OkStatus(); return absl::OkStatus();
} }
} // namespace api2
} // namespace mediapipe } // namespace mediapipe