cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update ImageFrameToGpuBufferCalculator to use api2 and GpuBuffer conversions

Browse Source 
PiperOrigin-RevId: 490374387
This commit is contained in:
Camillo Lugaresi 2022-11-22 17:25:55 -08:00 committed by Copybara-Service
parent fac97554df
commit 8ba9d87e66
2 changed files with 28 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mediapipe/gpu/BUILD
View File

@ -901,6 +901,8 @@ cc_library(
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
deps = [ deps = [
":gl_calculator_helper", ":gl_calculator_helper",
":gpu_buffer_storage_image_frame",
"//mediapipe/framework/api2:node",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",

62
mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc
View File

@ -12,73 +12,63 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_calculator_helper.h"
#ifdef __APPLE__
#include "mediapipe/objc/util.h"
#endif
namespace mediapipe { namespace mediapipe {
namespace api2 {
// Convert ImageFrame to GpuBuffer. class ImageFrameToGpuBufferCalculator
class ImageFrameToGpuBufferCalculator : public CalculatorBase { : public RegisteredNode<ImageFrameToGpuBufferCalculator> {
public: public:
ImageFrameToGpuBufferCalculator() {} static constexpr Input<ImageFrame> kIn{""};
static constexpr Output<GpuBuffer> kOut{""};
static absl::Status GetContract(CalculatorContract* cc); MEDIAPIPE_NODE_INTERFACE(ImageFrameToGpuBufferCalculator, kIn, kOut);
static absl::Status UpdateContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override; absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override; absl::Status Process(CalculatorContext* cc) override;
private: private:
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlCalculatorHelper helper_; GlCalculatorHelper helper_;
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}; };
REGISTER_CALCULATOR(ImageFrameToGpuBufferCalculator);
// static // static
absl::Status ImageFrameToGpuBufferCalculator::GetContract( absl::Status ImageFrameToGpuBufferCalculator::UpdateContract(
CalculatorContract* cc) { CalculatorContract* cc) {
cc->Inputs().Index(0).Set<ImageFrame>();
cc->Outputs().Index(0).Set<GpuBuffer>();
// Note: we call this method even on platforms where we don't use the helper, // Note: we call this method even on platforms where we don't use the helper,
// to ensure the calculator's contract is the same. In particular, the helper // to ensure the calculator's contract is the same. In particular, the helper
// enables support for the legacy side packet, which several graphs still use. // enables support for the legacy side packet, which several graphs still use.
MP_RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc)); return GlCalculatorHelper::UpdateContract(cc);
return absl::OkStatus();
} }
absl::Status ImageFrameToGpuBufferCalculator::Open(CalculatorContext* cc) { absl::Status ImageFrameToGpuBufferCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(TimestampDiff(0));
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
MP_RETURN_IF_ERROR(helper_.Open(cc)); MP_RETURN_IF_ERROR(helper_.Open(cc));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) { absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER auto image_frame = std::const_pointer_cast<ImageFrame>(
CFHolder<CVPixelBufferRef> buffer; mediapipe::SharedPtrWithPacket<ImageFrame>(kIn(cc).packet()));
MP_RETURN_IF_ERROR(CreateCVPixelBufferForImageFramePacket( auto gpu_buffer = api2::MakePacket<GpuBuffer>(
cc->Inputs().Index(0).Value(), &buffer)); std::make_shared<mediapipe::GpuBufferStorageImageFrame>(
cc->Outputs().Index(0).Add(new GpuBuffer(buffer), cc->InputTimestamp()); std::move(image_frame)))
#else .At(cc->InputTimestamp());
const auto& input = cc->Inputs().Index(0).Get<ImageFrame>(); // This calculator's behavior has been to do the texture upload eagerly, and
helper_.RunInGlContext([this, &input, &cc]() { // some graphs may rely on running this on a separate GL context to avoid
auto src = helper_.CreateSourceTexture(input); // blocking another context with the read operation. So let's request GPU
auto output = src.GetFrame<GpuBuffer>(); // access here to ensure that the behavior stays the same.
glFlush(); // TODO: have a better way to do this, or defer until later.
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); helper_.RunInGlContext(
src.Release(); [&gpu_buffer] { auto view = gpu_buffer->GetReadView<GlTextureView>(0); });
}); kOut(cc).Send(std::move(gpu_buffer));
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return absl::OkStatus(); return absl::OkStatus();
} }
} // namespace api2
} // namespace mediapipe } // namespace mediapipe