diff --git a/.vscode/settings.json b/.vscode/settings.json index e9f3a3a21..478a3c863 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,54 @@ { "files.associations": { "chrono": "cpp", - "ios": "cpp" + "ios": "cpp", + "olagraph.hpp": "c", + "face_mesh_module.h": "c", + "array": "cpp", + "atomic": "cpp", + "bit": "cpp", + "*.tcc": "cpp", + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "compare": "cpp", + "concepts": "cpp", + "cstddef": "cpp", + "cstdint": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "deque": "cpp", + "forward_list": "cpp", + "list": "cpp", + "map": "cpp", + "set": "cpp", + "unordered_map": "cpp", + "unordered_set": "cpp", + "vector": "cpp", + "exception": "cpp", + "functional": "cpp", + "initializer_list": "cpp", + "iosfwd": "cpp", + "istream": "cpp", + "limits": "cpp", + "memory": "cpp", + "new": "cpp", + "numbers": "cpp", + "ostream": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "type_traits": "cpp", + "tuple": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "utility": "cpp", + "variant": "cpp" } } \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index 36b0d33e5..3231e0d76 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -421,5 +421,5 @@ cc_crosstool(name = "crosstool") # android_ndk_repository(name = "androidndk", api_level=21, path = "/Users/wangrenzhu/Library/Android/sdk/ndk/21.2.6472646") # android_sdk_repository(name = "androidsdk", path = "/Users/wangrenzhu/Android/sdk") # android_ndk_repository(name = "androidndk", api_level=21, path = "/Users/wangrenzhu/Android/sdk/ndk/android-ndk-r21") -android_sdk_repository(name = "androidsdk") -android_ndk_repository(name = "androidndk", api_level=22) +# android_sdk_repository(name = "androidsdk") +# android_ndk_repository(name = "androidndk", api_level=22) diff --git a/WORKSPACE-e b/WORKSPACE-e index 0f3a6acd4..09d917dae 100644 --- a/WORKSPACE-e +++ b/WORKSPACE-e @@ -417,7 +417,9 @@ libedgetpu_dependencies() load("@coral_crosstool//:configure.bzl", "cc_crosstool") cc_crosstool(name = "crosstool") -# android_sdk_repository(name = "androidsdk", path = "/Users/wangrenzhu/Library/Android/sdk") -# android_ndk_repository(name = "androidndk", api_level=21, path = "/Users/wangrenzhu/Library/Android/sdk/ndk/21.2.6472646") +android_sdk_repository(name = "androidsdk", path = "/Users/wangrenzhu/Library/Android/sdk") +android_ndk_repository(name = "androidndk", api_level=21, path = "/Users/wangrenzhu/Library/Android/sdk/ndk/21.2.6472646") # android_sdk_repository(name = "androidsdk", path = "/Users/wangrenzhu/Android/sdk") # android_ndk_repository(name = "androidndk", api_level=21, path = "/Users/wangrenzhu/Android/sdk/ndk/android-ndk-r21") +# android_sdk_repository(name = "androidsdk", path = "/Users/wangrenzhu/Android/sdk") +# android_ndk_repository(name = "androidndk", api_level=21, path = "/Users/wangrenzhu/Android/sdk/ndk/android-ndk-r21") \ No newline at end of file diff --git a/mediapipe/render/module/beauty/BUILD b/mediapipe/render/module/beauty/BUILD index dd7011129..0cbfd160b 100644 --- a/mediapipe/render/module/beauty/BUILD +++ b/mediapipe/render/module/beauty/BUILD @@ -1,11 +1,18 @@ +package(default_visibility = ["//visibility:public"]) cc_library( name = "FaceMeshGPULibrary", copts = ["-std=c++17"], srcs = [ - "face_mesh_module.h", + "face_mesh_module.cc", + "face_mesh_beauty_render.cc", + "face_mesh_module_imp.cc", ], - hdrs = ["face_mesh_module.cpp"], + hdrs = [ + "face_mesh_module.h", + "face_mesh_beauty_render.h", + "face_mesh_module_imp.h", + ], data = [ "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb", "//mediapipe/modules/face_detection:face_detection_short_range.tflite", @@ -23,4 +30,11 @@ cc_library( "//mediapipe/framework/formats:landmark_cc_proto", ], }), + copts = select({ + "//mediapipe:apple": [ + "-x objective-c++", + "-fobjc-arc", # enable reference-counting + ], + "//conditions:default": [], + }), ) diff --git a/mediapipe/render/module/beauty/face_mesh_beauty_render.cpp b/mediapipe/render/module/beauty/face_mesh_beauty_render.cc similarity index 100% rename from mediapipe/render/module/beauty/face_mesh_beauty_render.cpp rename to mediapipe/render/module/beauty/face_mesh_beauty_render.cc diff --git a/mediapipe/render/module/beauty/face_mesh_beauty_render.hpp b/mediapipe/render/module/beauty/face_mesh_beauty_render.h similarity index 100% rename from mediapipe/render/module/beauty/face_mesh_beauty_render.hpp rename to mediapipe/render/module/beauty/face_mesh_beauty_render.h diff --git a/mediapipe/render/module/beauty/face_mesh_beauty_render_imp.cpp b/mediapipe/render/module/beauty/face_mesh_beauty_render_imp.cpp deleted file mode 100644 index e69de29bb..000000000 diff --git a/mediapipe/render/module/beauty/face_mesh_beauty_render_imp.hpp b/mediapipe/render/module/beauty/face_mesh_beauty_render_imp.hpp deleted file mode 100644 index e69de29bb..000000000 diff --git a/mediapipe/render/module/beauty/face_mesh_module.cpp b/mediapipe/render/module/beauty/face_mesh_module.cc similarity index 79% rename from mediapipe/render/module/beauty/face_mesh_module.cpp rename to mediapipe/render/module/beauty/face_mesh_module.cc index 495126b67..413d43c7a 100644 --- a/mediapipe/render/module/beauty/face_mesh_module.cpp +++ b/mediapipe/render/module/beauty/face_mesh_module.cc @@ -1,4 +1,5 @@ -#include "face_mesh_module.h" +#include "face_mesh_module_imp.h" + namespace Opipe { FaceMeshModule::FaceMeshModule() { diff --git a/mediapipe/render/module/beauty/face_mesh_module.h b/mediapipe/render/module/beauty/face_mesh_module.h index a4e88667f..dd1f8ac37 100644 --- a/mediapipe/render/module/beauty/face_mesh_module.h +++ b/mediapipe/render/module/beauty/face_mesh_module.h @@ -1,4 +1,4 @@ -#include "OlaGraph.hpp" +#include namespace Opipe { class FaceMeshModule { diff --git a/mediapipe/render/module/beauty/face_mesh_module_imp.cpp b/mediapipe/render/module/beauty/face_mesh_module_imp.cc similarity index 100% rename from mediapipe/render/module/beauty/face_mesh_module_imp.cpp rename to mediapipe/render/module/beauty/face_mesh_module_imp.cc diff --git a/mediapipe/render/module/beauty/face_mesh_module_imp.hpp b/mediapipe/render/module/beauty/face_mesh_module_imp.h similarity index 77% rename from mediapipe/render/module/beauty/face_mesh_module_imp.hpp rename to mediapipe/render/module/beauty/face_mesh_module_imp.h index 12d8bfc2d..e68cd0336 100644 --- a/mediapipe/render/module/beauty/face_mesh_module_imp.hpp +++ b/mediapipe/render/module/beauty/face_mesh_module_imp.h @@ -1,5 +1,5 @@ #include "mediapipe/render/module/common/OlaGraph.hpp" -#include "face_mesh_module.hpp" +#include "mediapipe/render/module/beauty/face_mesh_module.h" namespace Opipe { class FaceMeshModuleIMP : public FaceMeshModule { diff --git a/mediapipe/render/module/common/BUILD b/mediapipe/render/module/common/BUILD index 888a4b051..c59d1c328 100644 --- a/mediapipe/render/module/common/BUILD +++ b/mediapipe/render/module/common/BUILD @@ -2,8 +2,8 @@ load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_binary_graph") package(default_visibility = ["//visibility:private"]) cc_library( name = "olamodule_common_library", - srcs = ["OlaGraph.cpp"], - hdrs = ["OlaGraph.hpp"], + srcs = ["ola_graph.cc"], + hdrs = ["ola_graph.h"], visibility = ["//mediapipe/framework:mediapipe_internal"], deps = [ "//mediapipe/framework:calculator_framework", diff --git a/mediapipe/render/module/common/OlaGraph.cpp b/mediapipe/render/module/common/OlaGraph.cpp deleted file mode 100644 index 28c4f22fa..000000000 --- a/mediapipe/render/module/common/OlaGraph.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#include -#include "OlaGraph.hpp" -#include "absl/memory/memory.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/graph_service.h" - -#include "mediapipe/gpu/gl_base.h" -#include "mediapipe/gpu/gpu_shared_data_internal.h" -#if defined(__APPLE__) -#include "mediapipe/gpu/MPPGraphGPUData.h" -#include "mediapipe/objc/util.h" -#endif - - - -namespace Opipe { - -void CallFrameDelegate(void* wrapperVoid, const std::string& streamName, - MPPPacketType packetType, - const mediapipe::Packet& packet) { -} - -OlaGraph::OlaGraph(const mediapipe::CalculatorGraphConfig &config) { - _config = config; - _graph = absl::make_unique(); - -} - -OlaGraph::~OlaGraph() { - -} - -mediapipe::ProfilingContext* OlaGraph::getProfiler() { - return _graph->profiler(); -} - -void OlaGraph::setHeaderPacket(const mediapipe::Packet &packet, std::string streamName) { - _streamHeaders[streamName] = packet; -} - -void OlaGraph::setSidePacket(const mediapipe::Packet &packet, std::string name) { - _inputSidePackets[name] = packet; -} - -void OlaGraph::setServicePacket(mediapipe::Packet &packet,const mediapipe::GraphServiceBase &service) { - _servicePackets[&service] = std::move(packet); -} - -void OlaGraph::addSidePackets(const std::map &extraSidePackets) { - _inputSidePackets.insert(extraSidePackets.begin(), extraSidePackets.end()); -} - -void OlaGraph::addFrameOutputStream(const std::string &outputStreamName, - MPPPacketType packetType) { - std::string callbackInputName; - mediapipe::tool::AddCallbackCalculator(outputStreamName, &_config, &callbackInputName, - /*use_std_function=*/true); - // No matter what ownership qualifiers are put on the pointer, NewPermanentCallback will - // still end up with a strong pointer to MPPGraph*. That is why we use void* instead. - void* wrapperVoid = this; - _inputSidePackets[callbackInputName] = - mediapipe::MakePacket>( [wrapperVoid, outputStreamName, packetType] - (const mediapipe::Packet& packet) { - CallFrameDelegate(wrapperVoid, outputStreamName, packetType, packet); - }); -} - -bool OlaGraph::start() { - absl::Status status = performStart(); - _started = true; - return false; -} - -absl::Status OlaGraph::performStart() { - absl::Status status = _graph->Initialize(_config); - if (!status.ok()) { - return status; - } - for (const auto& service_packet : _servicePackets) { - status = _graph->SetServicePacket(*service_packet.first, service_packet.second); - if (!status.ok()) { - return status; - } - } - status = _graph->StartRun(_inputSidePackets, _streamHeaders); - if (!status.ok()) { - return status; - } - return status; -} - -bool OlaGraph::sendPacket(const mediapipe::Packet &packet, - const std::string &streamName) { - absl::Status status = _graph->AddPacketToInputStream(streamName, packet); - return status.ok(); -} - -bool OlaGraph::movePacket(mediapipe::Packet &&packet, const std::string &streamName) { - absl::Status status = _graph->AddPacketToInputStream(streamName, std::move(packet)); - return status.ok(); -} - -/// Sets the maximum queue size for a stream. Experimental feature, currently -/// only supported for graph input streams. Should be called before starting the -/// graph. -bool OlaGraph::setMaxQueueSize(int maxQueueSize, - const std::string &streamName) { - absl::Status status = _graph->SetInputStreamMaxQueueSize(streamName, maxQueueSize); - return status.ok(); -} - -#if defined(__APPLE__) -/// Creates a MediaPipe packet wrapping the given pixelBuffer; -mediapipe::Packet OlaGraph::packetWithPixelBuffer(CVPixelBufferRef pixelBuffer, - MPPPacketType packetType) { - mediapipe::Packet packet; - if (packetType == MPPPacketTypeImageFrame || packetType == MPPPacketTypeImageFrameBGRANoSwap) { - auto frame = CreateImageFrameForCVPixelBuffer( - pixelBuffer, /* canOverwrite = */ false, - /* bgrAsRgb = */ packetType == MPPPacketTypeImageFrameBGRANoSwap); - packet = mediapipe::Adopt(frame.release()); - #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - } else if (packetType == MPPPacketTypePixelBuffer) { - packet = mediapipe::MakePacket(pixelBuffer); - #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - } else if (packetType == MPPPacketTypeImage) { - #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - // GPU - packet = mediapipe::MakePacket(pixelBuffer); - #else - // CPU - auto frame = CreateImageFrameForCVPixelBuffer(imageBuffer, /* canOverwrite = */ false, - /* bgrAsRgb = */ false); - packet = mediapipe::MakePacket(std::move(frame)); - #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - } else { - assert(false); - } - return packet; -} - - - -/// Creates a MediaPipe packet of type Image, wrapping the given CVPixelBufferRef. -mediapipe::Packet OlaGraph::imagePacketWithPixelBuffer(CVPixelBufferRef pixelBuffer) { - return packetWithPixelBuffer(pixelBuffer, MPPPacketTypeImage); -} - -/// Sends a pixel buffer into a graph input stream, using the specified packet -/// type. The graph must have been started before calling this. Drops frames and -/// returns NO if maxFramesInFlight is exceeded. If allowOverwrite is set to YES, -/// allows MediaPipe to overwrite the packet contents on successful sending for -/// possibly increased efficiency. Returns YES if the packet was successfully sent. -bool OlaGraph::sendPixelBuffer(CVPixelBufferRef imageBuffer, - const std::string & inputName, - MPPPacketType packetType, - const mediapipe::Timestamp ×tamp, - bool allowOverwrite) { - if (_maxFramesInFlight && _framesInFlight >= _maxFramesInFlight) return false; - mediapipe::Packet packet = packetWithPixelBuffer(imageBuffer, packetType); - bool success; - if (allowOverwrite) { - packet = std::move(packet).At(timestamp); - success = movePacket(std::move(packet), inputName); - } else { - success = sendPacket(packet.At(timestamp), inputName); - } - if (success) _framesInFlight++; - return success; -} - -/// Sends a pixel buffer into a graph input stream, using the specified packet -/// type. The graph must have been started before calling this. Drops frames and -/// returns NO if maxFramesInFlight is exceeded. Returns YES if the packet was -/// successfully sent. -bool OlaGraph::sendPixelBuffer(CVPixelBufferRef imageBuffer, - const std::string & inputName, - MPPPacketType packetType, - const mediapipe::Timestamp ×tamp) { - return sendPixelBuffer(imageBuffer, inputName, packetType, timestamp, false); -} - -/// Sends a pixel buffer into a graph input stream, using the specified packet -/// type. The graph must have been started before calling this. The timestamp is -/// automatically incremented from the last timestamp used by this method. Drops -/// frames and returns NO if maxFramesInFlight is exceeded. Returns YES if the -/// packet was successfully sent. -bool OlaGraph::sendPixelBuffer(CVPixelBufferRef imageBuffer, - const std::string & inputName, - MPPPacketType packetType) { - if (_frameTimestamp < mediapipe::Timestamp::Min()) { - _frameTimestamp = mediapipe::Timestamp::Min(); - } else { - _frameTimestamp++; - } - return sendPixelBuffer(imageBuffer, inputName, packetType, _frameTimestamp); -} - -#endif - -/// Cancels a graph run. You must still call waitUntilDoneWithError: after this. -void OlaGraph::cancel() { - _graph->Cancel(); -} - -/// Check if the graph contains this input stream -bool OlaGraph::hasInputStream(const std::string &inputName) { - return _graph->HasInputStream(inputName); -} - -/// Closes an input stream. -/// You must close all graph input streams before stopping the graph. -/// @return YES if successful. -bool OlaGraph::closeInputStream(const std::string &inputName) { - absl::Status status = _graph->CloseInputStream(inputName); - return status.ok(); -} - -/// Closes all graph input streams. -/// @return YES if successful. -bool OlaGraph::closeAllInputStreams() { - absl::Status status = _graph->CloseAllInputStreams(); - return status.ok(); -} - -/// Stops running the graph. -/// Call this before releasing this object. All input streams must have been -/// closed. This call does not time out, so you should not call it from the main -/// thread. -/// @return YES if successful. -bool OlaGraph::waitUntilDone() { - absl::Status status = _graph->WaitUntilDone(); - _started = false; - return status.ok(); -} - -/// Waits for the graph to become idle. -bool OlaGraph::waitUntilIdle() { - absl::Status status = _graph->WaitUntilIdle(); - return status.ok(); -} - - -} diff --git a/mediapipe/render/module/common/OlaGraph.hpp b/mediapipe/render/module/common/OlaGraph.hpp deleted file mode 100644 index e89e4cb4f..000000000 --- a/mediapipe/render/module/common/OlaGraph.hpp +++ /dev/null @@ -1,192 +0,0 @@ -#include "absl/memory/memory.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/graph_service.h" -#include "mediapipe/gpu/gl_base.h" -#include "mediapipe/gpu/gpu_shared_data_internal.h" - -#ifdef __OBJC__ -#import -#endif // __OBJC__ - -namespace Opipe { - -enum MPPPacketType { - MPPPacketTypeRaw, - MPPPacketTypeImage, - MPPPacketTypeImageFrame, - MPPPacketTypeImageFrameBGRANoSwap, -#if defined(__APPLE__) - MPPPacketTypePixelBuffer, -#endif -}; -class OlaGraph; -class MPPGraphDelegate { -public: - -#if defined(__APPLE__) - virtual void outputPixelbuffer(OlaGraph *graph, CVPixelBufferRef pixelbuffer, - const std::string &streamName); - - virtual void outputPixelbuffer(OlaGraph *graph, CVPixelBufferRef pixelbuffer, - const std::string &streamName, - mediapipe::Timestamp &timstamp); - -#endif - - virtual void outputPacket(OlaGraph *graph, mediapipe::Packet &packet, - const std::string &streamName); - -}; - - -class OlaGraph { - -public: - OlaGraph(const mediapipe::CalculatorGraphConfig &config); - ~OlaGraph(); - - absl::Status AddCallbackHandler(std::string output_stream_name, - void *callback); - absl::Status AddMultiStreamCallbackHandler(std::vector output_stream_names, - void *callback, - bool observe_timestamp_bounds); - - void setDelegate(std::shared_ptr delegate) { - _delegate = delegate; - } - - mediapipe::ProfilingContext* getProfiler(); - - int maxFramesInFlight; - - mediapipe::CalculatorGraph::GraphInputStreamAddMode packetAddMode() { - return _graph->GetGraphInputStreamAddMode(); - } - - void setPacketAddMode(mediapipe::CalculatorGraph::GraphInputStreamAddMode mode) { - _graph->SetGraphInputStreamAddMode(mode); - }; - - virtual void setHeaderPacket(const mediapipe::Packet &packet, std::string streamName); - - virtual void setSidePacket(const mediapipe::Packet &packet, std::string name); - - virtual void setServicePacket(mediapipe::Packet &packet,const mediapipe::GraphServiceBase &service); - - virtual void addSidePackets(const std::map &extraSidePackets); - - virtual void addFrameOutputStream(const std::string &outputStreamName, - MPPPacketType packetType); - - virtual bool start(); - - virtual bool sendPacket(const mediapipe::Packet &packet, - const std::string &streamName); - - virtual bool movePacket(mediapipe::Packet &&packet, const std::string &streamName); - - /// Sets the maximum queue size for a stream. Experimental feature, currently - /// only supported for graph input streams. Should be called before starting the - /// graph. - virtual bool setMaxQueueSize(int maxQueueSize, - const std::string &streamName); - -#if defined(__APPLE__) - /// Creates a MediaPipe packet wrapping the given pixelBuffer; - mediapipe::Packet packetWithPixelBuffer(CVPixelBufferRef pixelBuffer, - MPPPacketType packetType); - - - - /// Creates a MediaPipe packet of type Image, wrapping the given CVPixelBufferRef. - mediapipe::Packet imagePacketWithPixelBuffer(CVPixelBufferRef pixelBuffer); - - /// Sends a pixel buffer into a graph input stream, using the specified packet - /// type. The graph must have been started before calling this. Drops frames and - /// returns NO if maxFramesInFlight is exceeded. If allowOverwrite is set to YES, - /// allows MediaPipe to overwrite the packet contents on successful sending for - /// possibly increased efficiency. Returns YES if the packet was successfully sent. - bool sendPixelBuffer(CVPixelBufferRef imageBuffer, - const std::string & inputName, - MPPPacketType packetType, - const mediapipe::Timestamp ×tamp, - bool allowOverwrite); - - /// Sends a pixel buffer into a graph input stream, using the specified packet - /// type. The graph must have been started before calling this. Drops frames and - /// returns NO if maxFramesInFlight is exceeded. Returns YES if the packet was - /// successfully sent. - bool sendPixelBuffer(CVPixelBufferRef imageBuffer, - const std::string & inputName, - MPPPacketType packetType, - const mediapipe::Timestamp ×tamp); - - /// Sends a pixel buffer into a graph input stream, using the specified packet - /// type. The graph must have been started before calling this. The timestamp is - /// automatically incremented from the last timestamp used by this method. Drops - /// frames and returns NO if maxFramesInFlight is exceeded. Returns YES if the - /// packet was successfully sent. - bool sendPixelBuffer(CVPixelBufferRef imageBuffer, - const std::string & inputName, - MPPPacketType packetType); - -#endif - - /// Cancels a graph run. You must still call waitUntilDoneWithError: after this. - void cancel(); - - /// Check if the graph contains this input stream - bool hasInputStream(const std::string &inputName); - - /// Closes an input stream. - /// You must close all graph input streams before stopping the graph. - /// @return YES if successful. - bool closeInputStream(const std::string &inputName); - - /// Closes all graph input streams. - /// @return YES if successful. - bool closeAllInputStreams(); - - /// Stops running the graph. - /// Call this before releasing this object. All input streams must have been - /// closed. This call does not time out, so you should not call it from the main - /// thread. - /// @return YES if successful. - bool waitUntilDone(); - - /// Waits for the graph to become idle. - bool waitUntilIdle(); - - - - -private: - std::unique_ptr _graph; - mediapipe::CalculatorGraphConfig _config; - /// Input side packets that will be added to the graph when it is started. - std::map _inputSidePackets; - /// Packet headers that will be added to the graph when it is started. - std::map _streamHeaders; - /// Service packets to be added to the graph when it is started. - std::map _servicePackets; - - /// Number of frames currently being processed by the graph. - std::atomic _framesInFlight; - - mediapipe::Timestamp _frameTimestamp; - - int64 _frameNumber; - - bool _started; - - std::weak_ptr _delegate; - - absl::Status performStart(); - - int _maxFramesInFlight = 0; -}; - -} diff --git a/mediapipe/render/module/common/ola_graph.cc b/mediapipe/render/module/common/ola_graph.cc new file mode 100644 index 000000000..c19c3a498 --- /dev/null +++ b/mediapipe/render/module/common/ola_graph.cc @@ -0,0 +1,309 @@ +#include +#include "ola_graph.h" +#include "absl/memory/memory.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/graph_service.h" + +#include "mediapipe/gpu/gl_base.h" +#include "mediapipe/gpu/gpu_shared_data_internal.h" +#if defined(__APPLE__) +#include "mediapipe/gpu/MPPGraphGPUData.h" +#include "mediapipe/objc/util.h" +#endif + +using namespace mediapipe; + +namespace Opipe +{ + + void CallFrameDelegate(void *wrapperVoid, const std::string &streamName, MPPPacketType packetType, + const mediapipe::Packet &packet) + { + OlaGraph *graph = (OlaGraph *)wrapperVoid; + if (graph->_delegate.expired()) + { + return; + } + if (packetType == MPPPacketTypeRaw) + { + graph->_delegate.lock()->outputPacket(graph, packet, packetType, streamName); + } +#if defined(__APPLE__) + else if (packetType == MPPPacketTypePixelBuffer || + packetType == MPPPacketTypeImage) + { + graph->_framesInFlight--; + CVPixelBufferRef pixelBuffer; + if (packetType == MPPPacketTypePixelBuffer) + pixelBuffer = mediapipe::GetCVPixelBufferRef(packet.Get()); + else + pixelBuffer = packet.Get().GetCVPixelBufferRef(); + + graph->_delegate.lock()->outputPixelbuffer(graph, pixelBuffer, streamName, packet.Timestamp().Value()); +#endif + } + else + { + } + } + + OlaGraph::OlaGraph(const mediapipe::CalculatorGraphConfig &config) + { + _config = config; + _graph = absl::make_unique(); + } + + OlaGraph::~OlaGraph() + { + } + + mediapipe::ProfilingContext *OlaGraph::getProfiler() + { + return _graph->profiler(); + } + + void OlaGraph::setHeaderPacket(const mediapipe::Packet &packet, std::string streamName) + { + _streamHeaders[streamName] = packet; + } + + void OlaGraph::setSidePacket(const mediapipe::Packet &packet, std::string name) + { + _inputSidePackets[name] = packet; + } + + void OlaGraph::setServicePacket(mediapipe::Packet &packet, const mediapipe::GraphServiceBase &service) + { + _servicePackets[&service] = std::move(packet); + } + + void OlaGraph::addSidePackets(const std::map &extraSidePackets) + { + _inputSidePackets.insert(extraSidePackets.begin(), extraSidePackets.end()); + } + + void OlaGraph::addFrameOutputStream(const std::string &outputStreamName, + MPPPacketType packetType) + { + std::string callbackInputName; + mediapipe::tool::AddCallbackCalculator(outputStreamName, &_config, &callbackInputName, + /*use_std_function=*/true); + // No matter what ownership qualifiers are put on the pointer, NewPermanentCallback will + // still end up with a strong pointer to MPPGraph*. That is why we use void* instead. + void *wrapperVoid = this; + _inputSidePackets[callbackInputName] = + mediapipe::MakePacket>([wrapperVoid, outputStreamName, packetType](const mediapipe::Packet &packet) + { CallFrameDelegate(wrapperVoid, outputStreamName, packetType, packet); }); + } + + bool OlaGraph::start() + { + absl::Status status = performStart(); + _started = true; + return false; + } + + absl::Status OlaGraph::performStart() + { + absl::Status status = _graph->Initialize(_config); + if (!status.ok()) + { + return status; + } + for (const auto &service_packet : _servicePackets) + { + status = _graph->SetServicePacket(*service_packet.first, service_packet.second); + if (!status.ok()) + { + return status; + } + } + status = _graph->StartRun(_inputSidePackets, _streamHeaders); + if (!status.ok()) + { + return status; + } + return status; + } + + bool OlaGraph::sendPacket(const mediapipe::Packet &packet, + const std::string &streamName) + { + absl::Status status = _graph->AddPacketToInputStream(streamName, packet); + return status.ok(); + } + + bool OlaGraph::movePacket(mediapipe::Packet &&packet, const std::string &streamName) + { + absl::Status status = _graph->AddPacketToInputStream(streamName, std::move(packet)); + return status.ok(); + } + + /// Sets the maximum queue size for a stream. Experimental feature, currently + /// only supported for graph input streams. Should be called before starting the + /// graph. + bool OlaGraph::setMaxQueueSize(int maxQueueSize, + const std::string &streamName) + { + absl::Status status = _graph->SetInputStreamMaxQueueSize(streamName, maxQueueSize); + return status.ok(); + } + +#if defined(__APPLE__) + /// Creates a MediaPipe packet wrapping the given pixelBuffer; + mediapipe::Packet OlaGraph::packetWithPixelBuffer(CVPixelBufferRef pixelBuffer, + MPPPacketType packetType) + { + mediapipe::Packet packet; + if (packetType == MPPPacketTypeImageFrame || packetType == MPPPacketTypeImageFrameBGRANoSwap) + { + auto frame = CreateImageFrameForCVPixelBuffer( + pixelBuffer, /* canOverwrite = */ false, + /* bgrAsRgb = */ packetType == MPPPacketTypeImageFrameBGRANoSwap); + packet = mediapipe::Adopt(frame.release()); +#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER + } + else if (packetType == MPPPacketTypePixelBuffer) + { + packet = mediapipe::MakePacket(pixelBuffer); +#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER + } + else if (packetType == MPPPacketTypeImage) + { +#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER + // GPU + packet = mediapipe::MakePacket(pixelBuffer); +#else + // CPU + auto frame = CreateImageFrameForCVPixelBuffer(imageBuffer, /* canOverwrite = */ false, + /* bgrAsRgb = */ false); + packet = mediapipe::MakePacket(std::move(frame)); +#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER + } + else + { + assert(false); + } + return packet; + } + + /// Creates a MediaPipe packet of type Image, wrapping the given CVPixelBufferRef. + mediapipe::Packet OlaGraph::imagePacketWithPixelBuffer(CVPixelBufferRef pixelBuffer) + { + return packetWithPixelBuffer(pixelBuffer, MPPPacketTypeImage); + } + + /// Sends a pixel buffer into a graph input stream, using the specified packet + /// type. The graph must have been started before calling this. Drops frames and + /// returns NO if maxFramesInFlight is exceeded. If allowOverwrite is set to YES, + /// allows MediaPipe to overwrite the packet contents on successful sending for + /// possibly increased efficiency. Returns YES if the packet was successfully sent. + bool OlaGraph::sendPixelBuffer(CVPixelBufferRef imageBuffer, + const std::string &inputName, + MPPPacketType packetType, + const mediapipe::Timestamp ×tamp, + bool allowOverwrite) + { + if (_maxFramesInFlight && _framesInFlight >= _maxFramesInFlight) + return false; + mediapipe::Packet packet = packetWithPixelBuffer(imageBuffer, packetType); + bool success; + if (allowOverwrite) + { + packet = std::move(packet).At(timestamp); + success = movePacket(std::move(packet), inputName); + } + else + { + success = sendPacket(packet.At(timestamp), inputName); + } + if (success) + _framesInFlight++; + return success; + } + + /// Sends a pixel buffer into a graph input stream, using the specified packet + /// type. The graph must have been started before calling this. Drops frames and + /// returns NO if maxFramesInFlight is exceeded. Returns YES if the packet was + /// successfully sent. + bool OlaGraph::sendPixelBuffer(CVPixelBufferRef imageBuffer, + const std::string &inputName, + MPPPacketType packetType, + const mediapipe::Timestamp ×tamp) + { + return sendPixelBuffer(imageBuffer, inputName, packetType, timestamp, false); + } + + /// Sends a pixel buffer into a graph input stream, using the specified packet + /// type. The graph must have been started before calling this. The timestamp is + /// automatically incremented from the last timestamp used by this method. Drops + /// frames and returns NO if maxFramesInFlight is exceeded. Returns YES if the + /// packet was successfully sent. + bool OlaGraph::sendPixelBuffer(CVPixelBufferRef imageBuffer, + const std::string &inputName, + MPPPacketType packetType) + { + if (_frameTimestamp < mediapipe::Timestamp::Min()) + { + _frameTimestamp = mediapipe::Timestamp::Min(); + } + else + { + _frameTimestamp++; + } + return sendPixelBuffer(imageBuffer, inputName, packetType, _frameTimestamp); + } + +#endif + + /// Cancels a graph run. You must still call waitUntilDoneWithError: after this. + void OlaGraph::cancel() + { + _graph->Cancel(); + } + + /// Check if the graph contains this input stream + bool OlaGraph::hasInputStream(const std::string &inputName) + { + return _graph->HasInputStream(inputName); + } + + /// Closes an input stream. + /// You must close all graph input streams before stopping the graph. + /// @return YES if successful. + bool OlaGraph::closeInputStream(const std::string &inputName) + { + absl::Status status = _graph->CloseInputStream(inputName); + return status.ok(); + } + + /// Closes all graph input streams. + /// @return YES if successful. + bool OlaGraph::closeAllInputStreams() + { + absl::Status status = _graph->CloseAllInputStreams(); + return status.ok(); + } + + /// Stops running the graph. + /// Call this before releasing this object. All input streams must have been + /// closed. This call does not time out, so you should not call it from the main + /// thread. + /// @return YES if successful. + bool OlaGraph::waitUntilDone() + { + absl::Status status = _graph->WaitUntilDone(); + _started = false; + return status.ok(); + } + + /// Waits for the graph to become idle. + bool OlaGraph::waitUntilIdle() + { + absl::Status status = _graph->WaitUntilIdle(); + return status.ok(); + } + +} diff --git a/mediapipe/render/module/common/ola_graph.h b/mediapipe/render/module/common/ola_graph.h new file mode 100644 index 000000000..0470300a9 --- /dev/null +++ b/mediapipe/render/module/common/ola_graph.h @@ -0,0 +1,190 @@ +#include "absl/memory/memory.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_graph.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/graph_service.h" +#include "mediapipe/gpu/gl_base.h" +#include "mediapipe/gpu/gpu_shared_data_internal.h" + +#ifdef __OBJC__ +#import +#endif // __OBJC__ + +using namespace mediapipe; +namespace Opipe +{ + + enum MPPPacketType + { + MPPPacketTypeRaw, + MPPPacketTypeImage, + MPPPacketTypeImageFrame, + MPPPacketTypeImageFrameBGRANoSwap, +#if defined(__APPLE__) + MPPPacketTypePixelBuffer, +#endif + }; + class OlaGraph; + class MPPGraphDelegate + { + public: +#if defined(__APPLE__) + virtual void outputPixelbuffer(OlaGraph *graph, CVPixelBufferRef pixelbuffer, + const std::string &streamName, + int64_t timstamp); + +#endif + + virtual void outputPacket(OlaGraph *graph, + const mediapipe::Packet &packet, + MPPPacketType packetType, + const std::string &streamName); + }; + + class OlaGraph + { + + public: + OlaGraph(const mediapipe::CalculatorGraphConfig &config); + ~OlaGraph(); + + absl::Status AddCallbackHandler(std::string output_stream_name, + void *callback); + absl::Status AddMultiStreamCallbackHandler(std::vector output_stream_names, + void *callback, + bool observe_timestamp_bounds); + + void setDelegate(std::shared_ptr delegate) + { + _delegate = delegate; + } + + mediapipe::ProfilingContext *getProfiler(); + + int maxFramesInFlight; + + mediapipe::CalculatorGraph::GraphInputStreamAddMode packetAddMode() + { + return _graph->GetGraphInputStreamAddMode(); + } + + void setPacketAddMode(mediapipe::CalculatorGraph::GraphInputStreamAddMode mode) + { + _graph->SetGraphInputStreamAddMode(mode); + }; + + virtual void setHeaderPacket(const mediapipe::Packet &packet, std::string streamName); + + virtual void setSidePacket(const mediapipe::Packet &packet, std::string name); + + virtual void setServicePacket(mediapipe::Packet &packet, const mediapipe::GraphServiceBase &service); + + virtual void addSidePackets(const std::map &extraSidePackets); + + virtual void addFrameOutputStream(const std::string &outputStreamName, + MPPPacketType packetType); + + virtual bool start(); + + virtual bool sendPacket(const mediapipe::Packet &packet, + const std::string &streamName); + + virtual bool movePacket(mediapipe::Packet &&packet, const std::string &streamName); + + /// Sets the maximum queue size for a stream. Experimental feature, currently + /// only supported for graph input streams. Should be called before starting the + /// graph. + virtual bool setMaxQueueSize(int maxQueueSize, + const std::string &streamName); + +#if defined(__APPLE__) + /// Creates a MediaPipe packet wrapping the given pixelBuffer; + mediapipe::Packet packetWithPixelBuffer(CVPixelBufferRef pixelBuffer, + MPPPacketType packetType); + + /// Creates a MediaPipe packet of type Image, wrapping the given CVPixelBufferRef. + mediapipe::Packet imagePacketWithPixelBuffer(CVPixelBufferRef pixelBuffer); + + /// Sends a pixel buffer into a graph input stream, using the specified packet + /// type. The graph must have been started before calling this. Drops frames and + /// returns NO if maxFramesInFlight is exceeded. If allowOverwrite is set to YES, + /// allows MediaPipe to overwrite the packet contents on successful sending for + /// possibly increased efficiency. Returns YES if the packet was successfully sent. + bool sendPixelBuffer(CVPixelBufferRef imageBuffer, + const std::string &inputName, + MPPPacketType packetType, + const mediapipe::Timestamp ×tamp, + bool allowOverwrite); + + /// Sends a pixel buffer into a graph input stream, using the specified packet + /// type. The graph must have been started before calling this. Drops frames and + /// returns NO if maxFramesInFlight is exceeded. Returns YES if the packet was + /// successfully sent. + bool sendPixelBuffer(CVPixelBufferRef imageBuffer, + const std::string &inputName, + MPPPacketType packetType, + const mediapipe::Timestamp ×tamp); + + /// Sends a pixel buffer into a graph input stream, using the specified packet + /// type. The graph must have been started before calling this. The timestamp is + /// automatically incremented from the last timestamp used by this method. Drops + /// frames and returns NO if maxFramesInFlight is exceeded. Returns YES if the + /// packet was successfully sent. + bool sendPixelBuffer(CVPixelBufferRef imageBuffer, + const std::string &inputName, + MPPPacketType packetType); + +#endif + + /// Cancels a graph run. You must still call waitUntilDoneWithError: after this. + void cancel(); + + /// Check if the graph contains this input stream + bool hasInputStream(const std::string &inputName); + + /// Closes an input stream. + /// You must close all graph input streams before stopping the graph. + /// @return YES if successful. + bool closeInputStream(const std::string &inputName); + + /// Closes all graph input streams. + /// @return YES if successful. + bool closeAllInputStreams(); + + /// Stops running the graph. + /// Call this before releasing this object. All input streams must have been + /// closed. This call does not time out, so you should not call it from the main + /// thread. + /// @return YES if successful. + bool waitUntilDone(); + + /// Waits for the graph to become idle. + bool waitUntilIdle(); + + std::weak_ptr _delegate; + std::atomic _framesInFlight = 2; + + private: + std::unique_ptr _graph; + mediapipe::CalculatorGraphConfig _config; + /// Input side packets that will be added to the graph when it is started. + std::map _inputSidePackets; + /// Packet headers that will be added to the graph when it is started. + std::map _streamHeaders; + /// Service packets to be added to the graph when it is started. + std::map _servicePackets; + + /// Number of frames currently being processed by the graph. + mediapipe::Timestamp _frameTimestamp; + + int64 _frameNumber; + + bool _started; + + absl::Status performStart(); + + int _maxFramesInFlight = 0; + }; + +}