// Copyright 2019 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #import "MPPGLViewRenderer.h" #include #include #include #include #include "absl/memory/memory.h" #include "mediapipe/gpu/gl_quad_renderer.h" #include "mediapipe/gpu/gl_simple_shaders.h" #include "mediapipe/gpu/shader_util.h" #import "mediapipe/objc/NSError+util_status.h" #import "GTMDefines.h" @implementation MPPGLViewRenderer { /// Used to create textures for the pixel buffers to be rendered. /// The use of this class allows the GPU to access the buffer's memory directly, /// without a memory copy. CVOpenGLESTextureCacheRef _textureCache; /// Internal renderer. std::unique_ptr renderer_; /// Used to synchronize access to _nextPixelBufferToRender. OSSpinLock _bufferLock; volatile CVPixelBufferRef _nextPixelBufferToRender; } - (instancetype)init { self = [super init]; if (self) { _glContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2]; _bufferLock = OS_SPINLOCK_INIT; _frameRotationMode = MPPFrameRotationNone; _frameScaleMode = MPPFrameScaleModeFit; } return self; } - (void)dealloc { if (_textureCache) CFRelease(_textureCache); CVPixelBufferRelease(_nextPixelBufferToRender); // Fixes crash during dealloc that only happens in iOS 9. More info at b/67095363. if ([EAGLContext currentContext] == _glContext) { [EAGLContext setCurrentContext:nil]; } } - (CVPixelBufferRef)nextPixelBufferToRender { OSSpinLockLock(&_bufferLock); CVPixelBufferRef buffer = _nextPixelBufferToRender; OSSpinLockUnlock(&_bufferLock); return buffer; } - (void)setNextPixelBufferToRender:(CVPixelBufferRef)buffer { OSSpinLockLock(&_bufferLock); if (_nextPixelBufferToRender != buffer) { CVPixelBufferRelease(_nextPixelBufferToRender); _nextPixelBufferToRender = buffer; CVPixelBufferRetain(_nextPixelBufferToRender); } OSSpinLockUnlock(&_bufferLock); } - (void)setupGL { CVReturn err; err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, _glContext, NULL, &_textureCache); _GTMDevAssert(err == kCVReturnSuccess, @"CVOpenGLESTextureCacheCreate failed: %d", err); renderer_ = absl::make_unique(); auto status = renderer_->GlSetup(); _GTMDevAssert(status.ok(), @"renderer setup failed: %@", [NSError gus_errorWithStatus:status]); } mediapipe::FrameScaleMode InternalScaleMode(MPPFrameScaleMode mode) { switch (mode) { case MPPFrameScaleModeFit: return mediapipe::FrameScaleMode::kFit; case MPPFrameScaleModeFillAndCrop: return mediapipe::FrameScaleMode::kFillAndCrop; } } mediapipe::FrameRotation InternalRotationMode(MPPFrameRotation rot) { switch (rot) { case MPPFrameRotationNone: return mediapipe::FrameRotation::kNone; case MPPFrameRotationCw90: return mediapipe::FrameRotation::k90; case MPPFrameRotationCw180: return mediapipe::FrameRotation::k180; case MPPFrameRotationCw270: return mediapipe::FrameRotation::k270; } } - (void)drawPixelBuffer:(CVPixelBufferRef)pixelBuffer width:(GLfloat)viewWidth height:(GLfloat)viewHeight { if (!_textureCache) [self setupGL]; size_t frameWidth = CVPixelBufferGetWidth(pixelBuffer); size_t frameHeight = CVPixelBufferGetHeight(pixelBuffer); CVOpenGLESTextureRef texture = NULL; OSType pixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); CVReturn error; if (pixelFormat == kCVPixelFormatType_OneComponent8) { error = CVOpenGLESTextureCacheCreateTextureFromImage( kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, GL_TEXTURE_2D, GL_LUMINANCE, (GLsizei)frameWidth, (GLsizei)frameHeight, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0, &texture); } else if (pixelFormat == kCVPixelFormatType_OneComponent32Float) { error = CVOpenGLESTextureCacheCreateTextureFromImage( kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, GL_TEXTURE_2D, GL_LUMINANCE, (GLsizei)frameWidth, (GLsizei)frameHeight, GL_LUMINANCE, GL_FLOAT, 0, &texture); } else { error = CVOpenGLESTextureCacheCreateTextureFromImage( kCFAllocatorDefault, _textureCache, pixelBuffer, NULL, GL_TEXTURE_2D, GL_RGBA, (GLsizei)frameWidth, (GLsizei)frameHeight, GL_BGRA, GL_UNSIGNED_BYTE, 0, &texture); } _GTMDevAssert(error == kCVReturnSuccess, @"CVOpenGLESTextureCacheCreateTextureFromImage failed: %d", error); glClear(GL_COLOR_BUFFER_BIT); glActiveTexture(GL_TEXTURE1); glBindTexture(CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture)); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // Note: we perform a vertical flip by swapping the top and bottom coordinates. // CVPixelBuffers have a top left origin and OpenGL has a bottom left origin. auto status = renderer_->GlRender( frameWidth, frameHeight, viewWidth, viewHeight, InternalScaleMode(_frameScaleMode), InternalRotationMode(_frameRotationMode), _mirrored, /*flip_vertical=*/false, /*flip_texture=*/true); _GTMDevAssert(status.ok(), @"render failed: %@", [NSError gus_errorWithStatus:status]); glBindTexture(CVOpenGLESTextureGetTarget(texture), 0); glBindTexture(GL_TEXTURE_2D, 0 ); CFRelease(texture); CVOpenGLESTextureCacheFlush(_textureCache, 0); } #pragma mark - GLKViewDelegate - (void)glkView:(GLKView *)view drawInRect:(CGRect)rect { CVPixelBufferRef pixelBuffer = NULL; OSSpinLockLock(&_bufferLock); pixelBuffer = _nextPixelBufferToRender; if (_retainsLastPixelBuffer) { CVPixelBufferRetain(pixelBuffer); } else { _nextPixelBufferToRender = NULL; } OSSpinLockUnlock(&_bufferLock); if (!pixelBuffer) return; [self drawPixelBuffer:pixelBuffer width:view.drawableWidth height:view.drawableHeight]; CVPixelBufferRelease(pixelBuffer); } @end