接入FaceMesh 美颜渲染 50%

.vscode/settings.json

@@ -63,6 +63,13 @@
         "stop_token": "cpp",
         "thread": "cpp",
         "target.hpp": "c",
-        "calculator_framework.h": "c"
+        "calculator_framework.h": "c",
+        "bitset": "cpp",
+        "complex": "cpp",
+        "cstdarg": "cpp",
+        "iostream": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "__config": "cpp"
     }
 }

mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt

@@ -12,6 +12,8 @@ output_stream: "output_video"
 # landmarks. (std::vector<NormalizedLandmarkList>)
 output_stream: "multi_face_landmarks"
 
+# output_stream: "face_detections"
+
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
 # (calculators and subgraphs) in the graph to finish their tasks before it
@@ -56,12 +58,22 @@ node {
   output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
 }
 
-# Subgraph that renders face-landmark annotation onto the input image.
+# Subgraph that renders face-landmark annotation onto the input image.  //画标记 用于调试
+# node {
+#   calculator: "FaceRendererGpu"
+#   input_stream: "IMAGE:throttled_input_video"
+#   input_stream: "LANDMARKS:multi_face_landmarks"
+#   input_stream: "NORM_RECTS:face_rects_from_landmarks"
+#   input_stream: "DETECTIONS:face_detections"
+#   output_stream: "IMAGE:output_video"
+# }
+
+# Draws annotations and overlays them on top of the input images.
 node {
-  calculator: "FaceRendererGpu"
+  calculator: "AnnotationOverlayCalculator"
-  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
-  input_stream: "LANDMARKS:multi_face_landmarks"
+  # input_stream: "detections_render_data"
-  input_stream: "NORM_RECTS:face_rects_from_landmarks"
+  # input_stream: "VECTOR:0:multi_face_landmarks_render_data"
-  input_stream: "DETECTIONS:face_detections"
+  # input_stream: "rects_render_data"
-  output_stream: "IMAGE:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }

mediapipe/render/core/AlphaBlendFilter.cpp

@@ -0,0 +1,45 @@
+#include "AlphaBlendFilter.hpp"
+
+namespace Opipe {
+    const std::string kAlphaBlendFragmentShaderString = SHADER_STRING
+    (
+         varying highp vec2 vTexCoord;
+         varying highp vec2 vTexCoord1;
+         uniform sampler2D colorMap;
+         uniform sampler2D colorMap1;
+         uniform lowp float mixturePercent;
+         void main() {
+             lowp vec4 textureColor = texture2D(colorMap, vTexCoord);
+             lowp vec4 textureColor2 = texture2D(colorMap1, vTexCoord1);
+             gl_FragColor = vec4(mix(textureColor.rgb, textureColor2.rgb, textureColor2.a * mixturePercent), textureColor.a);
+         }
+     );
+    AlphaBlendFilter::AlphaBlendFilter(Context *context) : Filter(context), _mix(1.0) {
+        
+    }
+    
+    AlphaBlendFilter* AlphaBlendFilter::create(Context *context) {
+        AlphaBlendFilter* ret = new (std::nothrow) AlphaBlendFilter(context);
+        if (!ret || !ret->init(context)) {
+            delete ret;
+            ret = 0;
+        }
+        return ret;
+    }
+    
+    bool AlphaBlendFilter::init(Context *context) {
+        if (!Filter::initWithFragmentShaderString(context,
+                                                  kAlphaBlendFragmentShaderString,
+                                                  2)) {
+            return false;
+        }
+        return true;
+    }
+
+
+    bool AlphaBlendFilter::proceed(float frameTime,
+                                          bool bUpdateTargets/* = true*/) {
+        _filterProgram->setUniformValue("mixturePercent", _mix);
+        return Filter::proceed(frameTime, bUpdateTargets);
+    }
+}

mediapipe/render/core/AlphaBlendFilter.hpp

@@ -0,0 +1,32 @@
+#ifndef AlphaBlendFilter_cpp
+#define AlphaBlendFilter_cpp
+
+#include <stdio.h>
+#include "Filter.hpp"
+
+namespace Opipe {
+    class AlphaBlendFilter : public virtual Filter {
+    public:
+        static AlphaBlendFilter* create(Context *context);
+        bool init(Context *context);
+        
+        virtual bool proceed(float fraAlpmeTime = 0.0,
+                             bool bUpdateTargets = true) override;
+        
+        float getMix() {
+            return _mix;
+        };
+        
+        void setMix(float mix) {
+            _mix = mix;
+        }
+        
+    public:
+        AlphaBlendFilter(Context *context);
+        virtual ~AlphaBlendFilter() {};
+        float _mix;
+        
+    };
+}
+
+#endif

mediapipe/render/core/BUILD

@@ -125,6 +125,14 @@ OLARENDER_SRCS = [
     "dispatch_queue.cpp",
     "GLThreadDispatch.cpp",
     "OpipeDispatch.cpp",
+    "OlaShareTextureFilter.cpp",
+    "AlphaBlendFilter.cpp",
+    "BilateralFilter.cpp",
+    "GaussianBlurFilter.cpp",
+    "GaussianBlurMonoFilter.cpp",
+    "LUTFilter.cpp",
+    "OlaContext.cpp",
+
 ]
 
 OLARENDER_HDRS = [
@@ -146,6 +154,13 @@ OLARENDER_HDRS = [
     "dispatch_queue.h",
     "GLThreadDispatch.h",
     "OpipeDispatch.hpp",
+    "OlaShareTextureFilter.hpp",
+    "AlphaBlendFilter.hpp",
+    "BilateralFilter.hpp",
+    "GaussianBlurFilter.hpp",
+    "GaussianBlurMonoFilter.hpp",
+    "LUTFilter.hpp",
+    "OlaContext.hpp",
 ]
 
 

mediapipe/render/core/BilateralFilter.cpp

@@ -0,0 +1,231 @@
+#include "BilateralFilter.hpp"
+
+NS_GI_BEGIN
+
+const std::string kBilateralBlurVertexShaderString = SHADER_STRING
+(
+ attribute vec4 position;
+ attribute vec4 texCoord;
+ 
+ const int GAUSSIAN_SAMPLES = 9;
+ 
+ uniform float texelSpacingU;
+ uniform float texelSpacingV;
+
+ varying vec2 blurCoordinates[GAUSSIAN_SAMPLES];
+ 
+ void main()
+ {
+     gl_Position = position;
+     vec2 texelSpacing = vec2(texelSpacingU, texelSpacingV);
+     for (int i = 0; i < GAUSSIAN_SAMPLES; i++)
+     {
+         blurCoordinates[i] = texCoord.xy + texelSpacing * float((i - ((GAUSSIAN_SAMPLES - 1) / 2)));
+     }
+ }
+ );
+
+const std::string kBilateralBlurFragmentShaderString = SHADER_STRING
+(
+ uniform sampler2D colorMap;
+ 
+ const lowp int GAUSSIAN_SAMPLES = 9;
+ 
+ varying highp vec2 blurCoordinates[GAUSSIAN_SAMPLES];
+ 
+ uniform mediump float distanceNormalizationFactor;
+ 
+ void main()
+ {
+     lowp vec4 centralColor;
+     lowp float gaussianWeightTotal;
+     lowp vec4 sum;
+     lowp vec4 sampleColor;
+     lowp float distanceFromCentralColor;
+     lowp float gaussianWeight;
+     
+     centralColor = texture2D(colorMap, blurCoordinates[4]);
+     gaussianWeightTotal = 0.18;
+     sum = centralColor * 0.18;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[0]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.05 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[1]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.09 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[2]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.12 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[3]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.15 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[5]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.15 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[6]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.12 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[7]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.09 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     sampleColor = texture2D(colorMap, blurCoordinates[8]);
+     distanceFromCentralColor = min(distance(centralColor, sampleColor) * distanceNormalizationFactor, 1.0);
+     gaussianWeight = 0.05 * (1.0 - distanceFromCentralColor);
+     gaussianWeightTotal += gaussianWeight;
+     sum += sampleColor * gaussianWeight;
+     
+     if (gaussianWeightTotal < 0.4) {
+         gl_FragColor = centralColor;
+     } else if (gaussianWeightTotal < 0.5) {
+         gl_FragColor = mix(sum / gaussianWeightTotal, centralColor, (gaussianWeightTotal - 0.4) / 0.1);
+     } else {
+         gl_FragColor = sum / gaussianWeightTotal;
+     }
+ }
+ );
+
+ BilateralMonoFilter::BilateralMonoFilter(Context *context, Type type) : Filter(context)
+ ,_type(type)
+,_texelSpacingMultiplier(4.0)
+,_distanceNormalizationFactor(8.0)
+ {
+ 
+ }
+ 
+ BilateralMonoFilter* BilateralMonoFilter::create(Context *context, Type type/* = HORIZONTAL*/) {
+     BilateralMonoFilter* ret = new (std::nothrow) BilateralMonoFilter(context, type);
+     if (!ret || !ret->init(context)) {
+         delete ret;
+         ret = 0;
+     }
+     return ret;
+ }
+
+bool BilateralMonoFilter::init(Context *context) {
+    if (Filter::initWithShaderString(context, kBilateralBlurVertexShaderString, kBilateralBlurFragmentShaderString)) {
+        return true;
+    }
+    return false;
+}
+
+bool BilateralMonoFilter::proceed(float frameTime, bool bUpdateTargets/* = true*/) {
+    __unused Framebuffer* inputFramebuffer = _inputFramebuffers.begin()->second.frameBuffer;
+    RotationMode inputRotation = _inputFramebuffers.begin()->second.rotationMode;
+    
+    if (rotationSwapsSize(inputRotation))
+    {
+        if (_type == HORIZONTAL) {
+            _filterProgram->setUniformValue("texelSpacingU", (float)0.0);
+            _filterProgram->setUniformValue("texelSpacingV", (float)(_texelSpacingMultiplier / _framebuffer->getWidth()));
+        } else {
+            _filterProgram->setUniformValue("texelSpacingU", (float)(_texelSpacingMultiplier / _framebuffer->getHeight()));
+            _filterProgram->setUniformValue("texelSpacingV", (float)0.0);
+        }
+    } else {
+        if (_type == HORIZONTAL) {
+            _filterProgram->setUniformValue("texelSpacingU", (float)(_texelSpacingMultiplier / _framebuffer->getWidth()));
+            _filterProgram->setUniformValue("texelSpacingV", (float)0.0);
+        } else {
+            _filterProgram->setUniformValue("texelSpacingU", (float)0.0);
+            _filterProgram->setUniformValue("texelSpacingV", (float)(_texelSpacingMultiplier / _framebuffer->getHeight()));
+        }
+    }
+    
+    
+    _filterProgram->setUniformValue("distanceNormalizationFactor", _distanceNormalizationFactor);
+    return Filter::proceed(frameTime, bUpdateTargets);
+}
+
+void BilateralMonoFilter::setTexelSpacingMultiplier(float multiplier) {
+    _texelSpacingMultiplier = multiplier;
+}
+
+void BilateralMonoFilter::setDistanceNormalizationFactor(float value) {
+    _distanceNormalizationFactor = value;
+}
+
+REGISTER_FILTER_CLASS(BilateralFilter)
+
+BilateralFilter::BilateralFilter(Context *context) : FilterGroup(context)
+,_hBlurFilter(0)
+,_vBlurFilter(0)
+{
+}
+
+BilateralFilter::~BilateralFilter() {
+    if (_hBlurFilter) {
+        _hBlurFilter->release();
+        _hBlurFilter = 0;
+    }
+    
+    if (_vBlurFilter) {
+        _vBlurFilter->release();
+        _vBlurFilter = 0;
+    }
+    
+}
+
+BilateralFilter* BilateralFilter::create(Context *context) {
+    BilateralFilter* ret = new (std::nothrow) BilateralFilter(context);
+    if (ret && !ret->init(context)) {
+        delete ret;
+        ret = 0;
+    }
+    return ret;
+}
+
+bool BilateralFilter::init(Context *context) {
+    if (!FilterGroup::init(context)) {
+        return false;
+    }
+    
+    _hBlurFilter = BilateralMonoFilter::create(context, BilateralMonoFilter::HORIZONTAL);
+    _vBlurFilter = BilateralMonoFilter::create(context, BilateralMonoFilter::VERTICAL);
+    _hBlurFilter->addTarget(_vBlurFilter);
+    addFilter(_hBlurFilter);
+
+    registerProperty("texelSpacingMultiplier", 4.0, "The texel spacing multiplier.", [this](float& texelSpacingMultiplier){
+        setTexelSpacingMultiplier(texelSpacingMultiplier);
+    });
+
+    registerProperty("distanceNormalizationFactor", 8.0, "The distance normalization factor.", [this](float& distanceNormalizationFactor){
+        setDistanceNormalizationFactor(distanceNormalizationFactor);
+    });
+
+    return true;
+}
+
+void BilateralFilter::setTexelSpacingMultiplier(float multiplier) {
+    _hBlurFilter->setTexelSpacingMultiplier(multiplier);
+    _vBlurFilter->setTexelSpacingMultiplier(multiplier);
+}
+
+void BilateralFilter::setDistanceNormalizationFactor(float value) {
+    _hBlurFilter->setDistanceNormalizationFactor(value);
+    _vBlurFilter->setDistanceNormalizationFactor(value);
+    
+}
+
+NS_GI_END

mediapipe/render/core/BilateralFilter.hpp

@@ -0,0 +1,50 @@
+#ifndef BilateralFilter_hpp
+#define BilateralFilter_hpp
+
+#include "FilterGroup.hpp"
+#include "GPUImageMacros.h"
+
+NS_GI_BEGIN
+
+class BilateralMonoFilter : public Filter {
+public:
+    enum Type {HORIZONTAL, VERTICAL};
+    
+    static BilateralMonoFilter* create(Context *context, Type type = HORIZONTAL);
+    bool init(Context *context);
+    
+    virtual bool proceed(float frameTime = 0, bool bUpdateTargets = true) override;
+    
+    void setTexelSpacingMultiplier(float multiplier);
+    void setDistanceNormalizationFactor(float value);
+protected:
+    BilateralMonoFilter(Context *context, Type type);
+    Type _type;
+    float _texelSpacingMultiplier;
+    float _distanceNormalizationFactor;
+};
+
+class BilateralFilter : public FilterGroup {
+public:
+    virtual ~BilateralFilter();
+    
+    static BilateralFilter* create(Context *context);
+    bool init(Context *context);
+    
+    void setTexelSpacingMultiplier(float multiplier);
+    void setDistanceNormalizationFactor(float value);
+    
+public:
+    BilateralFilter(Context *context);
+    
+private:
+    //friend BilateralMonoFilter;
+    BilateralMonoFilter* _hBlurFilter;
+    BilateralMonoFilter* _vBlurFilter;
+};
+
+
+NS_GI_END
+
+
+#endif

mediapipe/render/core/CVFramebuffer.cpp

@@ -1,6 +1,6 @@
 //
 //  CVFramebuffer.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by wangrenzhu on 2021/4/30.
 //
@@ -353,7 +353,7 @@ void CVFramebuffer::_bindFramebuffer() {
     
     CHECK_GL(glBindTexture(GL_TEXTURE_2D, 0));
     CHECK_GL(glBindFramebuffer(GL_FRAMEBUFFER, 0));
-    //    Opipe::Log("Quaramera", "_generateFramebuffer %d ", _framebuffer);
+    //    Opipe::Log("Opipe", "_generateFramebuffer %d ", _framebuffer);
     assert(_framebuffer < 100);
 }
 

mediapipe/render/core/CVFramebuffer.hpp

@@ -1,6 +1,6 @@
 //
 //  CVFramebuffer.hpp
-//  Quaramera
+//  Opipe
 //
 //  Created by wangrenzhu on 2021/4/30.
 //

mediapipe/render/core/Context.cpp

@@ -34,6 +34,27 @@ std::mutex Context::_mutex;
 std::string Context::activatedContextKey = "";
 std::map<std::string, Context*> Context::_ContextCache;
 
+Context::Context(EAGLContext *context)
+:_curShaderProgram(0)
+,isCapturingFrame(false)
+,captureUpToFilter(0)
+,capturedFrameData(0)
+,_eglContext(0)
+,_eglOfflinerenderContext(0)
+,_eglContextIO(0)
+,vertexArray(-1) {
+    _framebufferCache = new FramebufferCache(this);
+
+#if defined(__APPLE__)
+
+    _eglContext = context;
+    shareGroup = [_eglContext sharegroup];
+    _eglContextIO = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3 sharegroup:shareGroup];
+    _eglOfflinerenderContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3 sharegroup:shareGroup];
+    
+#endif
+}
+
 Context::Context()
 :_curShaderProgram(0)
 ,isCapturingFrame(false)
@@ -52,7 +73,6 @@ Context::Context()
     _eglContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3 sharegroup:shareGroup];
 
     _eglOfflinerenderContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3 sharegroup:shareGroup];
-    _eglUpipeContext = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3 sharegroup:shareGroup];
     
     NSDictionary * cacheAttributes = @{ (NSString *)kCVOpenGLESTextureCacheMaximumTextureAgeKey: @(0.0) };
 
@@ -92,7 +112,6 @@ Context::~Context() {
     _eglContextIO = NULL;
     _eglContext = NULL;
     _eglOfflinerenderContext = NULL;
-    _eglUpipeContext = NULL;
     shareGroup = NULL;
     
 #endif

mediapipe/render/core/Context.hpp

@@ -39,6 +39,9 @@ NS_GI_BEGIN
 class Filter;
 class Context {
 public:
+#if defined(__APPLE__)
+    Context(EAGLContext *context);
+#endif
 
     Context();
     ~Context();
@@ -66,7 +69,6 @@ public:
 
 #if defined(__APPLE__)
     EAGLContext* getEglContext() const { return _eglContext; };
-    EAGLContext* getEglUpipeContext() const { return _eglUpipeContext; };
     void renewOfflineRenderContext();
     void presentBufferForDisplay();
 #else
@@ -126,11 +128,9 @@ private:
             EGLDisplay _eglDisplay;
         };
 #endif
-
     EAGLContext* _eglContext;
     EAGLContext* _eglOfflinerenderContext;
     EAGLContext* _eglContextIO;
-    EAGLContext* _eglUpipeContext;
 };
 
 NS_GI_END

mediapipe/render/core/Filter.hpp

@@ -248,7 +248,7 @@ protected:
     void generateVBOBuffers();
     bool _enable = true;
     bool _forceEnable = false;
-    Quaramera::Mat4 _mvp_matrix;
+    Opipe::Mat4 _mvp_matrix;
     Vector2 _scaleResolution = Vector2(0.0, 0.0);
     bool _useScaleResolution = false;
     

mediapipe/render/core/GLProgram.cpp

@@ -190,7 +190,7 @@ namespace Opipe {
         }
     }
     
-    void GLProgram::setUniformValue(const std::string &uniformName, Quaramera::Mat4 value) {
+    void GLProgram::setUniformValue(const std::string &uniformName, Opipe::Mat4 value) {
         getContext()->setActiveShaderProgram(this);
         GLuint location = getUniformLocation(uniformName);
         if (location != -1) {
@@ -259,7 +259,7 @@ namespace Opipe {
         }
     }
     
-    void GLProgram::setUniformValue(int uniformLocation, Quaramera::Mat4 value) {
+    void GLProgram::setUniformValue(int uniformLocation, Opipe::Mat4 value) {
         getContext()->setActiveShaderProgram(this);
         CHECK_GL(glUniformMatrix4fv(uniformLocation, 1, GL_FALSE, (GLfloat *) &value));
     }

mediapipe/render/core/GLProgram.hpp

@@ -64,7 +64,7 @@ public:
     void setUniformValue(const std::string& uniformName, Vector2 value);
     void setUniformValue(const std::string& uniformName, Vector4 value);
     void setUniformValue(const std::string& uniformName, Matrix3 value);
-    void setUniformValue(const std::string& uniformName, Quaramera::Mat4 value);
+    void setUniformValue(const std::string& uniformName, Opipe::Mat4 value);
     
     void setUniformValue(int uniformLocation, int value);
     void setUniformValue(int uniformLocation, int count, int* value, int valueSize = 1);
@@ -73,7 +73,7 @@ public:
     void setUniformValue(int uniformLocation, Vector2 value);
     void setUniformValue(int uniformLocation, Vector4 value);
     void setUniformValue(int uniformLocation, Matrix3 value);
-    void setUniformValue(int uniformLocation, Quaramera::Mat4 value);
+    void setUniformValue(int uniformLocation, Opipe::Mat4 value);
 
     Context *getContext();
 private:

mediapipe/render/core/GaussianBlurFilter.cpp

@@ -0,0 +1,94 @@
+/*
+ * GPUImage-x
+ *
+ * Copyright (C) 2017 Yijin Wang, Yiqian Wang
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cmath>
+#include "GaussianBlurFilter.hpp"
+#include "util.h"
+
+NS_GI_BEGIN
+
+REGISTER_FILTER_CLASS(GaussianBlurFilter)
+
+GaussianBlurFilter::GaussianBlurFilter(Context *context) : FilterGroup(context)
+,_hBlurFilter(0)
+,_vBlurFilter(0)
+{
+}
+
+GaussianBlurFilter::~GaussianBlurFilter() {
+    if (_hBlurFilter) {
+        _hBlurFilter->release();
+        _hBlurFilter = 0;
+    }
+    
+    if (_vBlurFilter) {
+        _vBlurFilter->release();
+        _vBlurFilter = 0;
+    }
+    
+}
+
+GaussianBlurFilter* GaussianBlurFilter::create(Context *context, int radius/* = 4*/, float sigma/* = 2.0*/, float multiplier) {
+    GaussianBlurFilter* ret = new (std::nothrow) GaussianBlurFilter(context);
+    if (ret && !ret->init(context, radius, sigma, multiplier)) {
+        delete ret;
+        ret = 0;
+    }
+    return ret;
+}
+
+bool GaussianBlurFilter::init(Context *context, int radius, float sigma, float multiplier) {
+    if (!FilterGroup::init(context)) {
+        return false;
+    }
+    
+    _hBlurFilter = GaussianBlurMonoFilter::create(context, GaussianBlurMonoFilter::HORIZONTAL, radius, sigma, multiplier);
+    _vBlurFilter = GaussianBlurMonoFilter::create(context, GaussianBlurMonoFilter::VERTICAL, radius, sigma, multiplier);
+    _hBlurFilter->addTarget(_vBlurFilter);
+    addFilter(_hBlurFilter);
+    
+    registerProperty("radius", 4, "", [this](int& radius){
+        setRadius(radius);
+    });
+    
+    registerProperty("sigma", 2.0, "", [this](float& sigma){
+        setSigma(sigma);
+    });
+    
+    return true;
+}
+
+void GaussianBlurFilter::setRadius(int radius) {
+    _hBlurFilter->setRadius(radius);
+    _vBlurFilter->setRadius(radius);
+}
+
+void GaussianBlurFilter::setSigma(float sigma) {
+    _hBlurFilter->setSigma(sigma);
+    _vBlurFilter->setSigma(sigma);
+}
+
+void GaussianBlurFilter::setSigma_h(float sigma) {
+    _hBlurFilter->setSigma(sigma);
+}
+
+void GaussianBlurFilter::setSigma_v(float sigma) {
+    _vBlurFilter->setSigma(sigma);
+}
+
+NS_GI_END

mediapipe/render/core/GaussianBlurFilter.hpp

@@ -0,0 +1,50 @@
+/*
+ * GPUImage-x
+ *
+ * Copyright (C) 2017 Yijin Wang, Yiqian Wang
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef GaussianBlurFilter_hpp
+#define GaussianBlurFilter_hpp
+
+#include "GPUImageMacros.h"
+#include "FilterGroup.hpp"
+#include "GaussianBlurMonoFilter.hpp"
+
+NS_GI_BEGIN
+
+class GaussianBlurFilter : public FilterGroup {
+public:
+    virtual ~GaussianBlurFilter();
+    
+    static GaussianBlurFilter* create(Context *context, int radius = 4, float sigma = 2.0, float multiplier = 1.0);
+    bool init(Context *context, int radius, float sigma, float multiplier);
+    void setRadius(int radius);
+    void setSigma(float sigma);
+    void setSigma_h(float sigma);
+    void setSigma_v(float sigma);
+    
+protected:
+    GaussianBlurFilter(Context *context);
+    
+private:
+    GaussianBlurMonoFilter* _hBlurFilter;
+    GaussianBlurMonoFilter* _vBlurFilter;
+};
+
+
+NS_GI_END
+
+#endif /* GaussianBlurFilter_hpp */

mediapipe/render/core/GaussianBlurMonoFilter.cpp

@@ -0,0 +1,330 @@
+/*
+ * GPUImage-x
+ *
+ * Copyright (C) 2017 Yijin Wang, Yiqian Wang
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cmath>
+#include "GaussianBlurMonoFilter.hpp"
+#include "util.h"
+
+NS_GI_BEGIN
+
+REGISTER_FILTER_CLASS(GaussianBlurMonoFilter)
+
+GaussianBlurMonoFilter::GaussianBlurMonoFilter(Context *context, Type type/* = HORIZONTAL*/) : Filter(context)
+,_type(type)
+,_radius(4)
+,_sigma(2.0)
+,_multiplier(1.0)
+{
+}
+
+GaussianBlurMonoFilter* GaussianBlurMonoFilter::create(Context *context, Type type/* = HORIZONTAL*/, int radius/* = 4*/, float sigma/* = 2.0*/, float multiplier) {
+    GaussianBlurMonoFilter* ret = new (std::nothrow) GaussianBlurMonoFilter(context, type);
+    if (ret && !ret->init(context, radius, sigma, multiplier)) {
+        delete ret;
+        ret = 0;
+    }
+    return ret;
+}
+
+bool GaussianBlurMonoFilter::init(Context *context, int radius, float sigma, float multiplier) {
+    if (Filter::initWithShaderString(context, _generateOptimizedVertexShaderString(radius, sigma), _generateOptimizedFragmentShaderString(radius, sigma))) {
+        return true;
+    }
+    return false;
+}
+
+void GaussianBlurMonoFilter::setRadius(int radius) {
+    if (radius == _radius) return;
+    
+    _radius = radius;
+    
+    if (_filterProgram) {
+        delete _filterProgram;
+        _filterProgram = 0;
+    }
+    initWithShaderString(_context, _generateOptimizedVertexShaderString(_radius, _sigma), _generateOptimizedFragmentShaderString(_radius, _sigma));
+}
+
+void GaussianBlurMonoFilter::setSigma(float sigma) {
+    if (sigma == _sigma) return;
+    
+    _sigma = round(sigma);
+    
+    int calculatedSampleRadius = 0;
+    if (_sigma >= 1) // Avoid a divide-by-zero error here
+    {
+        // Calculate the number of pixels to sample from by setting a bottom limit for the contribution of the outermost pixel
+        float minimumWeightToFindEdgeOfSamplingArea = 1.0/256.0;
+        calculatedSampleRadius = floor(sqrt(-2.0 * pow(_sigma, 2.0) * log(minimumWeightToFindEdgeOfSamplingArea * sqrt(2.0 * M_PI * pow(_sigma, 2.0))) ));
+        calculatedSampleRadius += calculatedSampleRadius % 2; // There's nothing to gain from handling odd radius sizes, due to the optimizations I use
+    }
+    _radius = calculatedSampleRadius;
+    
+    if (_filterProgram) {
+        delete _filterProgram;
+        _filterProgram = 0;
+    }
+    initWithShaderString(_context, _generateOptimizedVertexShaderString(_radius, _sigma), _generateOptimizedFragmentShaderString(_radius, _sigma));
+}
+
+bool GaussianBlurMonoFilter::proceed(float frameTime, bool bUpdateTargets/* = true*/) {
+
+    RotationMode inputRotation = _inputFramebuffers.begin()->second.rotationMode;
+
+    if (rotationSwapsSize(inputRotation))
+    {
+        if (_type == HORIZONTAL) {
+            _filterProgram->setUniformValue("texelWidthOffset", (float)0.0);
+            _filterProgram->setUniformValue("texelHeightOffset", (float)(_multiplier / _framebuffer->getWidth()));
+        } else {
+            _filterProgram->setUniformValue("texelWidthOffset", (float)(_multiplier / _framebuffer->getHeight()));
+            _filterProgram->setUniformValue("texelHeightOffset", (float)0.0);
+        }
+    } else {
+        if (_type == HORIZONTAL) {
+            _filterProgram->setUniformValue("texelWidthOffset", (float)(_multiplier / _framebuffer->getWidth()));
+            _filterProgram->setUniformValue("texelHeightOffset", (float)0.0);
+        } else {
+            _filterProgram->setUniformValue("texelWidthOffset", (float)0.0);
+            _filterProgram->setUniformValue("texelHeightOffset", (float)(_multiplier / _framebuffer->getHeight()));
+        }
+    }
+    return Filter::proceed(frameTime, bUpdateTargets);
+}
+
+std::string GaussianBlurMonoFilter::_generateVertexShaderString(int radius, float sigma) {
+    if (radius < 1 || sigma <= 0.0)
+    {
+        return kDefaultVertexShader;
+    }
+
+    std::string shaderStr =
+            str_format("\
+        attribute vec4 position;\n\
+        attribute vec4 texCoord;\n\
+        uniform float texelWidthOffset;\n\
+        uniform float texelHeightOffset;\n\
+        varying vec2 blurCoordinates[%d];\n\
+        void main()\n\
+        {\n\
+            gl_Position = position;\n\
+            vec2 texelSpacing = vec2(texelWidthOffset, texelHeightOffset);\n\
+        ", radius * 2 + 1);
+
+    for (int i = 0; i < radius * 2 + 1; ++i) {
+        int offsetFromCenter = i - radius;
+        if (offsetFromCenter == 0) {
+            shaderStr = shaderStr + str_format("blurCoordinates[%d] = texCoord.xy;\n", i);
+        } else {
+            shaderStr = shaderStr + str_format("blurCoordinates[%d] = texCoord.xy + texelSpacing * (%f);\n", i, (float)offsetFromCenter);
+        }
+    }
+
+    shaderStr += "}\n";
+
+    return shaderStr;
+}
+
+std::string GaussianBlurMonoFilter::_generateFragmentShaderString(int radius, float sigma) {
+    if (radius < 1 || sigma <= 0.0)
+    {
+        return kDefaultFragmentShader;
+    }
+
+    float* standardGaussianWeights = new float[radius + 1];
+    float sumOfWeights = 0.0;
+    for (int i = 0; i < radius + 1; ++i)
+    {
+        standardGaussianWeights[i] = (1.0 / sqrt(2.0 * PI * pow(sigma, 2.0))) * exp(-pow(i, 2.0) / (2.0 * pow(sigma, 2.0)));
+
+        if (i == 0)
+        {
+            sumOfWeights += standardGaussianWeights[i];
+        } else {
+            sumOfWeights += 2.0 * standardGaussianWeights[i];
+        }
+    }
+    for (int i = 0; i < radius + 1; ++i)
+    {
+        standardGaussianWeights[i] = standardGaussianWeights[i] / sumOfWeights;
+    }
+
+    std::string shaderStr =
+            str_format("\
+           uniform sampler2D colorMap;\n\
+           varying highp vec2 blurCoordinates[%d];\n\
+           void main()\n\
+           {\n\
+               gl_FragColor = vec4(0.0);\n", radius * 2 + 1);
+    for (int i = 0; i < radius * 2 + 1; ++i) {
+        int offsetFromCenter = i - radius;
+        shaderStr += str_format("gl_FragColor += texture2D(colorMap, blurCoordinates[%d]) * %f;\n", i, standardGaussianWeights[abs(offsetFromCenter)]);
+    }
+    shaderStr += "}";
+
+    delete[] standardGaussianWeights; standardGaussianWeights = 0;
+
+    return shaderStr;
+}
+
+std::string GaussianBlurMonoFilter::_generateOptimizedVertexShaderString(int radius, float sigma)
+{
+    if (radius < 1 || sigma <= 0.0)
+    {
+        return kDefaultVertexShader;
+    }
+    
+    // 1. generate the normal Gaussian weights for a given sigma
+    float* standardGaussianWeights = new float[radius + 1];
+    float sumOfWeights = 0.0;
+    for (int i = 0; i < radius + 1; ++i)
+    {
+        standardGaussianWeights[i] = (1.0 / sqrt(2.0 * M_PI * pow(sigma, 2.0))) * exp(-pow(i, 2.0) / (2.0 * pow(sigma, 2.0)));
+        if (i == 0)
+            sumOfWeights += standardGaussianWeights[i];
+        else
+            sumOfWeights += 2.0 * standardGaussianWeights[i];
+    }
+    
+    // 2. normalize these weights to prevent the clipping of the Gaussian curve at the end of the discrete samples from reducing luminance
+    for (int i = 0; i < radius + 1; ++i)
+    {
+        standardGaussianWeights[i] = standardGaussianWeights[i] / sumOfWeights;
+    }
+    
+    // 3. From these weights we calculate the offsets to read interpolated values from
+    int numberOfOptimizedOffsets = fmin(radius / 2 + (radius % 2), 7);
+    float* optimizedGaussianOffsets = new float[numberOfOptimizedOffsets];
+    
+    for (int i = 0; i < numberOfOptimizedOffsets; ++i)
+    {
+        GLfloat firstWeight = standardGaussianWeights[i * 2 + 1];
+        GLfloat secondWeight = standardGaussianWeights[i * 2 + 2];
+        
+        GLfloat optimizedWeight = firstWeight + secondWeight;
+        
+        optimizedGaussianOffsets[i] = (firstWeight * (i * 2 + 1) + secondWeight * (i * 2 + 2)) / optimizedWeight;
+    }
+    
+    std::string shaderStr =
+    str_format("\
+               attribute vec4 position;\n\
+               attribute vec4 texCoord;\n\
+               uniform float texelWidthOffset;\n\
+               uniform float texelHeightOffset;\n\
+               varying highp vec2 blurCoordinates[%d];\n\
+               void main()\n\
+               {\n\
+               gl_Position = position;\n\
+               vec2 texelSpacing = vec2(texelWidthOffset, texelHeightOffset);\n\
+               ", numberOfOptimizedOffsets * 2 + 1);
+
+    shaderStr = shaderStr + str_format("blurCoordinates[0] = texCoord.xy;\n");
+    for (int i = 0; i < numberOfOptimizedOffsets; ++i) {
+        shaderStr = shaderStr + str_format(
+           "blurCoordinates[%d] = texCoord.xy + texelSpacing * (%f);\n\
+            blurCoordinates[%d] = texCoord.xy - texelSpacing * (%f);",
+            i * 2 + 1,
+            optimizedGaussianOffsets[i],
+            i * 2 + 2,
+            optimizedGaussianOffsets[i]);
+    }
+    
+    shaderStr += "}\n";
+    
+    delete[] standardGaussianWeights;
+    delete[] optimizedGaussianOffsets;
+    
+    return shaderStr;
+}
+
+std::string GaussianBlurMonoFilter::_generateOptimizedFragmentShaderString(int radius, float sigma)
+{
+    if (radius < 1 || sigma <= 0.0)
+    {
+        return kDefaultFragmentShader;
+    }
+    
+    // 1. generate the normal Gaussian weights for a given sigma
+    float* standardGaussianWeights = new float[radius + 1];
+    float sumOfWeights = 0.0;
+    for (int i = 0; i < radius + 1; ++i)
+    {
+        standardGaussianWeights[i] = (1.0 / sqrt(2.0 * M_PI * pow(sigma, 2.0))) * exp(-pow(i, 2.0) / (2.0 * pow(sigma, 2.0)));
+        if (i == 0)
+            sumOfWeights += standardGaussianWeights[i];
+        else
+            sumOfWeights += 2.0 * standardGaussianWeights[i];
+    }
+    
+    // 2. normalize these weights to prevent the clipping of the Gaussian curve at the end of the discrete samples from reducing luminance
+    for (int i = 0; i < radius + 1; ++i)
+    {
+        standardGaussianWeights[i] = standardGaussianWeights[i] / sumOfWeights;
+    }
+    
+    // 3. From these weights we calculate the offsets to read interpolated values from
+    int trueNumberOfOptimizedOffsets = radius / 2 + (radius % 2);
+    int numberOfOptimizedOffsets = fmin(trueNumberOfOptimizedOffsets, 7);
+
+    std::string shaderStr =
+    str_format("\
+               uniform sampler2D colorMap;\n\
+               uniform highp float texelWidthOffset;\n\
+               uniform highp float texelHeightOffset;\n\
+               varying highp vec2 blurCoordinates[%d];\n\
+               void main()\n\
+               {\n\
+               gl_FragColor = vec4(0.0);\n", numberOfOptimizedOffsets * 2 + 1);
+    
+    shaderStr += str_format("gl_FragColor += texture2D(colorMap, blurCoordinates[0]) * %f;\n", standardGaussianWeights[0]);
+    for (int i = 0; i < numberOfOptimizedOffsets; ++i) {
+        float firstWeight = standardGaussianWeights[i * 2 + 1];
+        float secondWeight = standardGaussianWeights[i * 2 + 2];
+        float optimizedWeight = firstWeight + secondWeight;
+        
+        shaderStr += str_format("gl_FragColor += texture2D(colorMap, blurCoordinates[%d]) * %f;\n", i * 2 + 1, optimizedWeight);
+        shaderStr += str_format("gl_FragColor += texture2D(colorMap, blurCoordinates[%d]) * %f;\n", i * 2 + 2, optimizedWeight);
+    }
+    
+    // If the number of required samples exceeds the amount we can pass in via varyings, we have to do dependent texture reads in the fragment shader
+    if (trueNumberOfOptimizedOffsets > numberOfOptimizedOffsets)
+    {
+        shaderStr += str_format("highp vec2 texelSpacing = vec2(texelWidthOffset, texelHeightOffset);\n");
+        
+        for (int i = numberOfOptimizedOffsets; i < trueNumberOfOptimizedOffsets; i++)
+        {
+            float firstWeight = standardGaussianWeights[i * 2 + 1];
+            float secondWeight = standardGaussianWeights[i * 2 + 2];
+            
+            float optimizedWeight = firstWeight + secondWeight;
+            float optimizedOffset = (firstWeight * (i * 2 + 1) + secondWeight * (i * 2 + 2)) / optimizedWeight;
+            
+            shaderStr += str_format("gl_FragColor += texture2D(colorMap, blurCoordinates[0] + texelSpacing * %f) * %f;\n", optimizedOffset, optimizedWeight);
+            
+            shaderStr += str_format("gl_FragColor += texture2D(colorMap, blurCoordinates[0] - texelSpacing * %f) * %f;\n", optimizedOffset, optimizedWeight);
+        }
+    }
+
+    shaderStr += "}";
+    
+    delete[] standardGaussianWeights; standardGaussianWeights = 0;
+    return shaderStr;
+}
+
+NS_GI_END

mediapipe/render/core/GaussianBlurMonoFilter.hpp

@@ -0,0 +1,56 @@
+/*
+ * GPUImage-x
+ *
+ * Copyright (C) 2017 Yijin Wang, Yiqian Wang
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef GaussianBlurMonoFilter_hpp
+#define GaussianBlurMonoFilter_hpp
+
+#include "GPUImageMacros.h"
+#include "FilterGroup.hpp"
+
+NS_GI_BEGIN
+
+class GaussianBlurMonoFilter : public Filter {
+public:
+    enum Type {HORIZONTAL, VERTICAL};
+    
+    static GaussianBlurMonoFilter* create(Context *context, Type type = HORIZONTAL, int radius = 4, float sigma = 2.0, float multiplier = 1.0);
+    bool init(Context *context, int radius, float sigma, float multiplier);
+    
+    void setRadius(int radius);
+    void setSigma(float sigma);
+    
+    virtual bool proceed(float frameTime = 0, bool bUpdateTargets = true) override;
+protected:
+    GaussianBlurMonoFilter(Context *context, Type type = HORIZONTAL);
+    Type _type;
+    int _radius;
+    float _sigma;
+    float _multiplier;
+
+private:
+    virtual std::string _generateVertexShaderString(int radius, float sigma);
+    virtual std::string _generateFragmentShaderString(int radius, float sigma);
+    
+    virtual std::string _generateOptimizedVertexShaderString(int radius, float sigma);
+    virtual std::string _generateOptimizedFragmentShaderString(int radius, float sigma);
+};
+
+
+NS_GI_END
+
+#endif /* GaussianBlurMonoFilter_hpp */

mediapipe/render/core/LUTFilter.cpp

@@ -0,0 +1,77 @@
+#include "LUTFilter.hpp"
+
+namespace Opipe
+{
+    const std::string kLookupFragmentShaderString = SHADER_STRING(
+        varying highp vec2 vTexCoord;
+        varying highp vec2 vTexCoord1; // TODO: This is not used
+
+        uniform sampler2D colorMap;
+        uniform sampler2D colorMap1; // lookup texture
+        uniform lowp float step;
+
+        void main() {
+            highp vec4 textureColor = texture2D(colorMap, vTexCoord);
+
+            highp float blueColor = textureColor.b * 63.0;
+
+            highp vec2 quad1;
+            quad1.y = floor(floor(blueColor) / 8.0);
+            quad1.x = floor(blueColor) - (quad1.y * 8.0);
+
+            highp vec2 quad2;
+            quad2.y = floor(ceil(blueColor) / 8.0);
+            quad2.x = ceil(blueColor) - (quad2.y * 8.0);
+
+            highp vec2 texPos1;
+            texPos1.x = (quad1.x * 0.125) + 0.5 / 512.0 + ((0.125 - 1.0 / 512.0) * textureColor.r);
+            texPos1.y = (quad1.y * 0.125) + 0.5 / 512.0 + ((0.125 - 1.0 / 512.0) * textureColor.g);
+
+            highp vec2 texPos2;
+            texPos2.x = (quad2.x * 0.125) + 0.5 / 512.0 + ((0.125 - 1.0 / 512.0) * textureColor.r);
+            texPos2.y = (quad2.y * 0.125) + 0.5 / 512.0 + ((0.125 - 1.0 / 512.0) * textureColor.g);
+
+            lowp vec4 newColor1 = texture2D(colorMap1, texPos1);
+            lowp vec4 newColor2 = texture2D(colorMap1, texPos2);
+
+            lowp vec4 newColor = mix(newColor1, newColor2, fract(blueColor));
+            lowp vec3 finalColor = mix(textureColor.rgb, newColor.rgb, step);
+
+            gl_FragColor = vec4(finalColor, textureColor.w);
+        });
+
+    LUTFilter::LUTFilter(Opipe::Context *context) : Filter(context), _step(1.0)
+    {
+    }
+
+    Opipe::LUTFilter *LUTFilter::create(Opipe::Context *context)
+    {
+        LUTFilter *ret = new (std::nothrow) LUTFilter(context);
+        if (!ret || !ret->init(context))
+        {
+            delete ret;
+            ret = 0;
+        }
+        return ret;
+    }
+
+    bool LUTFilter::init(Opipe::Context *context)
+    {
+        if (!Opipe::Filter::initWithFragmentShaderString(context, kLookupFragmentShaderString, 2))
+        {
+            return false;
+        }
+        return true;
+    }
+
+    void LUTFilter::setStep(float step)
+    {
+        _step = step;
+    }
+
+    bool LUTFilter::proceed(float frameTime, bool bUpdateTargets /* = true*/)
+    {
+        _filterProgram->setUniformValue("step", _step);
+        return Filter::proceed(frameTime, bUpdateTargets);
+    }
+}

mediapipe/render/core/LUTFilter.hpp

@@ -0,0 +1,26 @@
+#ifndef LookUpFilter_hpp
+#define LookUpFilter_hpp
+
+#include "Filter.hpp"
+#include "Context.hpp"
+
+namespace Opipe
+{
+
+    class LUTFilter : public Filter
+    {
+    public:
+        static LUTFilter *create(Context *context);
+        bool init(Context *context);
+        void setStep(float step);
+        virtual bool proceed(float frameTime = 0, bool bUpdateTargets = true) override;
+
+    public:
+        LUTFilter(Context *context);
+        ~LUTFilter(){};
+        float _step;
+    };
+
+}
+
+#endif

mediapipe/render/core/OlaContext.cpp

@@ -0,0 +1,31 @@
+#include "OlaContext.hpp"
+#include "Context.hpp"
+
+namespace Opipe {
+
+   
+    OlaContext::OlaContext() {
+        _currentContext = new Context();
+    }
+
+    OlaContext::~OlaContext() {
+
+    }
+
+    #if defined(__APPLE__)
+     OlaContext::OlaContext(EAGLContext *context) {
+        _currentContext = new Context(context);
+    }
+
+
+    EAGLContext* OlaContext::currentContext() {
+        return _currentContext->getEglContext();
+    }
+    
+    #else
+    #endif
+
+    Context* OlaContext::glContext() {
+        return _currentContext;
+    }
+}

mediapipe/render/core/OlaContext.hpp

@@ -0,0 +1,37 @@
+#ifndef OlaContext_hpp
+#define OlaContext_hpp
+
+#if defined(__APPLE__)
+#import <OpenGLES/EAGL.h>
+#import <OpenGLES/ES3/gl.h>
+#import <CoreVideo/CoreVideo.h>
+#else
+#include <EGL/egl.h>
+#endif
+
+namespace Opipe {
+    class Context;
+    class OlaContext {
+        public:
+#if defined(__APPLE__)
+        OlaContext(EAGLContext *context);
+#endif
+        OlaContext();
+        ~OlaContext();
+
+        #if defined(__APPLE__)
+        EAGLContext* currentContext();
+        #else
+        EGLContext* currentContext();
+        void initEGLContext(EGLContext shareContext);
+        #endif
+
+        Context* glContext();
+
+        private:
+        Context *_currentContext = nullptr;
+
+    };
+}
+
+#endif

mediapipe/render/core/OlaShareTextureFilter.cpp

@@ -0,0 +1,135 @@
+//
+//  OlaShareTextureFilter.cpp
+//  AREmotion
+//
+//  Created by Renzhu Wang on 07/12/2017.
+//  Copyright © 2022 olachat. All rights reserved.
+//
+
+#include "OlaShareTextureFilter.hpp"
+
+
+namespace Opipe {
+    
+    const std::string kOnScreenFragmentShaderString = SHADER_STRING
+    (
+     varying highp vec2 vTexCoord;
+     uniform sampler2D colorMap;
+     void main() {
+        lowp
+        vec4 textureColor = texture2D(colorMap, vTexCoord);
+        gl_FragColor = vec4(textureColor.rgb, textureColor.a);
+    });
+    
+    OlaShareTextureFilter::OlaShareTextureFilter(Context *context) : Opipe::Filter(context), targetTextureId(-1) {}
+    
+    OlaShareTextureFilter *OlaShareTextureFilter::create(Context *context) {
+        OlaShareTextureFilter *ret = new(std::nothrow) OlaShareTextureFilter(context);
+        if (!ret || !ret->init(context)) {
+            delete ret;
+            ret = 0;
+        }
+        return ret;
+    }
+    
+    OlaShareTextureFilter *
+    OlaShareTextureFilter::create(Context *context, GLuint targetTextureId, TextureAttributes attributes) {
+        auto *ret = new(std::nothrow) OlaShareTextureFilter(context);
+        if (!ret || !ret->init(context)) {
+            delete ret;
+            ret = nullptr;
+        }
+        ret->targetTextureId = targetTextureId;
+        ret->targetTextureAttr = attributes;
+        return ret;
+    }
+    
+    bool OlaShareTextureFilter::init(Context *context) {
+        if (!Opipe::Filter::initWithFragmentShaderString(context, kOnScreenFragmentShaderString, 1)) {
+            return false;
+        }
+        return true;
+    }
+    
+    
+    bool OlaShareTextureFilter::proceed(float frameTime, bool bUpdateTargets/* = true*/) {
+        if (!_filterProgram->isValid()) {
+            delete _filterProgram;
+            _filterProgram = 0;
+            _filterProgram = GLProgram::createByShaderString(_context,
+                                                             kDefaultVertexShader,
+                                                             kOnScreenFragmentShaderString);
+        }
+        
+        return Filter::proceed(frameTime, bUpdateTargets);
+    }
+    
+    OlaShareTextureFilter::~OlaShareTextureFilter() noexcept {
+        //不用去释放 交给Context 去Purge
+        if (_targetFramebuffer) {
+            delete _framebuffer;
+            _framebuffer = 0;
+        } else {
+            _framebuffer = 0;
+        }
+    }
+    
+    void OlaShareTextureFilter::updateTargetId(GLuint targetId) {
+        targetTextureId = targetId;
+    }
+    
+    void OlaShareTextureFilter::update(float frameTime) {
+        if (_inputFramebuffers.empty()) return;
+        
+        Framebuffer *firstInputFramebuffer = _inputFramebuffers.begin()->second.frameBuffer;
+        RotationMode firstInputRotation = _inputFramebuffers.begin()->second.rotationMode;
+        if (!firstInputFramebuffer) return;
+        
+        int rotatedFramebufferWidth = firstInputFramebuffer->getWidth();
+        int rotatedFramebufferHeight = firstInputFramebuffer->getHeight();
+        if (rotationSwapsSize(firstInputRotation)) {
+            rotatedFramebufferWidth = firstInputFramebuffer->getHeight();
+            rotatedFramebufferHeight = firstInputFramebuffer->getWidth();
+        }
+        
+        if (_framebufferScale != 1.0) {
+            rotatedFramebufferWidth = int(rotatedFramebufferWidth * _framebufferScale);
+            rotatedFramebufferHeight = int(rotatedFramebufferHeight * _framebufferScale);
+        }
+        
+
+        if (_framebuffer != nullptr && (_framebuffer->getWidth() != rotatedFramebufferWidth ||
+                                        _framebuffer->getHeight() != rotatedFramebufferHeight)) {
+            _framebuffer = nullptr;
+        }
+        
+        if (_framebuffer == nullptr || (_framebuffer && _framebuffer->getTexture() != targetTextureId)) {
+            if (_framebuffer) {
+                delete _framebuffer;
+                _framebuffer = 0;
+            }
+            if (targetTextureId == -1) {
+                _framebuffer = getContext()->getFramebufferCache()->
+                fetchFramebuffer(_context,
+                                 rotatedFramebufferWidth,
+                                 rotatedFramebufferHeight);
+                _framebuffer->lock();
+                targetTextureId = _framebuffer->getTexture();
+            } else {
+                _framebuffer = getContext()->getFramebufferCache()->
+                fetchFramebufferUseTextureId(_context,
+                                             rotatedFramebufferWidth,
+                                             rotatedFramebufferHeight,
+                                             targetTextureId,
+                                             false,
+                                             targetTextureAttr);
+                _framebuffer->lock();
+                _targetFramebuffer = true;
+            }
+            
+        }
+        
+        proceed(frameTime);
+    }
+    
+}

mediapipe/render/core/OlaShareTextureFilter.hpp

@@ -0,0 +1,38 @@
+//
+//  OlaShareTextureFilter.hpp
+//  AREmotion
+//
+//  Created by Renzhu Wang on 07/12/2017.
+//  Copyright © 2022 olachat. All rights reserved.
+//
+
+#ifndef OlaShareTextureFilter_hpp
+#define OlaShareTextureFilter_hpp
+
+#include "Filter.hpp"
+#include "Context.hpp"
+
+
+namespace Opipe {
+    class OlaShareTextureFilter : public Opipe::Filter {
+    public:
+        static OlaShareTextureFilter* create(Opipe::Context *context);
+        static OlaShareTextureFilter* create(Opipe::Context *context, GLuint targetTextureId, TextureAttributes attributes);
+        bool init(Opipe::Context *context);
+        virtual bool proceed(float frameTime = 0, bool bUpdateTargets = true) override;
+        virtual void update(float frameTime) override;
+        void updateTargetId(GLuint targetId);
+        
+        GLuint targetTextureId;
+        
+    public:
+        OlaShareTextureFilter(Opipe::Context *context);
+        virtual ~OlaShareTextureFilter() noexcept;
+        
+        TextureAttributes targetTextureAttr = Framebuffer::defaultTextureAttribures;
+    private:
+        bool _targetFramebuffer = false;
+    };
+}
+
+#endif /* OlaShareTextureFilter_hpp */

mediapipe/render/core/OpipeDispatch.cpp

@@ -1,6 +1,6 @@
 //
 //  OpipeDispatch.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by wangrenzhu2021 on 2021/12/14.
 //  Copyright © 2021 ola. All rights reserved.

mediapipe/render/core/OpipeDispatch.hpp

@@ -1,6 +1,6 @@
 //
 //  OpipeDispatch.hpp
-//  Quaramera
+//  Opipe
 //
 //  Created by wangrenzhu2021 on 2021/12/14.
 //  Copyright © 2021 ola. All rights reserved.

mediapipe/render/core/math/mat4.cpp

@@ -1,6 +1,6 @@
 //
 //  Mat4.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -10,7 +10,7 @@
 #include "math_utils.hpp"
 #include <cstring>
 
-namespace Quaramera {
+namespace Opipe {
     
     static const int MATRIX_SIZE = (sizeof(float) * 16);
     

mediapipe/render/core/math/mat4.hpp

@@ -1,6 +1,6 @@
 //
 //  mat4.h
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -12,7 +12,7 @@
 #include "vec3.hpp"
 #include "vec4.hpp"
 
-namespace Quaramera {
+namespace Opipe {
     
 class Mat4
 {

mediapipe/render/core/math/mat4.inl

@@ -8,7 +8,7 @@
 
 #include "mat4.hpp"
 
-namespace Quaramera {
+namespace Opipe {
     
     inline Mat4 Mat4::operator+(const Mat4& mat) const
     {

mediapipe/render/core/math/math_utils.cpp

@@ -1,6 +1,6 @@
 //
 //  math_utils.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -9,7 +9,7 @@
 #include "math_utils.hpp"
 #include <cstring>
 
-namespace Quaramera {
+namespace Opipe {
     
     static const int MATRIX_SIZE = (sizeof(float) * 16);
     

mediapipe/render/core/math/math_utils.hpp

@@ -1,6 +1,6 @@
 //
 //  math_utils.h
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -26,7 +26,7 @@
 #define MATH_FLOAT_EQUAL(src, dst)  (((src) >= (dst) - MATH_EPSILON) && ((src) <= (dst) + MATH_EPSILON))
 
 
-namespace Quaramera {
+namespace Opipe {
     
     class MathUtils
     {

mediapipe/render/core/math/vec2.cpp

@@ -1,6 +1,6 @@
 //
 //  vec2.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -8,7 +8,7 @@
 
 #include "vec2.hpp"
 #include "math_utils.hpp"
-namespace Quaramera {
+namespace Opipe {
     
     float Vec2::angle(const Vec2& v1, const Vec2& v2)
     {

mediapipe/render/core/math/vec2.hpp

@@ -1,6 +1,6 @@
 //
 //  vec2.h
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -12,7 +12,7 @@
 #include <algorithm>
 #include <cmath>
 
-namespace Quaramera {
+namespace Opipe {
     
     inline float clampf(float value, float min_inclusive, float max_inclusive) {
         if (min_inclusive > max_inclusive) {

mediapipe/render/core/math/vec2.inl

@@ -8,7 +8,7 @@
 
 #include "vec2.hpp"
 
-namespace Quaramera {
+namespace Opipe {
     
     inline Vec2::Vec2() : x(0.0f), y(0.0f)
     {

mediapipe/render/core/math/vec3.cpp

@@ -1,6 +1,6 @@
 //
 //  vec3.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -11,7 +11,7 @@
 #include "math_utils.hpp"
 
 
-namespace Quaramera {
+namespace Opipe {
     
     Vec3::Vec3() : x(0.0f), y(0.0f), z(0.0f) {
     }

mediapipe/render/core/math/vec3.hpp

@@ -1,6 +1,6 @@
 //
 //  vec3.h
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -9,7 +9,7 @@
 #ifndef VEC3_H
 #define VEC3_H
 
-namespace Quaramera {
+namespace Opipe {
     
     class Vec3
     {

mediapipe/render/core/math/vec3.inl

@@ -9,7 +9,7 @@
 #include "vec3.hpp"
 #include <cmath>
 
-namespace Quaramera {
+namespace Opipe {
     
     inline bool Vec3::is_zero() const
     {

mediapipe/render/core/math/vec4.cpp

@@ -1,6 +1,6 @@
 //
 //  vec4.cpp
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -10,7 +10,7 @@
 #include "math_utils.hpp"
 #include <cmath>
 
-namespace Quaramera {
+namespace Opipe {
     
     Vec4::Vec4() : x(0.0f), y(0.0f), z(0.0f), w(0.0f)
     {

mediapipe/render/core/math/vec4.hpp

@@ -1,6 +1,6 @@
 //
 //  vec4.h
-//  Quaramera
+//  Opipe
 //
 //  Created by Wang,Renzhu on 2018/11/20.
 //  Copyright © 2018年 Wang,Renzhu. All rights reserved.
@@ -9,7 +9,7 @@
 #ifndef VEC4_H
 #define VEC4_H
 
-namespace Quaramera {
+namespace Opipe {
     class Vec4
     {
     public:

mediapipe/render/core/math/vec4.inl

@@ -8,7 +8,7 @@
 
 #include "vec4.hpp"
 
-namespace Quaramera {
+namespace Opipe {
     
     inline Vec4 Vec4::operator+(const Vec4& v) const
     {

mediapipe/render/module/beauty/BUILD

@@ -30,6 +30,7 @@ FACEUNITY_SRCS = [
 
 FACEUNITY_HDRS = [
         "face_mesh_module.h",
+        "face_mesh_common.h",
         "face_mesh_beauty_render.h",
         "face_mesh_module_imp.h",
 ]
@@ -50,11 +51,13 @@ cc_library(
             "//mediapipe/render/core:core-ios",
             "//mediapipe/graphs/face_mesh:mobile_calculators",
             "//mediapipe/framework/formats:landmark_cc_proto",
+            "//mediapipe/render/module/beauty/filters:BeautyFilters",
          ],
         "//conditions:default": [
             "//mediapipe/render/core:core",
             "//mediapipe/graphs/face_mesh:mobile_calculators",
             "//mediapipe/framework/formats:landmark_cc_proto",
+            "//mediapipe/render/module/beauty/filters:BeautyFilters",
         ],
     }),
     copts = select({
@@ -65,3 +68,10 @@ cc_library(
         "//conditions:default": ["-std=c++17"],
     }),
 )
+
+
+exports_files(
+    srcs = [
+        "whiten.png",
+    ],
+)

mediapipe/render/module/beauty/face_mesh_beauty_render.cc

@@ -1 +1,72 @@
 #include "face_mesh_beauty_render.h"
+
+namespace Opipe
+{
+    FaceMeshBeautyRender::FaceMeshBeautyRender(Context *context)
+    {
+        _context = context;
+        _olaBeautyFilter = OlaBeautyFilter::create(context);
+        _isRendering = false;
+
+        _outputFilter = OlaShareTextureFilter::create(context);
+        _olaBeautyFilter->addTarget(_outputFilter);
+    }
+
+    FaceMeshBeautyRender::~FaceMeshBeautyRender()
+    {
+    }
+
+    void FaceMeshBeautyRender::suspend()
+    {
+        _isRendering = true;
+    }
+
+    void FaceMeshBeautyRender::resume()
+    {
+        _isRendering = false;
+    }
+
+    TextureInfo FaceMeshBeautyRender::renderTexture(TextureInfo inputTexture)
+    {
+        TextureInfo outputTexture;
+
+        if (!_inputFramebuffer)
+        {
+            _inputFramebuffer = new Framebuffer(_context, inputTexture.width, inputTexture.height,
+                                                Framebuffer::defaultTextureAttribures,
+                                                inputTexture.textureId);
+        }
+        else if (_inputFramebuffer->getWidth() != inputTexture.width || _inputFramebuffer->getHeight() != inputTexture.height)
+        {
+            _inputFramebuffer->unlock();
+            delete _inputFramebuffer;
+            _inputFramebuffer = nullptr;
+            _inputFramebuffer = new Framebuffer(_context, inputTexture.width, inputTexture.height,
+                                                Framebuffer::defaultTextureAttribures,
+                                                inputTexture.textureId);
+        }
+
+        return outputTexture;
+    }
+
+    float FaceMeshBeautyRender::getSmoothing()
+    {
+        return _smoothing;
+    }
+
+    float FaceMeshBeautyRender::getWhitening()
+    {
+        return _whitening;
+    }
+
+    void FaceMeshBeautyRender::setSmoothing(float smoothing)
+    {
+        _smoothing = smoothing;
+    }
+
+    void FaceMeshBeautyRender::setWhitening(float whitening)
+    {
+        _whitening = whitening;
+    }
+
+}

mediapipe/render/module/beauty/face_mesh_beauty_render.h

@@ -1,3 +1,46 @@
+#ifndef OPIPE_FaceMeshBeautyRender
+#define OPIPE_FaceMeshBeautyRender
+#include "face_mesh_common.h"
+#include "mediapipe/render/module/beauty/filters/OlaBeautyFilter.hpp"
+#include "mediapipe/render/core/OlaShareTextureFilter.hpp"
+
 namespace Opipe {
+    class FaceMeshBeautyRender {
+        public:
+            FaceMeshBeautyRender(Context *context);
+            ~FaceMeshBeautyRender();
+
+            void suspend();
+
+            void resume();
+
+            TextureInfo renderTexture(TextureInfo inputTexture);
+
+            /// 磨皮
+            float getSmoothing();
+            
+            /// 美白
+            float getWhitening();
+            
+            
+            /// 磨皮
+            /// @param smoothing 磨皮 0.0 - 1.0
+            void setSmoothing(float smoothing);
+            
+            
+            /// 美白
+            /// @param whitening 美白 0.0 - 1.0
+            void setWhitening(float whitening);
+        private:
+            OlaBeautyFilter *_olaBeautyFilter = nullptr;
+            OlaShareTextureFilter *_outputFilter = nullptr;
+            Framebuffer *_inputFramebuffer = nullptr;
+            float _smoothing = 0.0;
+            float _whitening = 0.0;
+            bool _isRendering = false;
+            Context *_context = nullptr;
+
+    };
     
 }
+#endif

mediapipe/render/module/beauty/face_mesh_common.h

@@ -0,0 +1,14 @@
+#ifndef OPIPE_FaceMeshCommon
+#define OPIPE_FaceMeshCommon
+
+#include <stdio.h>
+
+typedef struct {
+        int width;
+        int height;
+        int textureId;
+        int ioSurfaceId; // iOS 专属
+        int64_t frameTime;
+} TextureInfo;
+
+#endif

mediapipe/render/module/beauty/face_mesh_module.h

@@ -1,6 +1,8 @@
 #ifndef OPIPE_FaceMeshModule
 #define OPIPE_FaceMeshModule
 #include <stdio.h>
+#include "mediapipe/render/core/OlaContext.hpp"
+#include "face_mesh_common.h"
 #if defined(__APPLE__)
 #import <OpenGLES/ES3/gl.h>
 #import <OpenGLES/ES3/glext.h>
@@ -10,17 +12,91 @@
 #include <GLES3/gl3ext.h>
 #endif
 
-
 namespace Opipe
 {
+        struct OMat
+        {
+                int width = 0;
+                int height = 0;
+                char *data = 0;
+                int widthStep = 0;
+                int channels = 4; //暂时只支持4
+                bool create = false;
+                OMat()
+                {
+                        channels = 0;
+                }
+
+                OMat(int w, int h, int ws)
+                {
+                        width = w;
+                        height = h;
+                        channels = 4;
+                        widthStep = ws;
+                        data = new char[widthStep * height];
+                        memset(data, 0, sizeof(data));
+                        create = true;
+                }
+
+                OMat(int w, int h)
+                {
+                        width = w;
+                        height = h;
+                        channels = 4;
+                        if (w % 32 != 0)
+                        {
+                                widthStep = ((w / 32) + 1) * 32 * channels;
+                        }
+                        else
+                        {
+                                widthStep = w * channels;
+                        }
+
+                        data = new char[widthStep * height];
+                        memset(data, 0, sizeof(data));
+                        create = true;
+                }
+
+                OMat(int w, int h, char *d)
+                {
+                        width = w;
+                        height = h;
+                        channels = 4;
+                        data = d;
+                        if (w % 32 != 0)
+                        {
+                                widthStep = ((w / 32) + 1) * 32 * channels;
+                        }
+                        else
+                        {
+                                widthStep = w * channels;
+                        }
+                }
+
+                void release()
+                {
+                        if (create)
+                        {
+                                delete data;
+                        }
+                        data = 0;
+                }
+
+                bool empty()
+                {
+                        return data == 0;
+                }
+        };
+
         class FaceMeshModule
         {
         public:
                 FaceMeshModule();
                 virtual ~FaceMeshModule();
 
-        static FaceMeshModule* create();
+                static FaceMeshModule *create();
 
+                virtual OlaContext *currentContext() = 0;
 
                 // 暂停渲染
                 virtual void suspend() = 0;
@@ -34,7 +110,7 @@ namespace Opipe
 
                 virtual void stopModule() = 0;
 
-        virtual GLuint renderTexture(GLuint textureId, int64_t timeStamp, int width, int height) = 0;
+                virtual TextureInfo renderTexture(TextureInfo inputTexture) = 0;
 
 #if defined(__APPLE__)
                 virtual void processVideoFrame(CVPixelBufferRef pixelbuffer, int64_t timeStamp) = 0;

mediapipe/render/module/beauty/face_mesh_module_imp.cc

@@ -1,8 +1,8 @@
 #include "face_mesh_module_imp.h"
-#include "mediapipe/framework/formats/landmark.pb.h"
 
 static const char* kNumFacesInputSidePacket = "num_faces";
 static const char* kLandmarksOutputStream = "multi_face_landmarks";
+static const char* kDetectionsOutputStream = "face_detections";
 static const char* kOutputVideo = "output_video";
 
 namespace Opipe
@@ -23,27 +23,71 @@ namespace Opipe
     }
 #endif
 
+    void FaceMeshCallFrameDelegate::outputPacket(OlaGraph *graph, const mediapipe::Packet &packet, const std::string &streamName) {
+#if defined(__APPLE__)
+        NSLog(@"streamName:%@ ts:%lld 是否有人脸:%@", [NSString stringWithUTF8String:streamName.c_str()],
+              packet.Timestamp().Value(), @(_hasFace));
+#endif
+        if (_imp == nullptr) {
+            return;
+        }
+        
+        
+        if (streamName == kLandmarksOutputStream) {
+            _last_landmark_ts = packet.Timestamp().Value();
+            
+            if (_last_video_ts == _last_landmark_ts) {
+                //有人脸
+                _hasFace = true;
+                const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+                _lastLandmark = multi_face_landmarks[0];
+            }
+        }
+        
+        if (_last_video_ts != _last_landmark_ts) {
+            _hasFace = false;
+        }
+        
+        _last_video_ts = packet.Timestamp().Value();
+        
+        if (_hasFace) {
+            
+            _imp->setLandmark(_lastLandmark);
+        } else {
+            _imp->setLandmark(_emptyLandmark);
+        }
+    }
+
     void FaceMeshCallFrameDelegate::outputPacket(OlaGraph *graph, const mediapipe::Packet &packet,
                                                  MPPPacketType packetType, const std::string &streamName)
     {
 #if defined(__APPLE__)
-        if (streamName == kLandmarksOutputStream) {
+//        if (streamName == kLandmarksOutputStream) {
-          if (packet.IsEmpty()) {
+//          if (packet.IsEmpty()) {
-            NSLog(@"[TS:%lld] No face landmarks", packet.Timestamp().Value());
+//            NSLog(@"[TS:%lld] No face landmarks", packet.Timestamp().Value());
-            return;
+//            return;
-          }
+//          }
-          const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+//          const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
-          NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
+//          NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
-                multi_face_landmarks.size());
+//                multi_face_landmarks.size());
-          for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
+//          for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
-            const auto& landmarks = multi_face_landmarks[face_index];
+//            const auto& landmarks = multi_face_landmarks[face_index];
-            NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size());
+//            NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size());
-            for (int i = 0; i < landmarks.landmark_size(); ++i) {
+//            for (int i = 0; i < landmarks.landmark_size(); ++i) {
-              NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+//              NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
-                    landmarks.landmark(i).y(), landmarks.landmark(i).z());
+//                    landmarks.landmark(i).y(), landmarks.landmark(i).z());
-            }
+//            }
-          }
+//          }
-        }
+//        } else if (streamName == kDetectionsOutputStream) {
+//            if (packet.IsEmpty()) {
+//              NSLog(@"[TS:%lld] No face detections", packet.Timestamp().Value());
+//              return;
+//            }
+//            const auto& face_detections = packet.Get<std::vector<::mediapipe::Detection>>();
+//            NSLog(@"[TS:%lld] Number of face instances with detections: %lu", packet.Timestamp().Value(),
+//                  face_detections.size());
+//
+//        }
 #endif
     }
 
@@ -53,15 +97,36 @@ namespace Opipe
 
     FaceMeshModuleIMP::~FaceMeshModuleIMP()
     {
+        _delegate->attach(nullptr);
+        _delegate = 0;
+        
+        if (_olaContext) {
+            delete _olaContext;
+            _olaContext = nullptr;
+        }
+
+        _graph = nullptr;
+
+        if (_render) {
+            _dispatch->runSync([&] {
+                delete _render;
+                _render = nullptr;
+            });
+        }
+        
+        _context = nullptr;
+
+
     }
 
     void FaceMeshModuleIMP::suspend()
     {
-
+        _render->suspend();
     }
 
     void FaceMeshModuleIMP::resume()
     {
+        _render->resume();
     }
 
     bool FaceMeshModuleIMP::init(void *env, void *binaryData,
@@ -69,15 +134,18 @@ namespace Opipe
     {
         std::string graphName = "face_mesh_mobile_gpu";
         _delegate = std::make_shared<FaceMeshCallFrameDelegate>();
+        _delegate->attach(this);
         mediapipe::CalculatorGraphConfig config;
         config.ParseFromArray(binaryData, size);
-        _context = std::make_unique<Context>();
+        _olaContext = new OlaContext();
+        _context = _olaContext->glContext();
+        _render = new FaceMeshBeautyRender(_context);
 
     #if defined(__ANDROID__)
         _context->initEGLContext(env);
     #endif
 
-        _dispatch = std::make_unique<OpipeDispatch>(_context.get(), nullptr, nullptr);
+        _dispatch = std::make_unique<OpipeDispatch>(_context, nullptr, nullptr);
 
         _graph = std::make_unique<OlaGraph>(config);
         _graph->setDelegate(_delegate);
@@ -91,6 +159,22 @@ namespace Opipe
         return true;
     }
 
+    void FaceMeshModuleIMP::setLandmark(NormalizedLandmarkList landmark)
+    {
+        _lastLandmark = std::move(landmark);
+        if (_lastLandmark.landmark_size() == 0) {
+#if defined(__APPLE__)
+            NSLog(@"没有人脸");
+#endif
+        }
+        for (int i = 0; i < _lastLandmark.landmark_size(); ++i) {
+#if defined(__APPLE__)
+            NSLog(@"######## Set Landmark[%d]: (%f, %f, %f)", i, _lastLandmark.landmark(i).x(),
+                  _lastLandmark.landmark(i).y(), _lastLandmark.landmark(i).z());
+#endif
+        }
+    }
+
     void FaceMeshModuleIMP::startModule()
     {
         if (!_isInit)
@@ -141,18 +225,24 @@ namespace Opipe
         }
     }
 
-    GLuint FaceMeshModuleIMP::renderTexture(GLuint textureId,
+    TextureInfo FaceMeshModuleIMP::renderTexture(TextureInfo inputTexture)
-                                            int64_t timeStamp,
-                                            int width, int height)
     {
+        TextureInfo textureInfo;
+
         if (!_isInit)
         {
-            return textureId;
+            return textureInfo;
         }
-        _dispatch->runAsync([&] {
 
+        _dispatch->runSync([&] {
+            textureInfo = _render->renderTexture(inputTexture);
         });
-        return textureId;
+
+        return textureInfo;
     }
 
+    // OlaContext* currentContext() {
+    //     return _olaContext;
+    // }
+
 }

mediapipe/render/module/beauty/face_mesh_module_imp.h

@@ -3,11 +3,14 @@
 
 #include "mediapipe/render/module/common/ola_graph.h"
 #include "mediapipe/render/core/OpipeDispatch.hpp"
-
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/render/core/Context.hpp"
 #include "face_mesh_module.h"
+#include "face_mesh_beauty_render.h"
 
 namespace Opipe
 {
+    class FaceMeshModuleIMP;
     class FaceMeshCallFrameDelegate : public MPPGraphDelegate
     {
     public:
@@ -19,6 +22,21 @@ namespace Opipe
 #endif
         void outputPacket(OlaGraph *graph, const mediapipe::Packet &packet,
                           MPPPacketType packetType, const std::string &streamName) override;
+        
+        void outputPacket(OlaGraph *graph, const mediapipe::Packet &packet,
+                          const std::string &streamName) override;
+        
+        void attach(FaceMeshModuleIMP *imp) {
+            _imp = imp;
+        }
+        
+    private:
+        int64_t _last_landmark_ts = 0;
+        int64_t _last_video_ts = 0;
+        bool _hasFace = false;
+        NormalizedLandmarkList _lastLandmark;
+        NormalizedLandmarkList _emptyLandmark;
+        FaceMeshModuleIMP *_imp = nullptr;
     };
 
     class FaceMeshModuleIMP : public FaceMeshModule
@@ -40,6 +58,11 @@ namespace Opipe
 
         virtual void stopModule() override;
 
+        // 外部共享Context用
+        virtual OlaContext* currentContext() override {
+            return _olaContext;
+        } 
+
 #if defined(__APPLE__)
 
         /// 算法流输入
@@ -54,14 +77,19 @@ namespace Opipe
                                        int step,
                                        int64_t timeStamp) override;
 
-        virtual GLuint renderTexture(GLuint textureId, int64_t timeStamp, int width, int height) override;
+        virtual TextureInfo renderTexture(TextureInfo inputTexture) override;
+        
+        virtual void setLandmark(NormalizedLandmarkList landmark);
 
     private:
         std::unique_ptr<OpipeDispatch> _dispatch;
         std::unique_ptr<OlaGraph> _graph;
-        std::unique_ptr<Context> _context;
+        Context *_context = nullptr;
         bool _isInit = false;
+        NormalizedLandmarkList _lastLandmark;
         std::shared_ptr<FaceMeshCallFrameDelegate> _delegate;
+        FaceMeshBeautyRender *_render = nullptr;
+        OlaContext *_olaContext = nullptr;
     };
 }
 #endif

mediapipe/render/module/beauty/filters/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+
+BEAUTYFILTERS_SRCS = [
+        "BilateralAdjustFilter.cpp",
+        "FaceDistortionFilter.cpp",
+        "OlaBeautyFilter.cpp",
+        "UnSharpMaskFilter.cpp",
+]
+
+BEAUTYFILTERS_HDRS = [
+        "BilateralAdjustFilter.hpp",
+        "FaceDistortionFilter.hpp",
+        "OlaBeautyFilter.hpp",
+        "UnSharpMaskFilter.hpp",
+]
+
+cc_library(
+    name = "BeautyFilters",
+    srcs = BEAUTYFILTERS_SRCS,
+    hdrs = BEAUTYFILTERS_HDRS,
+    visibility = ["//visibility:public"],
+    alwayslink = True,
+    linkstatic = True,
+    deps = [
+    ] + select({
+        "//mediapipe:apple": [
+            "//mediapipe/render/core:core-ios",
+         ],
+        "//conditions:default": [
+            "//mediapipe/render/core:core",
+        ],
+    }),
+    copts = select({
+        "//mediapipe:apple": [
+            "-x objective-c++",
+            "-fobjc-arc",  # enable reference-counting
+        ],
+        "//conditions:default": ["-std=c++17"],
+    }),
+)

mediapipe/render/module/beauty/filters/BilateralAdjustFilter.cpp

@@ -0,0 +1,132 @@
+#include "BilateralAdjustFilter.hpp"
+
+namespace Opipe
+{
+
+    const std::string kbilateralAdjustFragmentShaderString = SHADER_STRING(
+        varying highp vec2 vTexCoord;
+        uniform sampler2D colorMap;
+        uniform sampler2D colorMap1;
+        lowp float factor1 = 2.782;
+        lowp float factor2 = 1.131;
+        lowp float factor3 = 1.158;
+        lowp float factor4 = 2.901;
+        lowp float factor5 = 0.979;
+        lowp float factor6 = 0.639;
+        lowp float factor7 = 0.963;
+        highp float blurOpacity = 0.460;
+        uniform lowp float filterOpacity;
+        precision highp float;
+
+        lowp vec3 rgb2hsv(lowp vec3 c) {
+            lowp vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0);
+            highp vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g));
+            highp vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r));
+            highp float d = q.x - min(q.w, q.y);
+            highp float e = 1.0e-10;
+            lowp vec3 hsv = vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x);
+            return hsv;
+        }
+
+        lowp vec3 ContrastSaturationBrightness(lowp vec3 color, lowp float brt, lowp float sat, lowp float con) {
+            const lowp float AvgLumR = 0.5;
+            const lowp float AvgLumG = 0.5;
+            const lowp float AvgLumB = 0.5;
+            const lowp vec3 LumCoeff = vec3(0.2125, 0.7154, 0.0721);
+            lowp vec3 AvgLumin = vec3(AvgLumR, AvgLumG, AvgLumB);
+            lowp vec3 brtColor = color * brt;
+            lowp vec3 intensity = vec3(dot(brtColor, LumCoeff));
+            lowp vec3 satColor = mix(intensity, brtColor, sat);
+            lowp vec3 conColor = mix(AvgLumin, satColor, con);
+            return conColor;
+        }
+
+        void main() {
+            lowp vec4 inputColor = texture2D(colorMap, vTexCoord);
+            lowp vec3 hsv = rgb2hsv(inputColor.rgb);
+            lowp float opacityLimit = 1.0;
+            if ((0.18 <= hsv.x && hsv.x <= 0.89) || hsv.z <= 0.2)
+            {
+                opacityLimit = 0.0;
+            }
+            if (0.16 < hsv.x && hsv.x < 0.18)
+            {
+                opacityLimit = min(opacityLimit, (0.18 - hsv.x) / 0.02);
+            }
+
+            if (0.89 < hsv.x && hsv.x < 0.91)
+            {
+                opacityLimit = min(opacityLimit, 1.0 - (0.91 - hsv.x) / 0.02);
+            }
+            if (0.2 < hsv.z && hsv.x < 0.3)
+            {
+                opacityLimit = min(opacityLimit, 1.0 - (0.3 - hsv.z) / 0.1);
+            }
+            if (opacityLimit == 0.0)
+            {
+                gl_FragColor = inputColor;
+                return;
+            }
+            lowp vec4 blurColor = texture2D(colorMap1, vTexCoord);
+            opacityLimit = blurOpacity * opacityLimit;
+            lowp float cDistance = distance(vec3(0.0, 0.0, 0.0), max(blurColor.rgb - inputColor.rgb, 0.0)) * factor1;
+            lowp vec3 brightColor = ContrastSaturationBrightness(inputColor.rgb, factor2, 1.0, factor3);
+            lowp vec3 mix11Color = mix(inputColor.rgb, brightColor.rgb, cDistance);
+            lowp float dDistance = distance(vec3(0.0, 0.0, 0.0), max(inputColor.rgb - blurColor.rgb, 0.0)) * factor4;
+            lowp vec3 darkColor = ContrastSaturationBrightness(inputColor.rgb, factor5, 1.0, factor6);
+            lowp vec3 mix115Color = mix(mix11Color.rgb, darkColor.rgb, dDistance);
+            lowp vec3 mix12Color;
+            if (factor7 < 0.999)
+            {
+                lowp vec3 mix116Color = mix(inputColor.rgb, mix115Color.rgb, factor7);
+                mix12Color = mix(mix116Color.rgb, blurColor.rgb, opacityLimit);
+            }
+            else
+            {
+                mix12Color = mix(mix115Color.rgb, blurColor.rgb, opacityLimit);
+            }
+            if (filterOpacity < 0.999)
+            {
+
+                float newAlpha = filterOpacity < 0.0 ? 0.0 : filterOpacity;
+                gl_FragColor = vec4(mix(inputColor.rgb, mix12Color.rgb, newAlpha), inputColor.a);
+            }
+            else
+            {
+                gl_FragColor = vec4(mix12Color.rgb, inputColor.a);
+            }
+        });
+
+    BilateralAdjustFilter::BilateralAdjustFilter(Context *context) : Filter(context), _opacityLimit(0.8)
+    {
+    }
+
+    BilateralAdjustFilter *BilateralAdjustFilter::create(Context *context)
+    {
+        BilateralAdjustFilter *ret =
+            new (std::nothrow) BilateralAdjustFilter(context);
+        if (!ret || !ret->init(context))
+        {
+            delete ret;
+            ret = 0;
+        }
+
+        return ret;
+    }
+
+    bool BilateralAdjustFilter::init(Context *context)
+    {
+        if (!Filter::initWithFragmentShaderString(context, kbilateralAdjustFragmentShaderString, 2))
+        {
+            return false;
+        }
+        return true;
+    }
+
+    bool BilateralAdjustFilter::proceed(float frameTime,
+                                        bool bUpdateTargets /* = true*/)
+    {
+        _filterProgram->setUniformValue("filterOpacity", _opacityLimit);
+        return Filter::proceed(frameTime, bUpdateTargets);
+    }
+}

mediapipe/render/module/beauty/filters/BilateralAdjustFilter.hpp

@@ -0,0 +1,30 @@
+#ifndef BilateralAdjustFilter_hpp
+#define BilateralAdjustFilter_hpp
+
+#include "mediapipe/render/core/Filter.hpp"
+#include "mediapipe/render/core/Context.hpp"
+
+namespace Opipe
+{
+    class BilateralAdjustFilter : public Opipe::Filter
+    {
+    public:
+        static BilateralAdjustFilter *create(Opipe::Context *context);
+        bool init(Opipe::Context *context);
+
+        virtual bool proceed(float frameTime = 0, bool bUpdateTargets = true) override;
+        float getOpacityLimit() { return _opacityLimit; };
+        void setOpacityLimit(float opacityLimit)
+        {
+            _opacityLimit = opacityLimit;
+        }
+
+    public:
+        BilateralAdjustFilter(Opipe::Context *context);
+        ~BilateralAdjustFilter(){};
+
+        float _opacityLimit;
+    };
+}
+
+#endif

mediapipe/render/module/beauty/filters/FaceDistortionFilter.cpp

@@ -0,0 +1,355 @@
+#include "FaceDistortionFilter.hpp"
+
+namespace Opipe
+{
+    const std::string kFaceDistortionVertexShaderString = SHADER_STRING(
+        precision highp float;
+        attribute vec4 texCoord;
+        varying vec2 vTexCoord;
+        uniform float aspectRatio;
+        uniform vec2 center[20];
+        uniform vec2 radius[20];
+
+        uniform float scale[20];
+        uniform float angle[20];
+        uniform float u_min[20];
+        uniform float u_max[20];
+        uniform int types[20];
+        uniform int count;
+        uniform float eye;
+        uniform float slim;
+        uniform int debug;
+        void main() {
+            vec2 uv = texCoord.xy;
+            gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0);
+            for (int i = 0; i < count; i++)
+            {
+                if (scale[i] == 0.0 || types[i] == 0)
+                {
+                    continue;
+                }
+                vec2 textureCoordinateToUse = uv;
+                float e1 = (textureCoordinateToUse.x - center[i].x) / (radius[i].x);
+                float e2 = (textureCoordinateToUse.y - center[i].y) / (radius[i].y / aspectRatio);
+                float d = (e1 * e1) + (e2 * e2);
+                if (d < 1.0)
+                {
+                    if (types[i] == 1)
+                    {
+                        vec2 dist = vec2(d * radius[i].x, d * radius[i].y);
+                        textureCoordinateToUse -= center[i];
+                        vec2 delta = ((radius[i] - dist) / radius[i]);
+                        float deltaScale = scale[i];
+                        if (deltaScale > 0.0)
+                        {
+                            deltaScale = smoothstep(u_min[i], u_max[i], deltaScale);
+                        }
+                        vec2 percent = 1.0 - ((delta * deltaScale) * eye);
+                        textureCoordinateToUse = textureCoordinateToUse * percent;
+                        uv = textureCoordinateToUse + center[i];
+                    }
+                    else if (types[i] == 2)
+                    {
+                        float dist = 1.0 - d;
+                        float delta = scale[i] * dist * slim;
+                        float deltaScale = smoothstep(u_min[i], u_max[i], dist);
+                        float directionX = cos(angle[i]) * deltaScale;
+                        float directionY = sin(angle[i]) * deltaScale / (3.0 / 4.0 * aspectRatio);
+                        uv = vec2(textureCoordinateToUse.x - (delta * directionX),
+                                  textureCoordinateToUse.y - (delta * directionY));
+                    }
+                }
+            }
+            vTexCoord = uv;
+        });
+
+    const std::string kFaceDistortionFragmentShaderString = SHADER_STRING(
+        precision highp float;
+        uniform sampler2D colorMap;
+        varying vec2 vTexCoord;
+        uniform vec2 facePoints[106];
+        void main() {
+            highp vec4 textureColor = texture2D(colorMap, vTexCoord);
+            gl_FragColor = textureColor;
+        });
+
+    FaceDistortionFilter::FaceDistortionFilter(Context *context) : Filter(context)
+    {
+    }
+
+    FaceDistortionFilter::~FaceDistortionFilter()
+    {
+        releaseDistoritionVBO();
+    }
+
+    void FaceDistortionFilter::generateDistoritionVBO(int numX,
+                                                      int numY,
+                                                      const GLfloat *imageTexUV)
+    {
+        if (vao == -1)
+        {
+            CHECK_GL(glGenBuffers(1, &vao));
+            CHECK_GL(glGenBuffers(1, &eao));
+
+            int vCount = numX * (numY + 1) * 2;
+            GLfloat *divideImageTexUV = (GLfloat *)malloc(vCount * 2 * sizeof(GLfloat));
+            GLushort *element = (GLushort *)malloc((vCount + numX - 1) * sizeof(GLushort));
+
+            float offsetX = ((imageTexUV[2] - imageTexUV[0]) / numX);
+            float offsetY = ((imageTexUV[5] - imageTexUV[1]) / numY);
+            int elementIndex = 0;
+            for (int i = 0; i < numX; i++)
+            {
+                for (int j = 0; j <= numY; j++)
+                {
+                    int offset = (i * (numY + 1) + j) * 4;
+
+                    divideImageTexUV[offset] = imageTexUV[0] + i * offsetX;
+                    divideImageTexUV[offset + 1] = imageTexUV[1] + j * offsetY;
+                    divideImageTexUV[offset + 2] = divideImageTexUV[offset] + offsetX;
+                    divideImageTexUV[offset + 3] = divideImageTexUV[offset + 1];
+
+                    element[elementIndex++] = offset / 2;
+                    element[elementIndex++] = offset / 2 + 1;
+                }
+                if (elementIndex < (vCount + numX - 1))
+                {
+                    element[elementIndex++] = 0xFFFF;
+                }
+            }
+            CHECK_GL(glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, eao));
+            CHECK_GL(glBufferData(GL_ELEMENT_ARRAY_BUFFER, (vCount + numX - 1) * sizeof(GLushort),
+                                  element, GL_STATIC_DRAW));
+
+            CHECK_GL(glBindBuffer(GL_ARRAY_BUFFER, vao));
+            CHECK_GL(glBufferData(GL_ARRAY_BUFFER, vCount * 2 * sizeof(GLfloat),
+                                  divideImageTexUV,
+                                  GL_STATIC_DRAW));
+            free(element);
+            free(divideImageTexUV);
+
+            CHECK_GL(glBindBuffer(GL_ARRAY_BUFFER, 0));
+            CHECK_GL(glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0));
+        }
+    }
+
+    void FaceDistortionFilter::releaseDistoritionVBO()
+    {
+        if (vao != -1)
+        {
+            CHECK_GL(glDeleteBuffers(0, &vao));
+            vao = -1;
+            CHECK_GL(glDeleteBuffers(0, &eao));
+            eao = -1;
+        }
+    }
+
+    FaceDistortionFilter *FaceDistortionFilter::create(Context *context)
+    {
+        FaceDistortionFilter *ret =
+            new (std::nothrow) FaceDistortionFilter(context);
+        if (!ret || !ret->init(context))
+        {
+            delete ret;
+            ret = 0;
+        }
+        return ret;
+    }
+
+    bool FaceDistortionFilter::init(Context *context)
+    {
+        if (!initWithShaderString(context,
+                                  kFaceDistortionVertexShaderString,
+                                  kFaceDistortionFragmentShaderString))
+        {
+            return false;
+        }
+        _eye = 0.0;
+        _slim = 0.0;
+        return true;
+    }
+
+    void FaceDistortionFilter::addPoint(Vector2 center,
+                                        float radiusX,
+                                        float radiusY,
+                                        float scale,
+                                        int type,
+                                        float angle,
+                                        float min,
+                                        float max)
+    {
+        _center[_count * 2] = center.x / 2 + 0.5;
+        _center[_count * 2 + 1] = center.y / 2 + 0.5;
+        _radius[_count * 2] = radiusX;
+        _radius[_count * 2 + 1] = radiusY;
+        _scale[_count] = scale;
+        _angle[_count] = angle;
+        _types[_count] = type;
+        _u_min[_count] = min;
+        _u_max[_count] = max;
+        _count++;
+    }
+
+    float getRadius(Vector2 anglePoint, Vector2 center)
+    {
+        float angle = 0;
+        if (fabs(anglePoint.x - center.x) <= 0.00001)
+        {
+            angle = anglePoint.y > center.y ? M_PI_2 : -M_PI_2;
+        }
+        else if (fabs(anglePoint.y - center.y) <= 0.00001)
+        {
+            angle = anglePoint.x > center.x ? 0 : M_PI;
+        }
+        else
+        {
+            float radius = (anglePoint.y - center.y) / (anglePoint.x - center.x);
+            angle = atanf(radius);
+            if ((angle > 0 && anglePoint.y - center.y < 0) ||
+                (angle < 0 && anglePoint.y - center.y > 0))
+            {
+                angle += M_PI;
+            }
+        }
+        return angle;
+    }
+
+    void FaceDistortionFilter::setUniform()
+    {
+        if (_facePoints.size() > 60)
+        {
+
+            _count = 0;
+            float width = (float)getFramebuffer()->getWidth();
+            float height = (float)getFramebuffer()->getHeight();
+            _filterProgram->setUniformValue("aspectRatio",
+                                            height /
+                                                width);
+
+            _filterProgram->setUniformValue("eye", _eye);
+            _filterProgram->setUniformValue("slim", _slim);
+
+            //左眼放大
+            {
+                Vector2 point1 = Vector2(_facePoints[75].x, _facePoints[75].y);
+                Vector2 point2 = Vector2(_facePoints[79].x, _facePoints[79].y);
+                Vector2 point3 = Vector2(_facePoints[65].x, _facePoints[65].y);
+                Vector2 center = point1.getCenter(point2);
+                float distance = center.distance(point3);
+                addPoint(center, distance / 2, distance / 2, 0.3, 1, 0.0f, 0.0f, 1);
+            }
+
+            //右眼放大
+            {
+                Vector2 point1 = Vector2(_facePoints[66].x, _facePoints[66].y);
+                Vector2 point2 = Vector2(_facePoints[70].x, _facePoints[70].y);
+                Vector2 point3 = Vector2(_facePoints[55].x, _facePoints[55].y);
+                Vector2 center = point1.getCenter(point2);
+                float distance = center.distance(point3);
+                addPoint(center, distance / 2, distance / 2, 0.3, 1, 0.0f, 0.0f, 1);
+            }
+            //瘦左脸
+            {
+
+                Vector2 point1 = Vector2(_facePoints[11].x, _facePoints[11].y);
+                Vector2 point2 = Vector2(_facePoints[60].x, _facePoints[60].y);
+                Vector2 point3 = Vector2(_facePoints[4].x, _facePoints[4].y);
+                Vector2 point4 = Vector2(_facePoints[16].x, _facePoints[16].y);
+
+                float angle = getRadius(point2, point1);
+                addPoint(point1, point1.distance(point3), point1.distance(point4), 0.02, 2, angle,
+                         0.0f,
+                         0.02);
+            }
+            //瘦右脸
+            {
+                Vector2 point1 = Vector2(_facePoints[21].x, _facePoints[21].y);
+                Vector2 point2 = Vector2(_facePoints[60].x, _facePoints[60].y);
+                Vector2 point3 = Vector2(_facePoints[28].x, _facePoints[28].y);
+                Vector2 point4 = Vector2(_facePoints[16].x, _facePoints[16].y);
+
+                float angle = getRadius(point2, point1);
+                addPoint(point1, point1.distance(point3), point1.distance(point4), 0.02, 2, angle,
+                         0.0f,
+                         0.02);
+            }
+
+            _filterProgram->setUniformValue("count", _count);
+            _filterProgram->setUniformValue("center", _count, _center, 2);
+            _filterProgram->setUniformValue("radius", _count, _radius, 2);
+            _filterProgram->setUniformValue("facePoints", (int)_facePoints.size(),
+                                            _u_facePoints, 2);
+
+            _filterProgram->setUniformValue("angle", _count, _angle);
+            _filterProgram->setUniformValue("scale", _count, _scale);
+            _filterProgram->setUniformValue("u_min", _count, _u_min);
+            _filterProgram->setUniformValue("u_max", _count, _u_max);
+            _filterProgram->setUniformValue("types", _count, _types);
+        }
+        else
+        {
+            _filterProgram->setUniformValue("count", 0);
+        }
+    }
+
+    bool FaceDistortionFilter::proceed(float frameTime, bool bUpdateTargets)
+    {
+#if DEBUG
+        _framebuffer->lock(typeid(*this).name());
+#else
+        _framebuffer->lock();
+#endif
+        setUniform();
+        _context->setActiveShaderProgram(_filterProgram);
+
+        _framebuffer->active();
+        CHECK_GL(glClearColor(_backgroundColor.r, _backgroundColor.g,
+                              _backgroundColor.b,
+                              _backgroundColor.a));
+        CHECK_GL(glClear(GL_COLOR_BUFFER_BIT));
+
+        const int numX = 20;
+        const int numY = 20;
+        for (std::map<int, InputFrameBufferInfo>::const_iterator it = _inputFramebuffers.begin();
+             it != _inputFramebuffers.end(); ++it)
+        {
+            int texIdx = it->first;
+            Framebuffer *fb = it->second.frameBuffer;
+            CHECK_GL(glActiveTexture(GL_TEXTURE0 + texIdx));
+            CHECK_GL(glBindTexture(GL_TEXTURE_2D, fb->getTexture()));
+            _filterProgram->setUniformValue(
+                texIdx == 0 ? "colorMap" : str_format("colorMap%d", texIdx),
+                texIdx);
+            // texcoord attribute
+            GLuint filterTexCoordAttribute =
+                _filterProgram->getAttribLocation(texIdx == 0 ? "texCoord" : str_format("texCoord%d", texIdx));
+            CHECK_GL(glEnableVertexAttribArray(filterTexCoordAttribute));
+            if (texIdx == 0)
+            {
+                const GLfloat *imageTexUV = _getTexureCoordinate(it->second.rotationMode);
+                generateDistoritionVBO(numX, numY, imageTexUV);
+                CHECK_GL(glBindBuffer(GL_ARRAY_BUFFER, vao));
+            }
+            CHECK_GL(glVertexAttribPointer(filterTexCoordAttribute, 2, GL_FLOAT, 0,
+                                           2 * sizeof(GLfloat), (void *)0));
+        }
+
+        CHECK_GL(glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, eao));
+        CHECK_GL(glEnable(GL_PRIMITIVE_RESTART_FIXED_INDEX));
+        CHECK_GL(glDrawElements(GL_TRIANGLE_STRIP, numX * (numY + 1) * 2 + numX - 1,
+                                GL_UNSIGNED_SHORT, (void *)0));
+
+        CHECK_GL(glBindBuffer(GL_ARRAY_BUFFER, 0));
+        CHECK_GL(glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0));
+        filter_externDraw();
+        _framebuffer->inactive();
+#if DEBUG
+        _framebuffer->unlock(typeid(*this).name());
+#else
+        _framebuffer->unlock();
+#endif
+        unPrepear();
+        return Source::proceed(frameTime, bUpdateTargets);
+    }
+
+}

mediapipe/render/module/beauty/filters/FaceDistortionFilter.hpp

@@ -0,0 +1,71 @@
+#ifndef FaceDistortionFilter_hpp
+#define FaceDistortionFilter_hpp
+
+#include <stdio.h>
+#include "mediapipe/render/core/math/vec2.hpp"
+#include "mediapipe/render/core/Filter.hpp"
+#include "mediapipe/render/core/Context.hpp"
+
+namespace Opipe
+{
+    class FaceDistortionFilter : public virtual Filter
+    {
+    public:
+        virtual ~FaceDistortionFilter();
+        FaceDistortionFilter(Context *context);
+
+        static FaceDistortionFilter *create(Context *context);
+        bool init(Context *context);
+        virtual bool proceed(float frameTime = 0.0, bool bUpdateTargets = true) override;
+
+    public:
+        void setEye(float eye)
+        {
+            _eye = eye;
+        };
+
+        void setSlim(float slim)
+        {
+            _slim = slim;
+        };
+
+        void setFacePoints(std::vector<Vec2> facePoints)
+        {
+            _facePoints = facePoints;
+            for (int i = 0; i < _facePoints.size(); i++)
+            {
+                auto facePoint = _facePoints[i];
+                _u_facePoints[i * 2] = facePoint.x;
+                _u_facePoints[i * 2 + 1] = facePoint.y;
+            }
+        }
+
+    private:
+        void setUniform();
+        void addPoint(Vector2 center, float radiusX, float radiusY,
+                      float scale, int type,
+                      float angle, float min = 0.0f, float max = 1.0f);
+        int _count;
+        float _center[40];
+        float _radius[40];
+        float _scale[20];
+        float _angle[20];
+        float _u_min[20];
+        float _u_max[20];
+        int _types[20];
+        float _u_facePoints[212];
+
+    private:
+        void generateDistoritionVBO(int numX, int numY, const GLfloat *imageTexUV);
+        void releaseDistoritionVBO();
+
+    private:
+        float _eye = 0.0;
+        float _slim = 0.0;
+        std::vector<Vec2> _facePoints; //暂时支持单个人脸
+        GLuint vao = -1;
+        GLuint eao = -1;
+    };
+}
+
+#endif

mediapipe/render/module/beauty/filters/OlaBeautyFilter.cpp

@@ -0,0 +1,149 @@
+#include "OlaBeautyFilter.hpp"
+
+namespace Opipe {
+    OlaBeautyFilter::OlaBeautyFilter(Context *context) : FilterGroup(context)
+    {
+
+    }
+
+    OlaBeautyFilter::~OlaBeautyFilter()
+    {
+        if (_lutImage) {
+            _lutImage->release();
+            _lutImage = nullptr;
+        }
+
+        if (_bilateralFilter) {
+            _bilateralFilter->release();
+            _bilateralFilter = nullptr;
+        }
+
+        if (_unSharpMaskFilter) {
+            _unSharpMaskFilter->release();
+            _unSharpMaskFilter = nullptr;
+        }
+
+        if (_alphaBlendFilter) {
+            _alphaBlendFilter->release();
+            _alphaBlendFilter = nullptr;
+        }
+
+        if (_lutFilter) {
+            _lutFilter->release();
+            _lutFilter = nullptr;
+        }
+
+        if (_bilateralAdjustFilter) {
+            _bilateralAdjustFilter->release();
+            _bilateralAdjustFilter = nullptr;
+        }
+
+        if (_faceDistortFilter) {
+            _faceDistortFilter->release();
+            _faceDistortFilter = nullptr;
+        }
+
+        if (_lookUpGroupFilter) {
+            _lookUpGroupFilter->release();
+            _lookUpGroupFilter = nullptr;
+        }
+    }
+
+    OlaBeautyFilter *OlaBeautyFilter::create(Context *context)
+    {
+        OlaBeautyFilter *ret = new (std::nothrow)OlaBeautyFilter(context);
+        if (ret && ret->init(context)) {
+            return ret;
+        } else {
+            delete ret;
+            return nullptr;
+        }
+    }
+
+    bool OlaBeautyFilter::init(Context *context) {
+        if (!FilterGroup::init(context)) {
+            return false;
+        }
+        
+        _bilateralFilter = BilateralFilter::create(context);
+        addFilter(_bilateralFilter);
+
+        _bilateralAdjustFilter = BilateralAdjustFilter::create(context);
+        addFilter(_bilateralAdjustFilter);
+        
+        _unSharpMaskFilter = UnSharpMaskFilter::create(context);
+        addFilter(_unSharpMaskFilter);
+
+        _lutFilter = LUTFilter::create(context);
+        _unSharpMaskFilter->addTarget(_lutFilter, 0);
+
+        _lookUpGroupFilter = FilterGroup::create(context);
+        _lookUpGroupFilter->addFilter(_unSharpMaskFilter);
+
+        _alphaBlendFilter = AlphaBlendFilter::create(context);
+        _faceDistortFilter = FaceDistortionFilter::create(context);
+        
+
+        _bilateralFilter->addTarget(_bilateralAdjustFilter, 1)->
+        addTarget(_alphaBlendFilter, 0);
+
+        _bilateralAdjustFilter->addTarget(_lookUpGroupFilter)->
+        addTarget(_alphaBlendFilter, 1)->addTarget(_faceDistortFilter);
+ 
+        _alphaBlendFilter->setMix(0.0);
+
+        _unSharpMaskFilter->setBlurRadiusInPixel(4.0f, true);
+        _unSharpMaskFilter->setBlurRadiusInPixel(2.0f, false);
+        _unSharpMaskFilter->setIntensity(1.365);
+        
+        _bilateralAdjustFilter->setOpacityLimit(0.6);
+        
+        _bilateralFilter->setDistanceNormalizationFactor(2.746);
+        _bilateralFilter->setTexelSpacingMultiplier(2.7);
+
+        setTerminalFilter(_faceDistortFilter);
+
+        std::vector<Vec2> defaultFace;
+
+        
+        return true;
+        
+    }
+
+    bool OlaBeautyFilter::proceed(float frameTime, bool bUpdateTargets) {
+        return FilterGroup::proceed(frameTime, bUpdateTargets);
+    }
+
+    void OlaBeautyFilter::update(float frameTime) {
+        FilterGroup::update(frameTime);
+    }
+
+
+    void OlaBeautyFilter::setInputFramebuffer(Framebuffer *framebuffer,
+                                                        RotationMode rotationMode,
+                                                        int texIdx,
+                                                        bool ignoreForPrepared) {
+        for (auto& filter : _filters) {
+            filter->setInputFramebuffer(framebuffer, rotationMode, texIdx, ignoreForPrepared);
+        }
+    }
+
+    void OlaBeautyFilter::setSmoothing(float smoothing) {
+        smoothing = smoothing < -1 ? -1 : smoothing;
+        smoothing = smoothing > 1 ? 1 : smoothing;
+        _bilateralAdjustFilter->setOpacityLimit(smoothing);
+    }
+    
+    float OlaBeautyFilter::getSmoothing() {
+        return _bilateralAdjustFilter->getOpacityLimit();
+    }
+    
+    void OlaBeautyFilter::setWhitening(float whitening) {
+        _alphaBlendFilter->setMix(whitening);
+    }
+    
+    float OlaBeautyFilter::getWhitening() {
+        return _alphaBlendFilter->getMix();
+    }
+
+}

mediapipe/render/module/beauty/filters/OlaBeautyFilter.hpp

@@ -0,0 +1,85 @@
+#include "mediapipe/render/core/Filter.hpp"
+#include "mediapipe/render/core/FilterGroup.hpp"
+#include "mediapipe/render/core/BilateralFilter.hpp"
+#include "mediapipe/render/core/AlphaBlendFilter.hpp"
+#include "mediapipe/render/core/LUTFilter.hpp"
+#include "mediapipe/render/core/SourceImage.hpp"
+#include "BilateralAdjustFilter.hpp"
+#include "UnSharpMaskFilter.hpp"
+#include "FaceDistortionFilter.hpp"
+
+namespace Opipe
+{
+    class OlaBeautyFilter : public FilterGroup
+    {
+    public:
+        float getSmoothing();
+
+        float getWhitening();
+
+        void setSmoothing(float smoothing);
+
+        void setWhitening(float whitening);
+
+    public:
+        static OlaBeautyFilter *create(Context *context);
+
+        bool init(Context *context);
+
+        bool proceed(float frameTime = 0, bool bUpdateTargets = true) override;
+
+        void update(float frameTime = 0) override;
+
+        virtual void setInputFramebuffer(Framebuffer *framebuffer,
+                                         RotationMode rotationMode =
+                                             RotationMode::NoRotation,
+                                         int texIdx = 0,
+                                         bool ignoreForPrepared = false) override;
+
+        void setLUTImage(SourceImage *image);
+
+        OlaBeautyFilter(Context *context);
+
+        virtual ~OlaBeautyFilter();
+
+        void setFacePoints(std::vector<Vec2> facePoints) {
+            _faceDistortFilter->setFacePoints(facePoints);
+        }
+
+        // "大眼 0.0 - 1.0"
+        void setEye(float eye) {
+            _faceDistortFilter->setEye(eye);
+        }
+
+        //1.0f, "瘦脸 0.0 - 1.0",
+        void setSlim(float slim) {
+            _faceDistortFilter->setSlim(slim);
+        }
+
+        // "磨皮 0.0 - 1.0"
+        void setSkin(float skin) {
+            if (skin == 0.0) {
+                _bilateralAdjustFilter->setEnable(false);
+            } else {
+                _bilateralAdjustFilter->setEnable(true);
+                _bilateralAdjustFilter->setOpacityLimit(skin);
+            }
+        }
+
+        // "美白 0.0 - 1.0"
+        void setWhiten(float whiten) {
+             _alphaBlendFilter->setMix(whiten);
+        }
+
+    private:
+        BilateralFilter *_bilateralFilter = 0;
+        AlphaBlendFilter *_alphaBlendFilter = 0;
+        LUTFilter *_lutFilter = 0;
+        BilateralAdjustFilter *_bilateralAdjustFilter = 0;
+        UnSharpMaskFilter *_unSharpMaskFilter = 0;
+        FaceDistortionFilter *_faceDistortFilter = 0;
+        FilterGroup *_lookUpGroupFilter = 0;
+
+        SourceImage *_lutImage = 0;
+    };
+}

mediapipe/render/module/beauty/filters/UnSharpMaskFilter.cpp

@@ -0,0 +1,123 @@
+#include "UnSharpMaskFilter.hpp"
+
+namespace Opipe {
+    const std::string kUnsharpMaskFragmentShaderString = SHADER_STRING
+    (
+     varying highp vec2 vTexCoord;
+     varying highp vec2 vTexCoord1;
+     uniform sampler2D colorMap;
+     uniform sampler2D colorMap1;
+     uniform highp float intensity;
+     void main()
+     {
+        lowp vec4 sharpImageColor = texture2D(colorMap, vTexCoord);
+        lowp vec4 blurredImageColor = texture2D(colorMap1, vTexCoord1);
+        gl_FragColor = vec4(sharpImageColor.rgb * intensity + blurredImageColor.rgb * (1.0 - intensity), blurredImageColor.a);
+     }
+   );
+    
+    class UnSharpFilter : public Filter {
+    public:
+        static UnSharpFilter* create(Context *context);
+        bool init(Context *context);
+        
+        virtual bool proceed(float frameTime = 0.0, bool bUpdateTargets = true) override;
+        
+        void setIntensity(float intensity);
+    protected:
+        UnSharpFilter(Context *context);
+        
+        float _intensity;
+    };
+    
+    UnSharpFilter::UnSharpFilter(Context *context) :
+    Filter(context),
+    _intensity(0.0) {
+        
+    }
+    
+    UnSharpFilter* UnSharpFilter::create(Context *context) {
+        UnSharpFilter* ret = new (std::nothrow) UnSharpFilter(context);
+        if (!ret || !ret->init(context)) {
+            delete ret;
+            ret = 0;
+        }
+        return ret;
+    }
+    
+    bool UnSharpFilter::init(Context *context) {
+        if (!Filter::initWithFragmentShaderString(context,
+                                                  kUnsharpMaskFragmentShaderString,
+                                                  2)) {
+            return false;
+        }
+        return true;
+    }
+    
+    void UnSharpFilter::setIntensity(float intensity) {
+        _intensity = intensity;
+    }
+    
+    bool UnSharpFilter::proceed(float frameTime, bool bUpdateTargets/* = true*/) {
+        _filterProgram->setUniformValue("intensity", _intensity);
+        return Filter::proceed(frameTime, bUpdateTargets);
+    }
+    
+    UnSharpMaskFilter::UnSharpMaskFilter(Context *context)
+    : FilterGroup(context) ,_blurFilter(0)
+    ,_unsharpMaskFilter(0) {
+        
+    }
+
+    UnSharpMaskFilter::~UnSharpMaskFilter() {
+        if (_blurFilter) {
+            _blurFilter->release();
+            _blurFilter = 0;
+        }
+        
+        if (_unsharpMaskFilter) {
+            _unsharpMaskFilter->release();
+            _unsharpMaskFilter = 0;
+        }
+    }
+
+    UnSharpMaskFilter *UnSharpMaskFilter::create(Context *context) {
+    UnSharpMaskFilter* ret = new (std::nothrow) UnSharpMaskFilter(context);
+        if (!ret || !ret->init(context)) {
+            delete ret;
+            ret = 0;
+        }
+        return ret;
+    }
+
+    bool UnSharpMaskFilter::init(Context *context) {
+        if (!FilterGroup::init(context)) {
+            return false;
+        }
+        
+        _blurFilter = GaussianBlurFilter::create(context);
+        addFilter(_blurFilter);
+        
+        _unsharpMaskFilter = UnSharpMaskFilter::create(context);
+        addFilter(_unsharpMaskFilter);
+        
+        _blurFilter->addTarget(_unsharpMaskFilter,1);
+        setTerminalFilter(_unsharpMaskFilter);
+        
+        return true;
+    }
+
+    void UnSharpMaskFilter::setIntensity(float intensity) {
+        ((UnSharpMaskFilter *)_unsharpMaskFilter)->setIntensity(intensity);
+    }
+
+    void UnSharpMaskFilter::setBlurRadiusInPixel(float blurRadius,
+                                                              bool isVertical) {
+        if (isVertical) {
+            _blurFilter->setSigma_v(blurRadius);
+        } else {
+           _blurFilter->setSigma_h(blurRadius);
+        }
+    }
+    
+}

mediapipe/render/module/beauty/filters/UnSharpMaskFilter.hpp

@@ -0,0 +1,27 @@
+#ifndef UnSharpMaskFilter_hpp
+#define UnSharpMaskFilter_hpp
+
+#include "mediapipe/render/core/FilterGroup.hpp"
+#include "mediapipe/render/core/GaussianBlurFilter.hpp"
+
+namespace Opipe {
+    class UnSharpMaskFilter : public FilterGroup {
+    public:
+        void setIntensity(float intensity);
+        void setBlurRadiusInPixel(float blurRadius, bool isVertical);
+        
+    public:
+        static UnSharpMaskFilter* create(Context *context);
+        
+        bool init(Context *context);
+    public:
+        UnSharpMaskFilter(Context *context);
+        ~UnSharpMaskFilter();
+        
+        GaussianBlurFilter *_blurFilter = nullptr;
+        Filter *_unsharpMaskFilter = nullptr;
+        
+    };
+}
+
+#endif

mediapipe/render/module/beauty/ios/example/OpipeBeautyModuleExample/OpipeBeautyModuleExample.xcodeproj/xcuserdata/wangrenzhu.xcuserdatad/xcschemes/xcschememanagement.plist

@@ -7,7 +7,7 @@
 		<key>OpipeBeautyModuleExample.xcscheme_^#shared#^_</key>
 		<dict>
 			<key>orderHint</key>
-			<integer>6</integer>
+			<integer>7</integer>
 		</dict>
 	</dict>
 </dict>

mediapipe/render/module/beauty/ios/framework/BUILD

@@ -3,13 +3,13 @@ load("@build_bazel_rules_apple//apple:ios.bzl", "ios_framework", "ios_static_fra
 # 用上面这条指令build
 
 
-ios_framework(
+ios_static_framework(
     name = "OlaFaceUnityFramework",
     hdrs = [
         "OlaFaceUnity.h",
     ],
-    infoplists = ["Info.plist"],
+    # infoplists = ["Info.plist"],
-    bundle_id = "com.ola.olarender.develop",
+    # bundle_id = "com.ola.olarender.develop",
     families = ["iphone", "ipad"],
     minimum_os_version = "11.0",
     deps = [
@@ -31,6 +31,7 @@ objc_library(
         "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb",
         "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
         "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
+        "//mediapipe/render/module/beauty:whiten.png",
     ],
      copts = select({
         "//mediapipe:apple": [
@@ -39,7 +40,7 @@ objc_library(
         ],
         "//conditions:default": [],
     }),
-    alwayslink = True,
+    # alwayslink = True,
     sdk_frameworks = [
         "AVFoundation",
         "CoreGraphics",

mediapipe/render/module/beauty/ios/framework/FaceUnityFramework.xcodeproj/xcshareddata/xcschemes/OlaFaceUnityFramework.xcscheme

@@ -1,29 +1,29 @@
 
 <Scheme version="1.3" LastUpgradeVersion="1000">
-    <BuildAction buildImplicitDependencies="YES" parallelizeBuildables="YES">
+    <BuildAction parallelizeBuildables="YES" buildImplicitDependencies="YES">
         <BuildActionEntries>
-            <BuildActionEntry buildForProfiling="YES" buildForTesting="YES" buildForAnalyzing="YES" buildForRunning="YES" buildForArchiving="YES">
+            <BuildActionEntry buildForTesting="YES" buildForRunning="YES" buildForProfiling="YES" buildForArchiving="YES" buildForAnalyzing="YES">
-                <BuildableReference ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableIdentifier="primary" BlueprintIdentifier="87427C8CCEDD951E00000000" BuildableName="OlaFaceUnityFramework.framework" BlueprintName="OlaFaceUnityFramework"></BuildableReference>
+                <BuildableReference BuildableName="OlaFaceUnityFramework.framework" BuildableIdentifier="primary" BlueprintIdentifier="F2FE34CE0C5C7AFE00000000" BlueprintName="OlaFaceUnityFramework" ReferencedContainer="container:FaceUnityFramework.xcodeproj"></BuildableReference>
             </BuildActionEntry>
         </BuildActionEntries>
     </BuildAction>
-    <TestAction buildConfiguration="__TulsiTestRunner_Debug" shouldUseLaunchSchemeArgsEnv="YES" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit">
+    <TestAction buildConfiguration="__TulsiTestRunner_Debug" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" shouldUseLaunchSchemeArgsEnv="YES" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit">
         <Testables></Testables>
         <BuildableProductRunnable runnableDebuggingMode="0">
-            <BuildableReference BlueprintName="OlaFaceUnityFramework" BuildableName="OlaFaceUnityFramework.framework" BlueprintIdentifier="87427C8CCEDD951E00000000" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableIdentifier="primary"></BuildableReference>
+            <BuildableReference BlueprintIdentifier="F2FE34CE0C5C7AFE00000000" BlueprintName="OlaFaceUnityFramework" BuildableName="OlaFaceUnityFramework.framework" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableIdentifier="primary"></BuildableReference>
         </BuildableProductRunnable>
     </TestAction>
-    <LaunchAction ignoresPersistentStateOnLaunch="NO" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" launchStyle="0" buildConfiguration="Debug" useCustomWorkingDirectory="NO" debugServiceExtension="internal" allowLocationSimulation="YES" debugDocumentVersioning="YES" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB">
+    <LaunchAction launchStyle="0" buildConfiguration="Debug" allowLocationSimulation="YES" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" useCustomWorkingDirectory="NO" ignoresPersistentStateOnLaunch="NO" debugDocumentVersioning="YES" debugServiceExtension="internal" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit">
         <EnvironmentVariables></EnvironmentVariables>
         <BuildableProductRunnable runnableDebuggingMode="0">
-            <BuildableReference BlueprintName="OlaFaceUnityFramework" BuildableIdentifier="primary" BuildableName="OlaFaceUnityFramework.framework" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BlueprintIdentifier="87427C8CCEDD951E00000000"></BuildableReference>
+            <BuildableReference BuildableName="OlaFaceUnityFramework.framework" BuildableIdentifier="primary" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BlueprintIdentifier="F2FE34CE0C5C7AFE00000000" BlueprintName="OlaFaceUnityFramework"></BuildableReference>
         </BuildableProductRunnable>
     </LaunchAction>
-    <ProfileAction buildConfiguration="__TulsiTestRunner_Release" shouldUseLaunchSchemeArgsEnv="YES" useCustomWorkingDirectory="NO" debugDocumentVersioning="YES">
+    <ProfileAction useCustomWorkingDirectory="NO" buildConfiguration="__TulsiTestRunner_Release" shouldUseLaunchSchemeArgsEnv="YES" debugDocumentVersioning="YES">
         <BuildableProductRunnable runnableDebuggingMode="0">
-            <BuildableReference BlueprintName="OlaFaceUnityFramework" BuildableName="OlaFaceUnityFramework.framework" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BlueprintIdentifier="87427C8CCEDD951E00000000" BuildableIdentifier="primary"></BuildableReference>
+            <BuildableReference BlueprintName="OlaFaceUnityFramework" BuildableIdentifier="primary" BuildableName="OlaFaceUnityFramework.framework" BlueprintIdentifier="F2FE34CE0C5C7AFE00000000" ReferencedContainer="container:FaceUnityFramework.xcodeproj"></BuildableReference>
         </BuildableProductRunnable>
     </ProfileAction>
     <AnalyzeAction buildConfiguration="Debug"></AnalyzeAction>
-    <ArchiveAction revealArchiveInOrganizer="YES" buildConfiguration="Release"></ArchiveAction>
+    <ArchiveAction buildConfiguration="Release" revealArchiveInOrganizer="YES"></ArchiveAction>
 </Scheme>

mediapipe/render/module/beauty/ios/framework/FaceUnityFramework.xcodeproj/xcshareddata/xcschemes/mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.xcscheme

@@ -1,29 +1,29 @@
 
-<Scheme LastUpgradeVersion="1000" version="1.3">
+<Scheme version="1.3" LastUpgradeVersion="1000">
     <BuildAction parallelizeBuildables="YES" buildImplicitDependencies="YES">
         <BuildActionEntries>
-            <BuildActionEntry buildForRunning="YES" buildForProfiling="YES" buildForAnalyzing="YES" buildForArchiving="YES" buildForTesting="YES">
+            <BuildActionEntry buildForProfiling="YES" buildForRunning="YES" buildForTesting="YES" buildForAnalyzing="YES" buildForArchiving="YES">
-                <BuildableReference BuildableIdentifier="primary" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary" BlueprintIdentifier="87427C8C9425E2A600000000" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a" ReferencedContainer="container:FaceUnityFramework.xcodeproj"></BuildableReference>
+                <BuildableReference BlueprintIdentifier="F2FE34CED4660C9200000000" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a" BuildableIdentifier="primary"></BuildableReference>
             </BuildActionEntry>
         </BuildActionEntries>
     </BuildAction>
-    <TestAction shouldUseLaunchSchemeArgsEnv="YES" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit" buildConfiguration="__TulsiTestRunner_Debug" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB">
+    <TestAction buildConfiguration="__TulsiTestRunner_Debug" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" shouldUseLaunchSchemeArgsEnv="YES">
         <Testables></Testables>
         <BuildableProductRunnable runnableDebuggingMode="0">
-            <BuildableReference BlueprintIdentifier="87427C8C9425E2A600000000" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableIdentifier="primary" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary"></BuildableReference>
+            <BuildableReference BuildableIdentifier="primary" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary" BlueprintIdentifier="F2FE34CED4660C9200000000" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a"></BuildableReference>
         </BuildableProductRunnable>
     </TestAction>
-    <LaunchAction buildConfiguration="Debug" useCustomWorkingDirectory="NO" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" ignoresPersistentStateOnLaunch="NO" debugServiceExtension="internal" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" allowLocationSimulation="YES" debugDocumentVersioning="YES" launchStyle="0">
+    <LaunchAction ignoresPersistentStateOnLaunch="NO" debugDocumentVersioning="YES" selectedLauncherIdentifier="Xcode.DebuggerFoundation.Launcher.LLDB" allowLocationSimulation="YES" launchStyle="0" selectedDebuggerIdentifier="Xcode.DebuggerFoundation.Debugger.LLDB" useCustomWorkingDirectory="NO" debugServiceExtension="internal" buildConfiguration="Debug" customLLDBInitFile="$(PROJECT_FILE_PATH)/.tulsi/Utils/lldbinit">
         <EnvironmentVariables></EnvironmentVariables>
         <MacroExpansion>
-            <BuildableReference BuildableIdentifier="primary" BlueprintIdentifier="87427C8C9425E2A600000000" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a" ReferencedContainer="container:FaceUnityFramework.xcodeproj"></BuildableReference>
+            <BuildableReference BuildableIdentifier="primary" BlueprintIdentifier="F2FE34CED4660C9200000000" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary" ReferencedContainer="container:FaceUnityFramework.xcodeproj"></BuildableReference>
         </MacroExpansion>
     </LaunchAction>
-    <ProfileAction buildConfiguration="__TulsiTestRunner_Release" shouldUseLaunchSchemeArgsEnv="YES" useCustomWorkingDirectory="NO" debugDocumentVersioning="YES">
+    <ProfileAction debugDocumentVersioning="YES" buildConfiguration="__TulsiTestRunner_Release" useCustomWorkingDirectory="NO" shouldUseLaunchSchemeArgsEnv="YES">
         <MacroExpansion>
-            <BuildableReference ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableIdentifier="primary" BlueprintIdentifier="87427C8C9425E2A600000000" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a" BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary"></BuildableReference>
+            <BuildableReference BlueprintName="mediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary" ReferencedContainer="container:FaceUnityFramework.xcodeproj" BuildableIdentifier="primary" BlueprintIdentifier="F2FE34CED4660C9200000000" BuildableName="libmediapipe-render-module-beauty-ios-framework-OlaFaceUnityLibrary.a"></BuildableReference>
         </MacroExpansion>
     </ProfileAction>
     <AnalyzeAction buildConfiguration="Debug"></AnalyzeAction>
-    <ArchiveAction revealArchiveInOrganizer="YES" buildConfiguration="Release"></ArchiveAction>
+    <ArchiveAction buildConfiguration="Release" revealArchiveInOrganizer="YES"></ArchiveAction>
 </Scheme>

mediapipe/render/module/beauty/ios/framework/FaceUnityFramework.xcodeproj/xcuserdata/wangrenzhu.xcuserdatad/xcschemes/xcschememanagement.plist

@@ -11,6 +11,11 @@
 			<key>orderHint</key>
 			<integer>5</integer>
 		</dict>
+		<key>_bazel_clean_.xcscheme_^#shared#^_</key>
+		<dict>
+			<key>orderHint</key>
+			<integer>6</integer>
+		</dict>
 		<key>_idx_Scheme.xcscheme_^#shared#^_</key>
 		<dict>
 			<key>isShown</key>

mediapipe/render/module/beauty/ios/framework/OlaFaceUnity.h

@@ -2,16 +2,30 @@
 #import <CoreVideo/CoreVideo.h>
 #import <AVFoundation/AVFoundation.h>
 #import <UIKit/UIKit.h>
+#import <OpenGLES/EAGL.h>
+
+typedef struct {
+        int width;
+        int height;
+        int textureId;
+        int ioSurfaceId; // iOS 专属
+        int64_t frameTime;
+} FaceTextureInfo;
 
 @interface OlaFaceUnity : NSObject
 
 
 + (instancetype)sharedInstance;
 
+- (void)currentContext;
 
+// 算法输入
 - (void)processVideoFrame:(CVPixelBufferRef)pixelbuffer
                 timeStamp:(int64_t)timeStamp;
 
+
+- (FaceTextureInfo)render:(FaceTextureInfo)inputTexture;
+
 - (void)dispose;
 
 @end

mediapipe/render/module/beauty/ios/framework/OlaFaceUnity.mm

@@ -9,7 +9,8 @@
 @end
 @implementation OlaFaceUnity
 
-- (void)dealloc {
+- (void)dealloc
+{
     if (_face_module) {
         delete _face_module;
         _face_module = nullptr;
@@ -25,7 +26,8 @@
     return self;
 }
 
-- (void)initModule {
+- (void)initModule
+{
     _face_module = Opipe::FaceMeshModule::create();
     NSBundle *bundle = [NSBundle bundleForClass:[self class]];
     NSURL* graphURL = [bundle URLForResource:@"face_mesh_mobile_gpu" withExtension:@"binarypb"];
@@ -34,8 +36,6 @@
         _face_module->init(nullptr, (void *)data.bytes, data.length);
         _face_module->startModule();
     }
-    
-    
 }
 
 + (instancetype)sharedInstance {
@@ -47,6 +47,42 @@
     return sharedInstance;
 }
 
+- (FaceTextureInfo)render:(FaceTextureInfo)inputTexture
+{
+    TextureInfo rs;
+    rs.ioSurfaceId = inputTexture.ioSurfaceId;
+    if (_face_module) {
+        TextureInfo input;
+        input.width = inputTexture.width;
+        input.height = inputTexture.height;
+        input.ioSurfaceId = inputTexture.ioSurfaceId;
+        input.textureId = inputTexture.textureId;
+        input.frameTime = inputTexture.frameTime;
+
+        rs = _face_module->renderTexture(input);
+    }
+    FaceTextureInfo result;
+    result.width = rs.width;
+    result.height = rs.height;
+    result.ioSurfaceId = rs.ioSurfaceId;
+    result.textureId = rs.textureId;
+    result.frameTime = rs.frameTime;
+
+    return result;
+}
+
+// - (EAGLContext *)currentContext
+// {
+//     if (_face_module) {
+//         return _face_module->currentContext()->currentContext();
+//     }
+// }
+
+- (void)currentContext {
+    if (_face_module) {
+         _face_module->currentContext()->currentContext();
+    }
+}
 
 - (void)processVideoFrame:(CVPixelBufferRef)pixelbuffer
                 timeStamp:(int64_t)timeStamp;

mediapipe/render/module/common/ola_graph.cc

@@ -23,6 +23,9 @@ namespace Opipe
         {
             return;
         }
+        
+        graph->_delegate.lock()->outputPacket(graph, packet, streamName);
+        
         if (packetType == MPPPacketTypeRaw)
         {
             graph->_delegate.lock()->outputPacket(graph, packet, packetType, streamName);
@@ -41,9 +44,8 @@ namespace Opipe
             graph->_delegate.lock()->outputPixelbuffer(graph, pixelBuffer, streamName, packet.Timestamp().Value());
 #endif
         }
-        else
+        
-        {
+        
-        }
     }
 
     OlaGraph::OlaGraph(const mediapipe::CalculatorGraphConfig &config)

mediapipe/render/module/common/ola_graph.h

@@ -44,6 +44,10 @@ namespace Opipe
                                   const mediapipe::Packet &packet,
                                   MPPPacketType packetType,
                                   const std::string &streamName) = 0;
+        
+        virtual void outputPacket(OlaGraph *graph,
+                                  const mediapipe::Packet &packet,
+                                  const std::string &streamName) = 0;
     };
 
     class OlaGraph
@@ -167,7 +171,7 @@ namespace Opipe
         bool waitUntilIdle();
 
         std::weak_ptr<MPPGraphDelegate> _delegate;
-        std::atomic<int32_t> _framesInFlight = 2;
+        std::atomic<int32_t> _framesInFlight = 0;
 
     private:
         std::unique_ptr<mediapipe::CalculatorGraph> _graph;
@@ -188,7 +192,7 @@ namespace Opipe
 
         absl::Status performStart();
 
-        int _maxFramesInFlight = 0;
+        int _maxFramesInFlight = 1;
     };
 
 }