mediapipe-rs/mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt

# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
# executed on CPU.)
#
# It is required that "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
# path during execution if `with_attention` is not set or set to `false`.
#
# It is required that "face_landmark_with_attention.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
# path during execution if `with_attention` is set to `true`.
#
# EXAMPLE:
#   node {
#     calculator: "FaceLandmarkGpu"
#     input_stream: "IMAGE:image"
#     input_stream: "ROI:face_roi"
#     input_side_packet: "WITH_ATTENTION:with_attention"
#     output_stream: "LANDMARKS:face_landmarks"
#   }

type: "FaceLandmarkGpu"

# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where a face is located.
# (NormalizedRect)
input_stream: "ROI:roi"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"

# 468 or 478 facial landmarks within the given ROI. (NormalizedLandmarkList)
#
# Number of landmarks depends on the WITH_ATTENTION flag. If it's `true` - then
# there will be 478 landmarks with refined lips, eyes and irises (10 extra
# landmarks are for irises), otherwise 468 non-refined landmarks are returned.
#
# NOTE: if a face is not present within the given ROI, for this particular
# timestamp there will not be an output packet in the LANDMARKS stream. However,
# the MediaPipe framework will internally inform the downstream calculators of
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:face_landmarks"

# Transforms the input image into a 192x192 tensor.
node: {
  calculator: "ImageToTensorCalculator"
  input_stream: "IMAGE_GPU:image"
  input_stream: "NORM_RECT:roi"
  output_stream: "TENSORS:input_tensors"
  options: {
    [mediapipe.ImageToTensorCalculatorOptions.ext] {
      output_tensor_width: 192
      output_tensor_height: 192
      output_tensor_float_range {
        min: 0.0
        max: 1.0
      }
      gpu_origin: TOP_LEFT
    }
  }
}

# Loads the face landmarks TF Lite model.
node {
  calculator: "FaceLandmarksModelLoader"
  input_side_packet: "WITH_ATTENTION:with_attention"
  output_side_packet: "MODEL:model"
}

# Generates a single side packet containing a TensorFlow Lite op resolver that
# supports custom ops needed by the model used in this graph.
node {
  calculator: "TfLiteCustomOpResolverCalculator"
  output_side_packet: "op_resolver"
}

# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of GPU tensors representing, for instance, detection boxes/keypoints
# and scores.
node {
  calculator: "InferenceCalculator"
  input_stream: "TENSORS:input_tensors"
  input_side_packet: "MODEL:model"
  input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
  output_stream: "TENSORS:output_tensors"
  options: {
    [mediapipe.InferenceCalculatorOptions.ext] {
      # Do not remove. Used for generation of XNNPACK/NNAPI graphs.
    }
  }
}

# Splits a vector of tensors into landmark tensors and face flag tensor.
node {
  calculator: "SwitchContainer"
  input_side_packet: "ENABLE:with_attention"
  input_stream: "output_tensors"
  output_stream: "landmark_tensors"
  output_stream: "face_flag_tensor"
  options {
    [mediapipe.SwitchContainerOptions.ext] {
      contained_node: {
        calculator: "SplitTensorVectorCalculator"
        options: {
          [mediapipe.SplitVectorCalculatorOptions.ext] {
            ranges: { begin: 0 end: 1 }
            ranges: { begin: 1 end: 2 }
          }
        }
      }
      contained_node: {
        calculator: "SplitTensorVectorCalculator"
        options: {
          [mediapipe.SplitVectorCalculatorOptions.ext] {
            ranges: { begin: 0 end: 6 }
            ranges: { begin: 6 end: 7 }
          }
        }
      }
    }
  }
}

# Converts the face-flag tensor into a float that represents the confidence
# score of face presence.
node {
  calculator: "TensorsToFloatsCalculator"
  input_stream: "TENSORS:face_flag_tensor"
  output_stream: "FLOAT:face_presence_score"
  options: {
    [mediapipe.TensorsToFloatsCalculatorOptions.ext] {
      activation: SIGMOID
    }
  }
}

# Applies a threshold to the confidence score to determine whether a face is
# present.
node {
  calculator: "ThresholdingCalculator"
  input_stream: "FLOAT:face_presence_score"
  output_stream: "FLAG:face_presence"
  options: {
    [mediapipe.ThresholdingCalculatorOptions.ext] {
      threshold: 0.5
    }
  }
}

# Drop landmarks tensors if face is not present.
node {
  calculator: "GateCalculator"
  input_stream: "landmark_tensors"
  input_stream: "ALLOW:face_presence"
  output_stream: "ensured_landmark_tensors"
}

# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
  calculator: "SwitchContainer"
  input_side_packet: "ENABLE:with_attention"
  input_stream: "TENSORS:ensured_landmark_tensors"
  output_stream: "LANDMARKS:landmarks"
  options: {
    [mediapipe.SwitchContainerOptions.ext] {
      contained_node: {
        calculator: "TensorsToFaceLandmarks"
      }
      contained_node: {
        calculator: "TensorsToFaceLandmarksWithAttention"
      }
    }
  }
}

# Projects the landmarks from the cropped face image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
  calculator: "LandmarkProjectionCalculator"
  input_stream: "NORM_LANDMARKS:landmarks"
  input_stream: "NORM_RECT:roi"
  output_stream: "NORM_LANDMARKS:face_landmarks"
}