mediapipe-rs/mediapipe/modules/iris_landmark/iris_landmark_gpu.pbtxt

# MediaPipe subgraph to calculate iris landmarks and eye contour landmarks for
# a single eye. (GPU input, and inference is executed on GPU.)
#
# It is required that "iris_landmark.tflite" is available at
# "mediapipe/modules/iris_landmark/iris_landmark.tflite"
# path during execution.
#
# EXAMPLE:
#   node {
#     calculator: "IrisLandmarkGpu"
#     input_stream: "IMAGE:image"
#     input_stream: "ROI:eye_roi"
#     input_stream: "IS_RIGHT_EYE:is_right_eye"
#     output_stream: "EYE_CONTOUR_LANDMARKS:eye_contour_landmarks"
#     output_stream: "IRIS_LANDMARKS:iris_landmarks"
#   }

type: "IrisLandmarkGpu"

# GPU buffer. (GpuBuffer)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where an eye is located.
# (NormalizedRect)
input_stream: "ROI:roi"
# Is right eye. (bool)
# (Model is trained to detect left eye landmarks only, hence for right eye,
# flipping is required to immitate left eye.)
input_stream: "IS_RIGHT_EYE:is_right_eye"

# TfLite model to detect iris landmarks.
# (std::unique_ptr<tflite::FlatBufferModel,
#      std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: currently, mediapipe/modules/iris_landmark/iris_landmark.tflite model
# only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:model"

# 71 refined normalized eye contour landmarks. (NormalizedLandmarkList)
output_stream: "EYE_CONTOUR_LANDMARKS:projected_eye_landmarks"
# 5 normalized iris landmarks. (NormalizedLandmarkList)
output_stream: "IRIS_LANDMARKS:projected_iris_landmarks"

node {
  calculator: "ImageCroppingCalculator"
  input_stream: "IMAGE_GPU:image"
  input_stream: "NORM_RECT:roi"
  output_stream: "IMAGE_GPU:eye_image"
  options: {
    [mediapipe.ImageCroppingCalculatorOptions.ext] {
      border_mode: BORDER_REPLICATE
    }
  }
}

node {
  calculator: "ImageTransformationCalculator"
  input_stream: "IMAGE_GPU:eye_image"
  input_stream: "FLIP_HORIZONTALLY:is_right_eye"
  output_stream: "IMAGE_GPU:transformed_eye_image"
  output_stream: "LETTERBOX_PADDING:eye_letterbox_padding"
  options: {
    [mediapipe.ImageTransformationCalculatorOptions.ext] {
      output_width: 64
      output_height: 64
      scale_mode: FIT
    }
  }
}

# Converts the transformed input image on CPU into an image tensor stored as a
# TfLiteTensor.
node {
  calculator: "TfLiteConverterCalculator"
  input_stream: "IMAGE_GPU:transformed_eye_image"
  output_stream: "TENSORS_GPU:image_tensor"
  options: {
    [mediapipe.TfLiteConverterCalculatorOptions.ext] {
      zero_center: false
    }
  }
}

# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
  calculator: "TfLiteInferenceCalculator"
  input_stream: "TENSORS_GPU:image_tensor"
  output_stream: "TENSORS:output_tensors"
  options: {
    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
      model_path: "mediapipe/modules/iris_landmark/iris_landmark.tflite"
    }
  }
}

# Splits a vector of TFLite tensors to multiple vectors according to the ranges
# specified in option.
node {
  calculator: "SplitTfLiteTensorVectorCalculator"
  input_stream: "output_tensors"
  output_stream: "eye_landmarks_tensor"
  output_stream: "iris_landmarks_tensor"
  options: {
    [mediapipe.SplitVectorCalculatorOptions.ext] {
      ranges: { begin: 0 end: 1 }
      ranges: { begin: 1 end: 2 }
    }
  }
}

# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
  calculator: "TfLiteTensorsToLandmarksCalculator"
  input_stream: "TENSORS:iris_landmarks_tensor"
  input_stream: "FLIP_HORIZONTALLY:is_right_eye"
  output_stream: "NORM_LANDMARKS:iris_landmarks"
  options: {
    [mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
      num_landmarks: 5
      input_image_width: 64
      input_image_height: 64
    }
  }
}

# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
  calculator: "TfLiteTensorsToLandmarksCalculator"
  input_stream: "TENSORS:eye_landmarks_tensor"
  input_stream: "FLIP_HORIZONTALLY:is_right_eye"
  output_stream: "NORM_LANDMARKS:eye_landmarks"
  options: {
    [mediapipe.TfLiteTensorsToLandmarksCalculatorOptions.ext] {
      num_landmarks: 71
      input_image_width: 64
      input_image_height: 64
    }
  }
}

node {
  calculator: "LandmarkLetterboxRemovalCalculator"
  input_stream: "LANDMARKS:0:iris_landmarks"
  input_stream: "LANDMARKS:1:eye_landmarks"
  input_stream: "LETTERBOX_PADDING:eye_letterbox_padding"
  output_stream: "LANDMARKS:0:padded_iris_landmarks"
  output_stream: "LANDMARKS:1:padded_eye_landmarks"
}

# Projects the landmarks from the cropped face image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
  calculator: "LandmarkProjectionCalculator"
  input_stream: "NORM_LANDMARKS:0:padded_iris_landmarks"
  input_stream: "NORM_LANDMARKS:1:padded_eye_landmarks"
  input_stream: "NORM_RECT:roi"
  output_stream: "NORM_LANDMARKS:0:projected_iris_landmarks"
  output_stream: "NORM_LANDMARKS:1:projected_eye_landmarks"
}