mediapipe/mediapipe/framework/calculator.proto

// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Forked from mediapipe/framework/calculator.proto.
// The forked proto must remain identical to the original proto and should be
// ONLY used by mediapipe open source project.
syntax = "proto3";

package mediapipe;

import public "mediapipe/framework/calculator_options.proto";

import "google/protobuf/any.proto";
import "mediapipe/framework/mediapipe_options.proto";
import "mediapipe/framework/packet_factory.proto";
import "mediapipe/framework/packet_generator.proto";
import "mediapipe/framework/status_handler.proto";
import "mediapipe/framework/stream_handler.proto";

option java_package = "com.google.mediapipe.proto";
option java_outer_classname = "CalculatorProto";

// Describes a MediaPipe Executor.
message ExecutorConfig {
  // The name of the executor (used by a CalculatorGraphConfig::Node or
  // PacketGeneratorConfig to specify which executor it will execute on).
  // This field must be unique within a CalculatorGraphConfig. If this field
  // is omitted or is an empty string, the ExecutorConfig describes the
  // default executor.
  //
  // NOTE: The names "default" and "gpu" are reserved and must not be used.
  string name = 1;
  // The registered type of the executor. For example: "ThreadPoolExecutor".
  // The framework will create an executor of this type (with the options in
  // the options field) for the CalculatorGraph.
  //
  // The ExecutorConfig for the default executor may omit this field and let
  // the framework choose an appropriate executor type. Note: If the options
  // field is used in this case, it should contain the
  // ThreadPoolExecutorOptions.
  //
  // If the ExecutorConfig for an additional (non-default) executor omits this
  // field, the executor must be created outside the CalculatorGraph and
  // passed to the CalculatorGraph for use.
  string type = 2;
  // The options passed to the Executor. The extension in the options field
  // must match the type field. For example, if the type field is
  // "ThreadPoolExecutor", then the options field should contain the
  // ThreadPoolExecutorOptions.
  MediaPipeOptions options = 3;
}

// A collection of input data to a CalculatorGraph.
message InputCollection {
  // The name of the input collection.  Name must match [a-z_][a-z0-9_]*
  string name = 1;
  // The names of each side packet.  The number of side_packet_name
  // must match the number of packets generated by the input file.
  repeated string side_packet_name = 2;
  // DEPRECATED: old way of referring to side_packet_name.
  repeated string external_input_name = 1002;

  // The input can be specified in several ways.
  enum InputType {
    // An invalid default value.  This value is guaranteed to be the
    // lowest enum value (i.e. don't add negative enum values).
    UNKNOWN = 0;
    // A recordio where each record is a serialized PacketManagerConfig.
    // Each PacketManagerConfig must have the same number of packet
    // factories in it as the number of side packet names.  Furthermore,
    // the output side packet name field in each PacketFactoryConfig
    // must not be set.  This is the most general input, and allows
    // multiple side packet values to be set in arbitrarily complicated
    // ways before each run.
    RECORDIO = 1;
    // A recordio where each record is a serialized packet payload.
    // For example a recordio of serialized OmniaFeature protos dumped
    // from Omnia.
    FOREIGN_RECORDIO = 2;
    // A text file where each line is a comma separated list. The number
    // of elements for each csv string must be the same as the number
    // of side_packet_name (and the order must match).  Each line must
    // be less than 1MiB in size.  Lines comprising of only whitespace
    // or only whitespace and a pound comment will be skipped.
    FOREIGN_CSV_TEXT = 3;
    // This and all higher values are invalid.  Update this value to
    // always be larger than any other enum values you add.
    INVALID_UPPER_BOUND = 4;
  }
  // Sets the source of the input collection data.
  // The default value is UNKNOWN.
  InputType input_type = 3;
  // A file name pointing to the data.  The format of the data is
  // specified by the "input_type" field.  Multiple shards may be
  // specified using @N or glob expressions.
  string file_name = 4;
}

// A convenient way to specify a number of InputCollections.
message InputCollectionSet {
  repeated InputCollection input_collection = 1;
}

// Additional information about an input stream.
message InputStreamInfo {
  // A description of the input stream.
  // This description uses the Calculator visible specification of
  // a stream.  The format is a tag, then an index with both being
  // optional.  If the tag is missing it is assumed to be "" and if
  // the index is missing then it is assumed to be 0.  If the index
  // is provided then a colon (':') must be used.
  // Examples:
  //   "TAG"     -> tag "TAG",   index 0
  //   ""        -> tag "",      index 0
  //   ":0"      -> tag "",      index 0
  //   ":3"      -> tag "",      index 3
  //   "VIDEO:0" -> tag "VIDEO", index 0
  //   "VIDEO:2" -> tag "VIDEO", index 2
  string tag_index = 1;
  // Whether the input stream is a back edge.
  // By default, MediaPipe requires graphs to be acyclic and treats cycles in a
  // graph as errors. To allow MediaPipe to accept a cyclic graph, set the
  // back_edge fields of the input streams that are back edges to true. A
  // cyclic graph usually has an obvious forward direction, and a back edge
  // goes in the opposite direction. For a formal definition of a back edge,
  // please see https://en.wikipedia.org/wiki/Depth-first_search.
  bool back_edge = 2;
}

// Configs for the profiler for a calculator. Not applicable to subgraphs.
message ProfilerConfig {
  // Size of the runtimes histogram intervals (in microseconds) to generate the
  // histogram of the Process() time. The last interval extends to +inf.
  // If not specified, the interval is 1000000 usec = 1 sec.
  int64 histogram_interval_size_usec = 1;

  // Number of intervals to generate the histogram of the Process() runtime.
  // If not specified, one interval is used.
  int64 num_histogram_intervals = 2;

  // TODO: clean up after migration to MediaPipeProfiler.
  // DEPRECATED: If true, the profiler also profiles the input output latency.
  // Should be true only if the packet timestamps corresponds to the
  // microseconds wall time from epoch.
  bool enable_input_output_latency = 3 [deprecated = true];

  // If true, the profiler starts profiling when graph is initialized.
  bool enable_profiler = 4;

  // If true, the profiler also profiles the stream latency and input-output
  // latency.
  // No-op if enable_profiler is false.
  bool enable_stream_latency = 5;

  // If true, the profiler uses packet timestamp (as production time and source
  // production time) for packets added by calling
  // CalculatorGraph::AddPacketToInputStream().
  // If false, uses profiler's clock.
  bool use_packet_timestamp_for_added_packet = 6;

  // The maximum number of trace events buffered in memory.
  // The default value buffers up to 20000 events.
  int64 trace_log_capacity = 7;

  // Trace event types that are not logged.
  repeated int32 trace_event_types_disabled = 8;

  // The output directory and base-name prefix for trace log files.
  // Log files are written to: StrCat(trace_log_path, index, ".binarypb")
  string trace_log_path = 9;

  // The number of trace log files retained.
  // The trace log files are named "trace_0.log" through "trace_k.log".
  // The default value specifies 2 output files retained.
  int32 trace_log_count = 10;

  // The interval in microseconds between trace log output.
  // The default value specifies trace log output once every 0.5 sec.
  int64 trace_log_interval_usec = 11;

  // The interval in microseconds between TimeNow and the highest times
  // included in trace log output.  This margin allows time for events
  // to be appended to the TraceBuffer.
  int64 trace_log_margin_usec = 12;

  // Deprecated, replaced by trace_log_instant_events.
  bool trace_log_duration_events = 13 [deprecated = true];

  // The number of trace log intervals per file. The total log duration is:
  // trace_log_interval_usec * trace_log_file_count * trace_log_interval_count.
  // The default value specifies 10 intervals per file.
  int32 trace_log_interval_count = 14;

  // An option to turn ON/OFF writing trace files to disk. Saving trace files to
  // disk is enabled by default.
  bool trace_log_disabled = 15;

  // If true, tracer timing events are recorded and reported.
  bool trace_enabled = 16;

  // False specifies an event for each calculator invocation.
  // True specifies a separate event for each start and finish time.
  bool trace_log_instant_events = 17;

  // Limits calculator-profile histograms to a subset of calculators.
  string calculator_filter = 18;
}

// Describes the topology and function of a MediaPipe Graph.  The graph of
// Nodes must be a Directed Acyclic Graph (DAG) except as annotated by
// "back_edge" in InputStreamInfo.  Use a mediapipe::CalculatorGraph object to
// run the graph.
message CalculatorGraphConfig {
  // A single node in the DAG.
  message Node {
    // The name of the node.  This field is optional and doesn't generally
    // need to be specified, but does improve error messaging.
    string name = 1;
    // The registered type of a calculator (provided via REGISTER_CALCULATOR),
    // or of a subgraph (via REGISTER_MEDIAPIPE_GRAPH).
    string calculator = 2;
    // A Calculator can choose to access its input streams, output
    // streams, and input side packets either by tag or by index.  If the
    // calculator chooses indexes then it will receive the streams or side
    // packets in the same order as they are specified in this proto.
    // If the calculator chooses to use tags then it must specify a
    // tag along with each name.  The field is given as "TAG:name".
    // Meaning a tag name followed by a colon followed by the name.
    // Tags use only upper case letters, numbers, and underscores, whereas
    // names use only lower case letters, numbers, and underscores.
    // Example:
    //   Node {
    //     calculator: "SomeAudioVideoCalculator"
    //     # This calculator accesses its inputs by index (no tag needed).
    //     input_stream: "combined_input"
    //     # This calculator accesses its outputs by tags, so all
    //     # output_streams must specify a tag.
    //     output_stream: "AUDIO:audio_stream"
    //     output_stream: "VIDEO:video_stream"
    //     # This calculator accesses its input side packets by tag.
    //     input_side_packet: "MODEL:model_01"
    //   }

    // String(s) representing "TAG:name" of the stream(s) from which the current
    // node will get its inputs. "TAG:" part is optional, see above.
    // A calculator with no input stream is a source.
    repeated string input_stream = 3;
    // String(s) representing "TAG:name" of the stream(s) produced by this node.
    // "TAG:" part is optional, see above. These must be different from any
    // other output_streams specified for other nodes in the graph.
    repeated string output_stream = 4;
    // String(s) representing "TAG:name" of the input side packet(s).
    // "TAG:" part is optional, see above.
    repeated string input_side_packet = 5;
    // String(s) representing "TAG:name" of the output side packet(s). Only
    // used by subgraphs.
    // "TAG:" part is optional, see above.
    repeated string output_side_packet = 6;
    // The options passed to the Calculator, in proto2 syntax.
    CalculatorOptions options = 7;
    // The options passed to the Calculator, in proto3 syntax.
    // Each node_options message must have a different message type.
    // If the same message type is specified in |options| and |node_options|,
    // only the message in |options| is used.
    repeated google.protobuf.Any node_options = 8;

    // Note: the following fields are only applicable to calculators, not
    // subgraphs.

    // For a Source Calculator (i.e. a calculator with no inputs),
    // this is the "layer" on which the calculator is executed.  For a
    // non-source calculator (i.e. a calculator with one or more input
    // streams) this field has no effect.  The sources on each layer
    // are completely exhausted before Process() is called on any source
    // calculator on a higher numbered layer.
    // Example:
    //   Decoder -> Median Frame (requires all frames) -> Image Subtraction
    //           --------------------------------------->
    // The entire video will be buffered on the edge from the decoder
    // to the Image subtraction.  To fix this problem, layers can be used.
    //   Decoder (layer 0) -> Median Frame -> Image Subtraction
    //   Decoder (layer 1) ----------------->
    // The frames from layer 0 will no longer be buffered, but the video
    // will be decoded again instead.  Note, that different options can
    // be used in the second decoder.
    int32 source_layer = 9;
    // Optional parameter that allows the user to indicate to the scheduler that
    // this node has a buffering behavior (i.e. waits for a bunch of packets
    // before emitting any) and specify the size of the buffer that is built up.
    // The scheduler will then try to keep the maximum size of any input queues
    // in the graph to remain below the maximum of all buffer_size_hints and
    // max_queue_size (if specified). The ideal value is typically something
    // larger than the actual number of buffered packets to maintain pipelining.
    // The default value 0 indicates that the node has no buffering behavior.
    int32 buffer_size_hint = 10;
    // Config for this node's InputStreamHandler.
    // If unspecified, the graph-level input stream handler will be used.
    InputStreamHandlerConfig input_stream_handler = 11;
    // Config for this node's OutputStreamHandler.
    // If unspecified, the graph-level output stream handler will be used.
    OutputStreamHandlerConfig output_stream_handler = 12;
    // Additional information about an input stream. The |name| field of the
    // InputStreamInfo must match an input_stream.
    repeated InputStreamInfo input_stream_info = 13;
    // Set the executor which the calculator will execute on.
    string executor = 14;
    // TODO: Remove from Node when switched to Profiler.
    // DEPRECATED: Configs for the profiler.
    ProfilerConfig profiler_config = 15 [deprecated = true];
    // The maximum number of invocations that can be executed in parallel.
    // If not specified, the limit is one invocation.
    int32 max_in_flight = 16;
    // Defines an option value for this Node from graph options or packets.
    repeated string option_value = 17;
    // DEPRECATED: For backwards compatibility we allow users to
    // specify the old name for "input_side_packet" in proto configs.
    // These are automatically converted to input_side_packets during
    // config canonicalization.
    repeated string external_input = 1005;
  }

  // The nodes.
  repeated Node node = 1;
  // Create a side packet using a PacketFactory.  This side packet is
  // created as close to the worker that does the work as possible.  A
  // PacketFactory is basically a PacketGenerator that takes no input side
  // packets and produces a single output side packet.
  repeated PacketFactoryConfig packet_factory = 6;
  // Configs for PacketGenerators.  Generators take zero or more
  // input side packets and produce any number of output side
  // packets.  For example, MediaDecoderCalculator takes an input
  // side packet with type DeletingFile.  However, most users want
  // to specify videos by ContentIdHex (i.e. video id).  By using
  // the VideoIdToLocalFileGenerator, a user can specify a video id
  // (as a string) and obtain a DeletingFile to use with the decoder.
  // PacketGenerators can take as a input side packet the output side
  // packet of another PacketGenerator.  The graph of PacketGenerators
  // must be a directed acyclic graph.
  // DEPRECATED: Use a Node to generate side packets.
  repeated PacketGeneratorConfig packet_generator = 7 [deprecated = true];
  // Number of threads for running calculators in multithreaded mode.
  // If not specified, the scheduler will pick an appropriate number
  // of threads depending on the number of available processors.
  // To run on the calling thread, specify "ApplicationThreadExecutor"
  // see: http://g3doc/mediapipe/g3doc/running.md.
  int32 num_threads = 8;
  // Configs for StatusHandlers that will be called after each call to
  // Run() on the graph.  StatusHandlers take zero or more input side
  // packets and the absl::Status returned by a graph run.  For example,
  // a StatusHandler could store information about graph failures and
  // their causes for later monitoring.  Note that graph failures during
  // initialization may cause required input side packets (created by a
  // PacketFactory or PacketGenerator) to be missing.  In these cases,
  // the handler with missing input side packets will be skipped.
  repeated StatusHandlerConfig status_handler = 9;
  // Specify input streams to the entire graph.  Streams specified here may have
  // packets added to them using CalculatorGraph::AddPacketToInputStream.  This
  // works much like a source calculator, except that the source is outside of
  // the mediapipe graph.
  repeated string input_stream = 10;
  // Output streams for the graph when used as a subgraph.
  repeated string output_stream = 15;
  // Input side packets for the graph when used as a subgraph.
  repeated string input_side_packet = 16;
  // Output side packets for the graph when used as a subgraph.
  repeated string output_side_packet = 17;
  // Maximum queue size of any input stream in the graph. This can be used to
  // control the memory usage of a MediaPipe graph by preventing fast sources
  // from flooding the graph with packets. Any source that is connected to an
  // input stream that has hit its maximum capacity will not be scheduled until
  // the queue size falls under the specified limits, or if the scheduler queue
  // is empty and no other nodes are running (to prevent possible deadlocks due
  // to a incorrectly specified value). This global parameter is set to 100
  // packets by default to enable pipelining. If any node indicates that it
  // buffers packets before emitting them, then the max(buffer_size_hint,
  // max_queue_size) is used. Set this parameter to -1 to disable throttling
  // (i.e. the graph will use as much memory as it requires). If not specified,
  // the limit is 100 packets.
  int32 max_queue_size = 11;
  // If true, the graph run fails with an error when throttling prevents all
  // calculators from running.  If false, max_queue_size for an input stream
  // is adjusted when throttling prevents all calculators from running.
  bool report_deadlock = 21;
  // Config for this graph's InputStreamHandler.
  // If unspecified, the framework will automatically install the default
  // handler, which works as follows.
  // The calculator's Process() method is called for timestamp t when:
  // - at least one stream has a packet available at t; and,
  // - all other streams either have packets at t, or it is known that they will
  // not have packets at t (i.e. their next timestamp bound is greater than t).
  // The handler then provides all available packets with timestamp t, with no
  // preprocessing.
  InputStreamHandlerConfig input_stream_handler = 12;
  // Config for this graph's OutputStreamHandler.
  // If unspecified, the default output stream handler will be automatically
  // installed by the framework which does not modify any outgoing packets.
  OutputStreamHandlerConfig output_stream_handler = 13;
  // Configs for Executors.
  // The names of the executors must be distinct. The default executor, whose
  // name is the empty string, is predefined. The num_threads field of the
  // CalculatorGraphConfig specifies the number of threads in the default
  // executor. If the config for the default executor is specified, the
  // CalculatorGraphConfig must not have the num_threads field.
  repeated ExecutorConfig executor = 14;
  // The default profiler-config for all calculators.  If set, this defines the
  // profiling settings such as num_histogram_intervals for every calculator in
  // the graph.  Each of these settings can be overridden by the
  // |profiler_config| specified for a node.
  ProfilerConfig profiler_config = 18;

  // The namespace used for class name lookup within this graph.
  // An unqualified or partially qualified class name is looked up in
  // this namespace first and then in enclosing namespaces.
  string package = 19;

  // The type name for the graph config, used for registering and referencing
  // the graph config.
  string type = 20;

  // The types and default values for graph options, in proto2 syntax.
  MediaPipeOptions options = 1001;

  // The types and default values for graph options, in proto3 syntax.
  repeated google.protobuf.Any graph_options = 1002;
}