// Copyright 2019 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { // Implementation of the input stream handler for the MuxCalculator. // // One of the input streams is the control stream; all the other input streams // are data streams. To make MuxInputStreamHandler work properly, the tag of the // input streams must obey the following rules: // Let N be the number of input streams. Data streams must use tag "INPUT" with // index 0, ..., N - 2; the control stream must use tag "SELECT". // // The control stream carries packets of type 'int'. The 'int' value in a // control stream packet must be a valid index in the range 0, ..., N - 2 and // select the data stream at that index. The selected data stream must have a // packet with the same timestamp as the control stream packet. // // When the control stream is done, GetNodeReadiness() returns // NodeReadiness::kReadyForClose. // // TODO: pass the input stream tags to the MuxInputStreamHandler // constructor so that it can refer to input streams by tag. See b/30125118. class MuxInputStreamHandler : public InputStreamHandler { public: MuxInputStreamHandler() = delete; MuxInputStreamHandler(std::shared_ptr tag_map, CalculatorContextManager* cc_manager, const MediaPipeOptions& options, bool calculator_run_in_parallel) : InputStreamHandler(std::move(tag_map), cc_manager, options, calculator_run_in_parallel) {} protected: // In MuxInputStreamHandler, a node is "ready" if: // - the control stream is done (need to call Close() in this case), or // - we have received the packets on the control stream and the selected data // stream at the next timestamp. NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { DCHECK(min_stream_timestamp); absl::MutexLock lock(&input_streams_mutex_); const auto& control_stream = input_stream_managers_.Get(input_stream_managers_.EndId() - 1); bool empty; *min_stream_timestamp = control_stream->MinTimestampOrBound(&empty); if (empty) { if (*min_stream_timestamp == Timestamp::Done()) { // Calculator is done if the control input stream is done. return NodeReadiness::kReadyForClose; } // Calculator is not ready to run if the control input stream is empty. return NodeReadiness::kNotReady; } Packet control_packet = control_stream->QueueHead(); CHECK(!control_packet.IsEmpty()); int control_value = control_packet.Get(); CHECK_LE(0, control_value); CHECK_LT(control_value, input_stream_managers_.NumEntries() - 1); const auto& data_stream = input_stream_managers_.Get( input_stream_managers_.BeginId() + control_value); // Data stream may contain some outdated packets which failed to be popped // out during "FillInputSet". (This handler doesn't sync input streams, // hence "FillInputSet" can be triggerred before every input stream is // filled with packets corresponding to the same timestamp.) data_stream->ErasePacketsEarlierThan(*min_stream_timestamp); Timestamp stream_timestamp = data_stream->MinTimestampOrBound(&empty); if (empty) { if (stream_timestamp <= *min_stream_timestamp) { // "data_stream" didn't receive a packet corresponding to the current // "control_stream" packet yet. return NodeReadiness::kNotReady; } // "data_stream" timestamp bound update detected. return NodeReadiness::kReadyForProcess; } if (stream_timestamp > *min_stream_timestamp) { // The earliest packet "data_stream" holds corresponds to a control packet // yet to arrive, which means there won't be a "data_stream" packet // corresponding to the current "control_stream" packet, which should be // indicated as timestamp boun update. return NodeReadiness::kReadyForProcess; } CHECK_EQ(stream_timestamp, *min_stream_timestamp); return NodeReadiness::kReadyForProcess; } // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. void FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) override { CHECK(input_timestamp.IsAllowedInStream()); CHECK(input_set); absl::MutexLock lock(&input_streams_mutex_); const CollectionItemId control_stream_id = input_stream_managers_.EndId() - 1; auto& control_stream = input_stream_managers_.Get(control_stream_id); int num_packets_dropped = 0; bool stream_is_done = false; Packet control_packet = control_stream->PopPacketAtTimestamp( input_timestamp, &num_packets_dropped, &stream_is_done); CHECK_EQ(num_packets_dropped, 0) << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, control_stream->Name()); CHECK(!control_packet.IsEmpty()); int control_value = control_packet.Get(); AddPacketToShard(&input_set->Get(control_stream_id), std::move(control_packet), stream_is_done); const CollectionItemId data_stream_id = input_stream_managers_.BeginId() + control_value; CHECK_LE(input_stream_managers_.BeginId(), data_stream_id); CHECK_LT(data_stream_id, control_stream_id); auto& data_stream = input_stream_managers_.Get(data_stream_id); stream_is_done = false; Packet data_packet = data_stream->PopPacketAtTimestamp( input_timestamp, &num_packets_dropped, &stream_is_done); CHECK_EQ(num_packets_dropped, 0) << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, data_stream->Name()); AddPacketToShard(&input_set->Get(data_stream_id), std::move(data_packet), stream_is_done); // Discard old packets on other streams. // Note that control_stream_id is the last valid id. auto next_timestamp = input_timestamp.NextAllowedInStream(); for (CollectionItemId id = input_stream_managers_.BeginId(); id < control_stream_id; ++id) { if (id == data_stream_id) continue; auto& other_stream = input_stream_managers_.Get(id); other_stream->ErasePacketsEarlierThan(next_timestamp); } } private: // Must be acquired when manipulating the control and data streams to ensure // we have a consistent view of the two streams. absl::Mutex input_streams_mutex_; }; REGISTER_INPUT_STREAM_HANDLER(MuxInputStreamHandler); } // namespace mediapipe