From 017c1dc7eaf2d45ca95769194d9c83a86a82876d Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 10 May 2021 12:19:00 -0700 Subject: [PATCH] Project import generated by Copybara. GitOrigin-RevId: 2146b10f0a498f665f246e16033b686c7947b92d --- MANIFEST.in | 3 + docs/framework_concepts/framework_concepts.md | 9 + docs/framework_concepts/realtime.md | 187 ++++++++++++++++++ .../images/mobile/pose_tracking_pck_chart.png | Bin 0 -> 56981 bytes docs/solutions/pose.md | 38 ++-- mediapipe/calculators/core/BUILD | 16 ++ .../core/default_side_packet_calculator.cc | 3 +- mediapipe/calculators/image/BUILD | 2 + .../image/image_properties_calculator.cc | 96 +++++---- mediapipe/calculators/tensor/BUILD | 1 + .../image_to_tensor_converter_gl_buffer.cc | 2 +- .../tensor/image_to_tensor_converter_metal.cc | 2 +- .../image_to_tensor_converter_opencv.cc | 2 +- .../util/landmarks_smoothing_calculator.cc | 25 ++- .../util/landmarks_smoothing_calculator.proto | 10 +- ...efine_landmarks_from_heatmap_calculator.cc | 37 +++- ...refine_landmarks_from_heatmap_calculator.h | 3 +- ...ne_landmarks_from_heatmap_calculator.proto | 2 + ..._landmarks_from_heatmap_calculator_test.cc | 13 +- mediapipe/framework/BUILD | 2 + mediapipe/framework/formats/image.h | 3 +- mediapipe/framework/graph_validation_test.cc | 50 +++++ .../framework/profiler/graph_profiler.cc | 1 - mediapipe/framework/tool/BUILD | 2 + mediapipe/framework/tool/switch_container.cc | 39 ++++ .../framework/tool/switch_container_test.cc | 93 +++++++++ .../framework/tool/switch_demux_calculator.cc | 14 +- .../framework/tool/switch_mux_calculator.cc | 14 +- mediapipe/graphs/holistic_tracking/BUILD | 2 - .../holistic_tracking_cpu.pbtxt | 14 -- .../holistic_tracking_gpu.pbtxt | 14 -- mediapipe/graphs/pose_tracking/BUILD | 6 - .../pose_tracking/pose_tracking_cpu.pbtxt | 36 +--- .../pose_tracking/pose_tracking_gpu.pbtxt | 36 +--- .../hand_landmark_tracking_cpu.pbtxt | 2 +- .../holistic_landmark_cpu.pbtxt | 2 +- .../holistic_landmark_gpu.pbtxt | 2 +- .../modules/objectron/objectron_cpu.pbtxt | 2 +- .../pose_landmark_by_roi_cpu.pbtxt | 2 +- .../pose_landmark_by_roi_gpu.pbtxt | 2 +- .../pose_landmark/pose_landmark_cpu.pbtxt | 6 +- .../pose_landmark_filtering.pbtxt | 78 ++++---- .../pose_landmark/pose_landmark_gpu.pbtxt | 4 +- .../pose_landmark_model_loader.pbtxt | 3 +- mediapipe/python/solutions/download_utils.py | 37 ++++ mediapipe/python/solutions/hands.py | 6 +- mediapipe/python/solutions/holistic.py | 19 +- mediapipe/python/solutions/objectron.py | 34 ++-- mediapipe/python/solutions/pose.py | 19 +- mediapipe/util/filtering/one_euro_filter.cc | 5 +- mediapipe/util/filtering/one_euro_filter.h | 2 +- setup.py | 4 +- 52 files changed, 708 insertions(+), 298 deletions(-) create mode 100644 docs/framework_concepts/realtime.md create mode 100644 docs/images/mobile/pose_tracking_pck_chart.png create mode 100644 mediapipe/python/solutions/download_utils.py diff --git a/MANIFEST.in b/MANIFEST.in index ba8014db8..8d5c4ec50 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,3 +14,6 @@ exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite +exclude mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite +exclude mediapipe/modules/pose_landmark/pose_landmark_lite.tflite +exclude mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite diff --git a/docs/framework_concepts/framework_concepts.md b/docs/framework_concepts/framework_concepts.md index b39adf154..dcf446a9d 100644 --- a/docs/framework_concepts/framework_concepts.md +++ b/docs/framework_concepts/framework_concepts.md @@ -110,3 +110,12 @@ Other policies are also available, implemented using a separate kind of component known as an InputStreamHandler. See [Synchronization](synchronization.md) for more details. + +### Realtime data streams + +MediaPipe calculator graphs are often used to process streams of video or audio +frames for interactive applications. Normally, each Calculator runs as soon as +all of its input packets for a given timestamp become available. Calculators +used in realtime graphs need to define output timestamp bounds based on input +timestamp bounds in order to allow downstream calculators to be scheduled +promptly. See [Realtime data streams](realtime.md) for details. diff --git a/docs/framework_concepts/realtime.md b/docs/framework_concepts/realtime.md new file mode 100644 index 000000000..36b606825 --- /dev/null +++ b/docs/framework_concepts/realtime.md @@ -0,0 +1,187 @@ +--- +layout: default +title: Processing real-time data streams +nav_order: 6 +has_children: true +has_toc: false +--- + +# Processing real-time data streams +{: .no_toc } + +1. TOC +{:toc} +--- + +## Realtime timestamps + +MediaPipe calculator graphs are often used to process streams of video or audio +frames for interactive applications. The MediaPipe framework requires only that +successive packets be assigned monotonically increasing timestamps. By +convention, realtime calculators and graphs use the recording time or the +presentation time of each frame as its timestamp, with each timestamp indicating +the microseconds since `Jan/1/1970:00:00:00`. This allows packets from various +sources to be processed in a globally consistent sequence. + +## Realtime scheduling + +Normally, each Calculator runs as soon as all of its input packets for a given +timestamp become available. Normally, this happens when the calculator has +finished processing the previous frame, and each of the calculators producing +its inputs have finished processing the current frame. The MediaPipe scheduler +invokes each calculator as soon as these conditions are met. See +[Synchronization](synchronization.md) for more details. + +## Timestamp bounds + +When a calculator does not produce any output packets for a given timestamp, it +can instead output a "timestamp bound" indicating that no packet will be +produced for that timestamp. This indication is necessary to allow downstream +calculators to run at that timestamp, even though no packet has arrived for +certain streams for that timestamp. This is especially important for realtime +graphs in interactive applications, where it is crucial that each calculator +begin processing as soon as possible. + +Consider a graph like the following: + +``` +node { + calculator: "A" + input_stream: "alpha_in" + output_stream: "alpha" +} +node { + calculator: "B" + input_stream: "alpha" + input_stream: "foo" + output_stream: "beta" +} +``` + +Suppose: at timestamp `T`, node `A` doesn't send a packet in its output stream +`alpha`. Node `B` gets a packet in `foo` at timestamp `T` and is waiting for a +packet in `alpha` at timestamp `T`. If `A` doesn't send `B` a timestamp bound +update for `alpha`, `B` will keep waiting for a packet to arrive in `alpha`. +Meanwhile, the packet queue of `foo` will accumulate packets at `T`, `T+1` and +so on. + +To output a packet on a stream, a calculator uses the API functions +`CalculatorContext::Outputs` and `OutputStream::Add`. To instead output a +timestamp bound on a stream, a calculator can use the API functions +`CalculatorContext::Outputs` and `CalculatorContext::SetNextTimestampBound`. The +specified bound is the lowest allowable timestamp for the next packet on the +specified output stream. When no packet is output, a calculator will typically +do something like: + +``` +cc->Outputs().Tag("output_frame").SetNextTimestampBound( + cc->InputTimestamp().NextAllowedInStream()); +``` + +The function `Timestamp::NextAllowedInStream` returns the successive timestamp. +For example, `Timestamp(1).NextAllowedInStream() == Timestamp(2)`. + +## Propagating timestamp bounds + +Calculators that will be used in realtime graphs need to define output timestamp +bounds based on input timestamp bounds in order to allow downstream calculators +to be scheduled promptly. A common pattern is for calculators to output packets +with the same timestamps as their input packets. In this case, simply outputting +a packet on every call to `Calculator::Process` is sufficient to define output +timestamp bounds. + +However, calculators are not required to follow this common pattern for output +timestamps, they are only required to choose monotonically increasing output +timestamps. As a result, certain calculators must calculate timestamp bounds +explicitly. MediaPipe provides several tools for computing appropriate timestamp +bound for each calculator. + +1\. **SetNextTimestampBound()** can be used to specify the timestamp bound, `t + +1`, for an output stream. + +``` +cc->Outputs.Tag("OUT").SetNextTimestampBound(t.NextAllowedInStream()); +``` + +Alternatively, an empty packet with timestamp `t` can be produced to specify the +timestamp bound `t + 1`. + +``` +cc->Outputs.Tag("OUT").Add(Packet(), t); +``` + +The timestamp bound of an input stream is indicated by the packet or the empty +packet on the input stream. + +``` +Timestamp bound = cc->Inputs().Tag("IN").Value().Timestamp(); +``` + +2\. **TimestampOffset()** can be specified in order to automatically copy the +timestamp bound from input streams to output streams. + +``` +cc->SetTimestampOffset(0); +``` + +This setting has the advantage of propagating timestamp bounds automatically, +even when only timestamp bounds arrive and Calculator::Process is not invoked. + +3\. **ProcessTimestampBounds()** can be specified in order to invoke +`Calculator::Process` for each new "settled timestamp", where the "settled +timestamp" is the new highest timestamp below the current timestamp bounds. +Without `ProcessTimestampBounds()`, `Calculator::Process` is invoked only with +one or more arriving packets. + +``` +cc->SetProcessTimestampBounds(true); +``` + +This setting allows a calculator to perform its own timestamp bounds calculation +and propagation, even when only input timestamps are updated. It can be used to +replicate the effect of `TimestampOffset()`, but it can also be used to +calculate a timestamp bound that takes into account additional factors. + +For example, in order to replicate `SetTimestampOffset(0)`, a calculator could +do the following: + +``` +absl::Status Open(CalculatorContext* cc) { + cc->SetProcessTimestampBounds(true); +} + +absl::Status Process(CalculatorContext* cc) { + cc->Outputs.Tag("OUT").SetNextTimestampBound( + cc->InputTimestamp().NextAllowedInStream()); +} +``` + +## Scheduling of Calculator::Open and Calculator::Close + +`Calculator::Open` is invoked when all required input side-packets have been +produced. Input side-packets can be provided by the enclosing application or by +"side-packet calculators" inside the graph. Side-packets can be specified from +outside the graph using the API's `CalculatorGraph::Initialize` and +`CalculatorGraph::StartRun`. Side packets can be specified by calculators within +the graph using `CalculatorGraphConfig::OutputSidePackets` and +`OutputSidePacket::Set`. + +Calculator::Close is invoked when all of the input streams have become `Done` by +being closed or reaching timestamp bound `Timestamp::Done`. + +**Note:** If the graph finishes all pending calculator execution and becomes +`Done`, before some streams become `Done`, then MediaPipe will invoke the +remaining calls to `Calculator::Close`, so that every calculator can produce its +final outputs. + +The use of `TimestampOffset` has some implications for `Calculator::Close`. A +calculator specifying `SetTimestampOffset(0)` will by design signal that all of +its output streams have reached `Timestamp::Done` when all of its input streams +have reached `Timestamp::Done`, and therefore no further outputs are possible. +This prevents such a calculator from emitting any packets during +`Calculator::Close`. If a calculator needs to produce a summary packet during +`Calculator::Close`, `Calculator::Process` must specify timestamp bounds such +that at least one timestamp (such as `Timestamp::Max`) remains available during +`Calculator::Close`. This means that such a calculator normally cannot rely upon +`SetTimestampOffset(0)` and must instead specify timestamp bounds explicitly +using `SetNextTimestampBounds()`. diff --git a/docs/images/mobile/pose_tracking_pck_chart.png b/docs/images/mobile/pose_tracking_pck_chart.png new file mode 100644 index 0000000000000000000000000000000000000000..8b781e630334d26a13002a464c091814d5cf740c GIT binary patch literal 56981 zcmeFYhgXxqwl@rjB1%yZ5JW%}kQSN^<}BJ@cErpIeyS;XWa9 zf`x^J+wks9OBR-+Pgq#kmX9A}=2*1bRk5&~zOw!xWuutFPCzU&xk&MCM+y`;#6*9Y}Ouv^@N+4W8@M*8Kv$M9P zhwjorBfF?&^DKgA^Vo{c{8Z)-Iy>Jz!Kj(Th%1NoP0eL$QEw!S@X3j9Pu0W~;NU|A zr+_Cxy%#+m^uS*fxprT<9QqW7`0xz&)wYi+<`DIQ-!QbMu-YhD?35J zS@H6*a_5uNta1kG)4XdRT>{Ru#Ap0a_;9>Ffmkd)$DyjsJ|!^v zcvr%$pRXzDY?{n6@008IEF?48%5Pl1d!jCG(xBWz*ou>sC{kQh%>T{ym+Cmj{F7rP zr+;zp3g7l@|l-czw#|Tb#dWSUVT&k#?=H^AqW+P zvW}$+f9>pZZM;$|J9+fnW7E$b&N*%xu6xe*@8Q#W>!(7wl6t)s6#~UR92Y&8lPv$0 z^y&O3_{aGk?j^lN!*JbBT|Jhf$3lxIw^sh#KJYr=J^&qP8a#J43AlLk)Q89JkB1*0 zdCc=k{gbjwL6ylBsmN1q&xAj>em>!sgU~iLmfGdgr6rG~uLvq8x8;UrhUQaj9q+)aM?kpZg<9ARdg516ZU}eb z?dP|XZ==3?nzmP>eqetfSf)w^F<;qlI8P*qXcPc_E*w#tsGy{olbC|cBKL>Y4h=LVZ+ zYYS-lVQ*#Y(EL~tu6Q&%N`Y3ZY+Y}Cy+)*l)u%~)ObgRK5c%z?2fsJy(U`}Mcm4Od zGNe5r6hrqQKFoKI>Y;SQUpT&ZEOAVlKNoc}DkUb1uE_D?MqICVQfSgtN_L7)Usp<4 zL6w<{84=!I(5=;>nGDZ|-*I^9VCfLr!KG{^{QBjqm&WyHZN#T0>SY}Dk-(0X_VXdM z-uYSGv99m*8J)3+&Ik@Xcf{H7_qvpbex21q*+Tuu!Oq>TX(x53)GqEW?e_1MkI|-Ez~W(2ur7uiqmhxtNM++<8$FW8cKZmNEr%oYm`lR- zgtlYc?}HzRS|9PbGA$ME{+oI#=x*>`_x>e|WQ)K@cg~6minb}P8+-`-pxmj*cJsYt z+bx}!S6(^rQPI)s2mRFhJ4H8VZq+?oOR2c6l;ZyqCcebq48+4~ZELTz$(yMFfb|zP zZghg9Z95fQ{67sou6b_vJmE9t5>o`0exuWDhex#qxn#7Zgk*3|kJ^=Ef3DoJOKuW` z+tq$})Ai@U327<4a+Q^n+6j)mV#e;}8)oNM{se^vIR$hfvB--$yiUTKUU_%yY-gK; zQ+NGVFGefDfQRKhHWsqs@Y9+1;9D2eB%1+HNqM~*sdGi}^qA{al0#<-G2e#=FwF1s z#fw`_QvSxF3c>S5YPFlsGHqTpft2;>(S`j z+f;d!^(aS9X$~|ytnM4QG4#t1Bm7`YP4WHens)1R^#D8FPU!$sy4~XJP%BDsRxn$j z+g=D&UwXIlk?HquVevS10r@k@$sCg%4Gj!2de;xy0yodOP`^LcpA?an-u8A*`|cM# zm$JmyVO4)zTtTxH3&SJ%d;TB&Px~XgCVYHia)WFQX1=w(_p8IMVvXBS*v9a_m7u=u zHTY-vD7@*#w-*%0+Z}D|`<4R@>kZr0-ZOPwQN0_yQ6HlG*U1Z5hdPj>9i4jTV`>=X z=!Q?7O0_I5b-oC3I1vBg$UKKI$2sp-Z(|?SBq#A9`5u{Y&Ot;(r1hMJ#z)i9iq|&= zlIq1N+H(uO^PLS(CM_e`MPG|WFhGOmh}5T|B(1NSlLyUPn2uCXYEH}bI<(y#>>s}- z%4bS9|3{g2tvT(KL)RVSc|&@}{mOAOKRr_&$Kc3ClaINLX7>o_=2g$6cEPZD{hWY} zNh*~PrS#b0L$Jz^MWnySUaqlNj+s+vNA%)}B_r&8LQS2nnQmWf$UZr|0SY0i=lGDO z92>%Rhl$!Ty~M9G`7jNo1=soZS@8v2m^Ot1k6&h;Illq-3~S!#?#>B=95C`Y-f&3q zW{G9%kHV~W@~IF-RnM7f1JuXDzVE4Z>D&7Ik?b@gEZt6$g=D(c;P6nHI&it|7@tfD~3v+aP?UuUWi4s4w z*uJ;pMwj_oxq;a#DoRnJC(tanCJuc15NQxz=_VQJo4zVh-e;~|I8j;7`n!TBH<_Qc zSQ2_P@Z+w7uGF{P*mzmC!{?Qro3eq&ia(x<8+a|x)fgpx$M&IwQ$nOwmSJzHI*F_&i%XM8AEu1-KC&N|wM4YZ)`hJIb z0t&rbdsr!U=>PHI(Uc#Xlp+=r0dhLu<6-6w*V_Gop@*p{iySk3oP{;Xi-nz;Vr4!e z%!h^LNcvM24(6MW`P?jK`)BFVC&fqpnPyx5ThRb?!_biV2Dt^fyZb{QL4vI`H4T_e zO?g>82!3E{qU8qh1GqedxVi(v{Q~}yu;_$qF_V7o!7dWve!l)tt#Dn*f7H-orvGLu zNlN^qO0bWvf3n47Q0Z@pk_tds>A%`$lIr};)w1vkclWiu z>E*{153>(FjVsDJ|ET|eO8&dYf1!Nv-;^p>RMh^J^k0hpH|c$-d(aJtAG6P3z5kw? z|0Mp`!v7@HQTogJzlh>Lr2d?BahCUVm*3K!sy}i%gV~V(L3_c`jDvo`mvaroIM>(*Sw>nW5}9rYr+0hHK^vQ zugNXRYjG^BZ0yHRN&IUfzWUU(t6cWzb-w?R{m+WmE=pYXz4dR4dd_ejKVg2)#op@w zH2RnN_z8QC|JB!jG^%`yE#VW$YFaSs-wcP9SvL94&i`k*$?Le{-)98cZb|*?`2Sfq zuK0b(zvGXU4RlH3vd*n1e)l;4C+Gh`$z<>R|6}j}NZ|j?-v3;D|G9Mjf3i4h4V3lR zY#f@qI~hwF*_#JHvg1AWyB%*n{RUf3n zH?0RtWc=^>cc32hIJ=v^>IpwyJNsVc)g9}VZ7Sr@3%E%mCb0$f!+ZwiQx`&;D`N(v zaxyV6N^?wQ^Mtj1Rt#mPl8SZDaokVY$!dr$FNkGOS3(gTjuD?PihU0&xF_-_QT(7d zdbd;@u4asApQsz;(O$QJt=PcI*Sl#O&=d`Sa6#l+wflT~$clehhdpwtYeOAEG1sT> zZsQK=I5$wm4q)h6Ohj*=o)zq14JB+>AF8$bYQ>s1AJJTS2sxxcR@#a6L~XbtVypX!yR93ud{JCh|KKMt%|ng$nAC=?e<&W_ ze<3MT-A8(9cKw!+qgD5xFG>_gJnYCOw1}43F}DJsciWA>=Z*exWF^n1-fJ?yR@*Sg zDbkC&TdD78U1xW{@b_bxOvSce6pt$IL#Ul?*a&B`UE#{h7QC2D)_raIY!FO~sL_Tg zf|{&RX6o~GDUGS5js`mN6J}xM1AHlLlsRhqrUw5?dw*p4j$bXjsJvUB@a`m9O`p+c zOrLq%7dsPEXfM{i_cK{6w(%lk^N<0Fz|pQc9bb5}5s;*vnWVe!0%ZiJ^jd?2`tzX64Xw(6EC#7Rri7bSYCD+21B^R;pKJ<0e0DMA!FZYe)^qbrG@zr_w}aHS z@~bozAkO%mgwC>Yn(_6VS;)@lrnEL>)#9`_KDbd|;r_f@G5G|Mc5qf5xh>O~))ekD zl~r=|u^PQ0CTxl8SX%eiD@f+y-XQ~*0Gf?Gq@!gk3oI$KJ`J`@g`u-vE5i^NlwUzO zsrYwT_YVA)B1CYTv4@JKF6lFtj>?-iU%S2An+!f!l=ei?I#3Ehf?$$oR9Af6W6G_|<)S#JiF z34VL71|VW&@rNk@eUCw@lW7u@evl|^H(IrS$rEreu}^ms+j#>d%xGLR7S*f^A&$oj z=JQRP^vDVwv(|D9|M^6XcpYC92L^;+E&A-`>R5_B7)w=;@-dHokE>tFAGt}2u7R_-h~_)RRI+g-A3 z6Acj6SqYj8?8_3T|GHy}-^pvoUM_=$3>e^vOs(_7B=Px`Nc&%sz1$je7KJ+ngv_FY zWto+GsgYLvXBd^S`<3{DMPW?%ZCif}O(LZROkYdixoq&$Ap7^SiLpg^&|uJV_9cNQ zv)f7>kJo>VZeNlWc#)*HTOwui>uP1|^kv21sECY)1$V|AL2t@RVRY10l+C{Mp}bPY z#k3QKrkntWkcs=ID>A)&mrZ=iP=At9Ri!WKK2Z_P$}>vKFJk4Qsm4#Pto{|ijuz)( zP@_`V)1e&iIIr%&Xv8)fi9A68=3)u>`uoTa^pvp9QiZmOG5iQ-)AM8}$q$mR{)4_5 z8yhj2QFz5?`cllBuk9h()QvC{t=829J$*)jpggg}^cxoZxz2^qO%zy<_wqtyRYkZXhL7 z4dGY@A=W8HYpp7ML%Xc}WGB&a(;K$i!=d*+dJR!D4Eou_5f1qN`x-DHe@iAg!sHA zG!>r`&yUfwwK0;0-*jXZ$B2O%G@58~7}gH~)3+ag|JNE0Uy)wfSihy%@;ducary7` z4=;5t$G1qB{gAP7}H-cX`NMLvA1xrn~rG-uxuyQ#a?4T^Vh`(GUF4dj%LV|9y^6_++EF!<_ zU_}XXOrKnBuIROumWfq~B~8KVB5;fxjtch}bX|1r)JG2SImRr_{lq$wP^g9)g1M;3 z@eQ1zoqb=&QN z7Y`LiKe^Y#DA~>Q2`cxZIu>Lmoe^~B7UYR~4Bb7-*oaC(;9$d1ht}54(^$`JY1-Hq zc7f3s_4E4l_bGm3-7ys-Ssm@^O^3!d`kc4j_1|RDhclQuKB!WAzPfYWUCHO^3hw=& zo5wNGzI5N0{cjUG4gg+~k)urAE$Eb^*aC7oGsze~oqkLNSbxx~e{ch&F%)=_wRN{7 zZwc$bd)?G$YmS`35mj+`AnK!f$gVmYvXxh~tP)Z)KGo#puE#l2?hj8&z)P+MjkZkI zHP2Nim8ED;9bG9G!>ZL4TwG?mQDf@7>|SwUYWnEK;FQVuPbeHJ)N_) z22u1I`x&wy5#urGJUYL7v<5pHY*UFl;@ir@3z-rQZK-cAT`q7-ZMhO33;}`cy73oa zKweYg`Yvdp!3fLZgsrZSp*#ev!M2j)-W~=S zt&13P>|H(g0c@0$r@k^$mp9t#G)W5<8tV>d%=Ejul7{-MvIc~}VQVwUix*!_^3k5z zxSj$+?w(v)@Y7c$fI@*nh6mE~hF^eNuRcHIv+1YWaC=Wt!_WJ%8vOhYD0)8RD91t` z3=IuP=Dh7u>5v%ZmuYnc)Hn?G*N*+2C_WcW|7)@E10P6RLXkz2FJubC7i+&;#t(x- zBWJwYj%YOBcKC5a``ZLe;zq($7jI5~ih8VVz12|jdPrr#5${5=LL+-q;Oh~CtFcJ( zC5~Q7{8jButGQX!0EAx$1NP4{^}(MHoh~s8y5*F!2ES&@b_2-jU8xW&7fMANg0f|^ zkXCu1t^U1^&F(1NpmzcqsjDD!*%rh2!OJKHx)}B}f-N*{I*I%#Y`Rb(+dY_B4jTNFmDMDnL)vt1Y_NQGj9@Um3B$98vl%M`1_cwiQUN}7v z>K)d}>_gOEiEK@Dk%X&huR^L;s#lC@7t&vx^DX|dw!yn`d&|^;z(MppM z@w}Gf-EqHWKSz4D9b)7OVWB6w&Y1ZbSeN_HH_#qu`T@Kv{nL|2kdY#Ss%7rW@CkS;G3uoUxPdfQxxaZYjc600?)kRkW{iS-n*L$#0As!!H zgkrxfWRzG-Bb?0*Q6&=#N-+Xw>a97ywskn8_j7izq1`#{!`5TICrO#fcb==Y>|wTy z;7YMGMTCz{q@t^? zYzmt)?!H&m?qI*q_t@CowIq{6o@t!wrzhoSW3iiBqQ}#7%35Al*Em$A=NgnDx~LhW*D>QVC1D9Gl0Psu-7?G_ z*$bO-&mqItgF46L%9$IAQ^o-b94(s5Uekua_Y)kA%PuzL>W`Z-n^DI5E&i1)lvfbT z{orBFTQP5L@;v3#p;XM?S$};;47hWJvJe=(p>ztSEbPXl%SY3H1X#*}x@d{fBLil%hgYyL-N1gsdw`0E@IgxM346Ib zAh3aGPq~9e)+uML%$>IY+3iZZ!(ZOwKJF==BNzdksiuAKT_RZq~`pn=%wdwegt&$K9;f6xGYc+Qu{8uBSkd&TsY~ zB|K_%DvVrq7&D1c4WW*=HoekYku~@7fO}17`3)P}mAi+x&5JTm@FOZ$c81`(hMd0O z81e*JCK(1FZznsjOo3x#KuJdQ=8A=ds6vN)p7i4IT#{&bB*4yAcG$p=q!(|#76ZkwkLQ-C+$miXhL1TZVf+d0u7I} z0_?X7;XXW))z>Po@BF%>-wv#rkpEeIrlLH=5|)FVhZYY!zd@tC6XnWCDr0{>TL

z7zx6MHn4XF$xpftmgP==jcpAHqIXVDtwh@NWf~@6TSGj|6h@77-1Akvab`I|BS(O$ zIv+O+`o}YiA981|2-3>fX={qjoFjc@6)o2=UtX2X-m#*O5I?bPVQ>zz0$*UfZ;9vi z1=>ZP-9>NMMGS-ks|__ITMM8+B@JG-NO~cZYsXBCUuT$URa4*St?=p8VPRbs=_Iv} z?;^{iE8(GX4KXAiV}Im31Cnnt^7wZ%WO0_YB`I?<6c}OXu9?+aw!HVgm&a!zma$U= zF$>zhAka7qd8kKV>th0F*!7b^3F-W+La0AqOy%LxMVU!s~I zma9IY#B|2L`OtXKCrW!^Zu^0+UPWN1gM+!Fa`k!TXR#^bGl)vqZU++yQG3Oc_gZL6 ziTrKO(2{xlSuMI6GBsWU0-M6Cu0E_(BE8@ZN{kwCSOI`-4vjyjvfKtNwj zWC1#ObXrnq(WmEv;>`Z}(#l4Edfrs0N5ut0baBbFA-Xj< zsm;}TiTK@doNLa>mn!cO5u>{Rd%2(Lj|y-VNAFm2|0(-a*b^w{qwY?CXC$@w*UPfLDYM$0W&ODy(~-Wn zq7;kdaoY7sFx{b(^S=dkHAe$k%2&b_JGAK0^&GX+lbL$oy^S|p0FcVHOlN1^01^|< zl#OCYd(qV)k@)WeW0%v)iL<H(}|4DPp^(y*%Kp4%3 z&`Vq8f3QkYlhH%|04aHUL_zjWqL6)8u>|L-pTml&AMqYd71l&K+jk4uC1n4qaTn-f z_jqQ9OC(79?m^dPOv4@#|K`DwPh_-czW2M2ZAW-Ci;+8un4Y_F^%@hAHcu;)(_vnA z#7Uq(;_#i<%TW;}M@;!hda}B@B4N642R?oyJ*ZLpmAp4`I65ctwn*N1oHg8L)5lI= zYNv5oQRoA~IB@~Ij6AKrasVJBTDV3cZiqm~$!grrqY{^7!#pO&qwb-2AYr+$-O^h$ z;~l=$g@slyu#{n-^~kx5AcrC)MbE=bC}{z@R=_do>{ACCQpXL~(f0EF0f2GnQ(pur zM2R3vHcKmj>~7dcRcciET@AC5{!)JaRQ-rQXZodRGoJ>*V2H7)Z zMq;`C)d$hxxv|n=aG}u-mK9v)W8S zM`k+TWAOd5y}a`2ABb!2k64i}Wv5M1Wce|XU4|q&tvzZ)J!~O@U^_G1F5Eo(<030@ zymXfQcAPURHs|?xr1Jviz6?5S)Hgm9+-&PfZ0tgmJ>$^?-5XThLPjoulL%1<*Yb6ITGhtDd)V* z6?5Vu)w1I{5Eq_Ln-A*ZUpFmp4&`W^jxJLVPWWVqdAEuUh-cz&n7^iOjD8)s+>=b)wjvydTzU zW~7!mx2UyavO6vB>d1@ZuXl3Vp(Ja+l6wBpM;F7k%e!BPsK?_?nl-q6Vqfo1I1oUb z*QH~t`=*jlcdR#|O(ywFHW2g6KNr+nm3~aQIMGg)CAZ`Wt_cDZJ!wau5N0t7EwO95 z`VG{oUQiL{Xbz0)#`cts6tG+Gl>`(60=u}8+%&1H{6Rtbny>u-T4ran0#wa&Vol89h z*@5JbN2(vSfa(wM@51UVDm$LQwC5-ji#nR`VtTyH8@e}zY6tIcKVj1&??w1=4Eada z(qk4pBlu34d8djewp=zz9&F(`nC{cN3SDV#>oMz*N8f`(TlEEZ$`_oe(M$d#VQBKm zlX&*W8GJ3K#^^*Itt!sPgto<97i)3$sS$9teAcfyMgiaw-&O={IxS^SC@!}=C=y+a@BJllT2n3JL9khDxUzo6TfSGDRE_~0-d zEbqPgrQN8ydG=k{6wldRZJx?bp^TtF4RT$isL{M^AHEaMgK&8Ej?T4*J@gb?o7B%R zn?ZC885w^Zr(?-Xv)o2lv~QOjxb8a{N=|_3D)lZf?%$YlCj}82x7pi=l zU`W0?)nw8t6{Y=anupPr(>RIXv&a^dNz-OsJ0BHTL!Mm#SAMsoaC;48$1jqrSc&+Rw@ExClX+K6l4XeZ{iM|NYbc4f8zvA*qJYOAl_ zt}enhv?dRR;%3P-Bo}edsn7VDMR@0+k84o=HPd-7M8i16zGwehH9Ksy!Hy=l`LZa{ z+?3>hWK=|%E;!Y5&+d)%n^~n+`7Y+gEOsbSaCQmroG$;&i*-uxQA?lgAdM3AP&g>c z5#kMQtTmf4s9d@2PcTNDqZ)scV}0+Ch2gx()5IxKiNn$EQNQe!4`znntJ z&8UZA&CCj*r=?{CNg2toG5huRR@P%|mx)1?4ARQGu_|Ij+YCFA?^OT&gBB69L#I`! zYAn)GRbVGl+b+S6^MsNot!77T*GK!&0?*BH>UhTJ{)|r;n|x!82ey7*v>Z2-VXhZ3 za#G$=6kFj(a~PbRycMA!6}zPA&lX5#3*87u>ve$#+j37X12@VGGdO|>h+3j8l zha^I_0&|PGA5@X$<_CaPc2pm%xHx9*8Ekp_a<*sP>JuWQY#Nz0-J4E>EfB;=1o z+l+%RI`mWq%<;xUCm=_)qi4NUbTOcGDof}MZGn=7Nd}s|zSi+p6MV9B{m+`BXSGRN z*yK;6;`d_jcaZ0~Oi*9ce}^%ze1~blW4o~+kiR|2v^s#W<`g>3xghp`j(Ws+o9W@%VF`qg33 zbw~ThGHG)SUp_sTDV%Oe7#O$?LCu6{8OPM2;ruNvK?~@i*Fg>E825>bHHw9Q{^Dh-(ylpiz>wZSKN-qE_8Z zR%9~l>BP>6NsZn&jj*GXLhZ78+YTNgg3(sFNyo4Ubr)<%q+4nf%1ab%V(a>$-B)09<|^1i0dIv%Y5dTVU3ZBL2gz10Ka#JcE}?d92)Zet_k zR}SR*^J4@#dUV~{d~wBXFt>%fa8PRtUZ3Nu=(sm}n!R*PRTJ@IN-!DdV)g}9_@;bw zdG?)FbA$G8PhFy|8ZrW}Ry3pYUh;@)s#Uo6)H&hF`hyk2Zj}MB|1I%{b}3;UsiQ4V z>#YZD#GFPqgg$`3>B*R?|FA%-qP1Ps_?I#>N^$f~n^d z9nM5B9iM>Bij_wm>H+ul;{7V?LsxqlO+e_)L0wAYTOt(UG*Cb_0x#?F8zl20SwOu!Z)2Wy-r9`_Pi zLW%P3Bc~+H1d8x;;8-(fJZFgR-=W^)Tpb zZxuuhAZ!7qVPA|UsbL){nAZC)DLCN3CL+dT{;u6dqaVdmi8OYJX{AG&DSE(7AD?%~ zhilzHULEw8*>>>yTOIu1QW;l6%xBLcxc?O1m{c5yD(-hcnVwe;smg1@)bX|%hutJe zIGp!vY-mlLti4D&CR&ZCD0IwG8mqxFTq|{P;TaAHGqj9E52N%RLs$_gE~j4dQ?3}wy-?Fp?0l~s%nLrZk1rh9JcD=Af>Y4YSxI*F39 z^8w?<4tjs@UWbjF#nEjR7$oy%PA^lbF9!1`(nW{@O^(Wbz? zDjY$y^6K?!Tohb~b^eZtQ*O!-U%4vM!brJl;j7xzhgCJg$~lOe(G!)J9cl zGi7L&vdt3TuOV=I;@#wGI|H*G0=XfUKBS|1C{p&))Kboe>>a9Cc^}#Z*KJtUy2vs4 zGQ;}g%tsjsYu-7ksobwtwjxaR!%0}MTOoGfSZ?+j92Prv2(gRDmt92^mVe4+dC--f z`@%k3)+T2uXAr7ge&350&p~!cA_dNF@;@?5tE+e-rx~KtKtF2vQ=2mFMvJA!-e?{! zSJeO;v9x7}By<2-&2H&`WVFwZZxRRmwLiQeIoDT`Ji1pKH0vCiNafqvFYKAyC&#pd z@Q$)Qy>l2Pa!d;EA^Az*y%80R(!;qX!6Mm8@A37&tP0;NOllB-{+{eK{<1K}wGErMsc@&w)9VkLqkqv% z0|E^sesqu}-ZXoV9p#(v)1EA>$B89e4{2VP+1C|)w?Ce&37qs0&n2mU{0SH=#e4CA zL3_1fUzRm9we|7FGk~Bzn*^@vi!0ebD5NbY{VKVVeZUfBy7b&mX-K6!CO)JdhtbgK zYl!5qNU0fco6LyhaMm^R6oJmm>EU(>`~FO&U7cE~myg{AIaRAk73a5%yG13`Rzu6= zU#utcE2FPE+Amly@tbLubUTjwtlcE}Uy6)vVJhy)lDsA}sq%P(5stq;R96Vb!0d9f zH_#mRBqH!SEI)ae?4NXS)zN?1H41R^t6FRq0TyvKvJfkwsF2p~W!ie-I(=?aop7J5 zfxJ5&wdI4PRTgM?;xcw-DG||YJGOhwn~8dwXDqg!fbT>tJh=kyUJFZ{^=RIZ=}t#E zaJ`60IpSpkLgW_2Ub?DM^)(DOL#~~mv`_&=64rGnjd&O zmv?)J03RUCU7MB$dGxf$cT4hD%UpBEy9Mfm-#?2DamsbFyC@T$w`WK?Vp#ung;E+S z)xo&B)P^JMxLS5vo;Gx^A5oKG<$U(Aui%849lllmB8Oi=Vq)3O2A)g$3#i^oH$2`A|w&Ed`ipi5> z?=2nt}a|7#?@0j}K%((k&?xe{q(m^Y3FWazUs=H@gdn1S< zgw#CzB-9~ zrMh&NwlVY)v~v%vRtsO#S@WT3Z{D$Sp7@HrqVdw7vRr$jc0N38x0JEpM_%dyBRcQP zibsC;JEk{pV5W@7FMr}>_Z}};@uYs&Z=|!?c6lubpTii~{*9z48t1E#sIYv1gr(O) z)Y3$UU3D(x;K3z{0{cq7-?l@nnuxEJQbBh1p2%&Mj!<`4H%Q6A}u1YJd{UyfgB zd34s==$57cdM<@Bn$sQppxJgZIQAe91<)v9>@b$I)n7-zg7XPo#lAcJNc;8SoiPEb z9MQBbcUK@=X0?J%-zH2VvEO2{qs3#eTOzBBlF`kfLWhm@gPNLOc{-xdjvK0k8;CUs zl<4N4g=WZ`&yI9IEO{|w$;6l%3{cYIGxcMifauw7P32z^b)R=9V+mO+*~U1v7F)kY zOa^+bE0&U$T{8SoO}cXw>nX%^qnOo;vU&2?f4MrfoaZ|%J>q=B->tIK5HU(={}FI& z9RctdcYn1rO&7Cf{4TBp5D~qMdjQFbFAoy{$oIKR#&&{)Kd*~tiiuXa3;`R5@1i-D zVVEsw$r z%CX8hQXDBdUeNNQw%j8&48eTM)9(;zN)!#wmR%=V$8|`O`hsOw^ITU}kIOP?4vK+$ z#k|H!lz->zkIdj7a&eOIRtlU%hMuHT!g?c%B*NIrl1-6ax(IY@M}qAygz-E>FmE9U zWv*u#2sT1JLBn%!++gQ&1O~uF`dVTr^(w$XUmS3P79+IsJ)fgUNG5GEi1QS$y!*c6 z26mzjf~tCJibl;{$U$W>y-&cG8oZ`Ql~$&k3s(9!N4;Cy97q~|z1B|7Oq=3xsvn21 z0IV^Rhj%0pGY(v}yhZlw2d^}M%wVPrP=@_S}O#3m$ zI)+gT6Rn<_2wTS)xmx7TxlVm)1qhEHm?ZUF;;fzZXea@U2;L}USg1i3*|h{e&_k0^ z@;*S@c|Tef#1B~zz0b|cwsi45U98+8PyXr5SZYn&~D7B15%2vpZ9BK%B_dq-((rEkn3 z_?*#XoA%n9riqV9T<){dM$RJaZ6B8S(Sl*&-z)?}lIO+lSP; zgVr6z)H83z{Blala--^Bo1d*pFRtUZ`D5={S`6#BKCJxh2|?ZoB41* zr2Ep5ietHfxEKj#_rJky);|Pu{!RQ){Hox$ zn9spEsB)Qq6T;>^$V6W&YkL#{|BPw-*CxgknNe)>E0EQH7sd8QmFWd1D<6<-{BJ_w zdV)_fxoW~R&{IkdU^9ravVJXh>n@l%E7;)(S!PD+;My5|_h9LBrw_SL zdcmE2Kdrb%ZIlI<@a{CRiw0M=$&rbSoP#_fzE(;L12^hHOP1VNK)^8$V*6Y0)y>9* zhulgpw#x@iq@~74st)qP6$K0WuGR=MZ0S3R3#XJOot;a-h6}7@cl>j$%fO%MP}~LfGW)t zA@A1$r=y_mw78(farxj8fWnqA`zfsc2G{aE9Mt|Auu-8-iLV?9*#~lb-$>i8`g82Y z64aj}u{-X$6`yVLC8jHB6|NGynWHd#Nx-M$qWdM| zJrQ)Mt7hi;X|;4A+8f!Rt0hZ|lXk2o9!{#iCN!h!3~sX?Y-7zX4_!L+oI8&Am{%I~ zT7B*7FSVKj$sYBNNJh5_-hWJmXcDG2|H0O{H0FaVo`s=>2zwjTKW@Vt&lAyX_m``- zH7quQK7)!pf~CJgbwB7lh)Pu*k;<>61#>VSV9WQ(MI5-PF%`_;!2M_9T&7(#Xoo<@ z6Xv0BY@mgsnL~E{1lVa2)a5|4R^_APZ+E&nalJyZLGPQIY<_EkKiOo126M{xLm81p zHX+hE?JW?ZE!%4!yUDfTDMEH}=OCz|p@1uaPpMC>-E$zAr?W^t$Nps8(yhoQHCpfJ z9FyAYYDuL4Uz30$26=#XksSJm4a!A$8YdG7Rw0pVyIGzY5nG;=X_CK?3EF`)Feaf{ zxNe8hKt|)ivd^sn4J*eZvP&n@M-|wh)hqQ8jRb?b#;Aw*bIkBTfhHAxUJOSHC%e8| z&YMfD3*jrZV(>N_^vSsl&_E2K#AgNiu3Re1Kz_xxVzhd>mT0F-@e_JHnv*HsxZN)J z4Xj3ZFxkAw#Uiqde!UiSBUs2cbr(8f2?fk7`^l`LY*)Jq7m!hrPjs!Mk3OQ^4fL4r zvXk{bKP2$45C!3{GogZc7e0(IZSS}R(4xN}Obb^CkS4FMWJ)twWW3!%6zwgk!z`HX%8Vkqd z1B&{*9W8r~4SG)%v4(b>Nyy3hIKR*@$YGh9R_&`Ke&1JKBjVq16ixO>#G;Ty^Q&*R zj9`gOTk15_v`95Ai+dNMA2?yqnA?vAWp$?VpMVV^vFhwND_wHc3GSQxs#90E)(m{* z0;qDYTETNP7?;7fvj<`Zb3KezvX$ z<$NmCK57E3e3S(`98D5Hw^}ULr8JGsgc`GUd9|NwWLWt0748b0ZIQ=M-{P#%GPw@b zZ;wpyO}z*Y%vAk+k>4@z(GnU;n!xkQy-D5fkUH59Vr2yzD(13WEe=*Le6;NiP^UJ? z*8_D{nk<4Es*rjh8}jHP)T&hv1AUW^nnAnf>AGsrfBC4npHmONrd8akw@;JZO8uTo zWnE_^1MN!>05yxU&AxBtF*$9EMWDp_tRAwuV#Q$S0u6MXjdE$e_t3e#Z zGiM@r2qUg@6CHd@O=~!7qh#=%b!)>7R=ktu6zFkZKn#qr4rdOVH{RvA&D_cY8#j~x z?9=6FG2Y+1_Sg+Z9@XZc5WfZMfe=&WBFuo(BG-5Vu7}=xWhI; zo%#UXgU^slHpc~jI-z;CSkPOZBX5Hy41}0{#aPaJ@oTAni^#82{hLbUmjHi>a(?OR z-zMyrS^DA-e_7za9n+^1R&YrVg`Yj#;{Zm;WPMpvaT;^%M{+!?COKF-Y)~%s&B`qP-8x^JBp9Nnm@2`k(%JN|QrJM^T+z3(Lj< z1OT~PIN!PcpHm@aK6+|X`234|4}P#^VV<7=)Awv%8E~o)|MuS-Sr~QbyVZ|w14t3E zrs(VEpYZM79l(^*H~_o1aQu^C=29;T7AuQ%^FTF5771<_S;Smi{VkNZCH9zcd$1U@ z|I6yFTDU06Ln#mV*A#7&csV(2@*T64mFg<)n>G2p!DqLQ0~kV5?0-#Feoetqe(EdMXs-dx*>RljZAE)eX>b_+xi zgkaI^@+%f4(<$r4A~L$VKKRuej`TCKq{7Q6F*d}WtOQoF{=d)PdJm9|t#9^5MjrfT zGYVplyqsu`_wk_5tb)bkca_dUo%uG$wHQ|1m#RnqFC7L(= zXaXQ(wE%e7!jX?^r@4Q&8$X&AIc`tUL+cFN_U4`lK1Vw4Tlr>tjcRj|Y9;jD!7tbJ zGD&>(a}=e*Of{;sP4zw4I@_l!6YA2lilof?ChzBwTJQ!gg_Nb038l5)K!NRnPzScG zn9|-aD+s-*$@Pj*KG0=T4S&}6=Ir$jSkRKZ4aunTyf?Efa@DPxOQZ;`ulf>vZ0#rY zOZPD;C%xs~}w5Wd%KIcErrzVI>TE3_`Rm!odW`GNljGW70WF@fzm z%ga@7xY|Gss2@`K`qeAA?t+_?WoX3Je>C>{ALRCc=LjP1Z@t0s%JV<_tfRzbC7IRG zY0U~04`6iadYajqN<)9;#O&Z4i*UEWcRT(i z_0aSFXP?a!8Z58<`J-~HxnO9M15h;7>PuQ*CKA_VIvcBG3Zf$U}3Qyh(#07eDR3Va%=$fYJZQU+1 z-Ws9A#eM4la9$Vru)+_iiJBMCHdT~A5 zH;`8S%y|FIvc|OScYCQ28&#CZ4t9UL1$h9LX{Hjft|zlAy_Xh7lhH4qbd6G>_@;gh zmY|c6fCEq0s@*=b@zdEVgAkS1Fvn#rdoKodNnT^RW+Y_}MhlQP6$Q?|oz}DuSu4j)b8^VAk}n*ux2{kEP+ofOxAOfHxLR z08!7gZN%iUml&};>QljujG&W2a~(#pItNRVh(_|&^C0(}2WY?=0_Jc17QDnYw7^Zg z+Tei>Jih#5RZh#M#rvQgdIzwjmRyNn)pux*9H6?QftI}yEqy;k353;-QvlE=P81H9 zdYOM~tlA8KJ(GOYHasCoTO;J1W+_9)An9IUnF;q91L1al9#61jK?1BgWk z7WH=O)JN`wvi?Fka|}@bj68o9wm&s7$PWdoj4ad{UClAhxGviiRj5S*MnNvN#>Q+b zA6O+{-Q$YuaVeWW{U4w%ZW+{tAm9>EDTMvQcW5W^sNlw?Yf!ponfHX+_sA~6L>=(r z!6;POK3(&G={JpIfUiv524omMF?=%2`W!}K|B3qncs5{(KGq!da+y7#va-vcQc(|3 zOWNp=B*pur)yMlje5;p}~}7TzZE%+nf+Ee$lgc(2X|M!{C~#ZS-b zc_rh{iY!#|C{|OkfP<~P{n|{!yZ_ix-Bl7#lc$aY(yNr_0f=C01R@DDDf4!Q*uKs~ zJzO(S_yeD-0-@+n@>K*}j|VGeTWuHAKxz!^ML^PLz=R_Ww#-?cjW%@4fb_=vI41~O>>3D@E(3CQIPwnmu0?^J#xbckRAqs zBIy{%?2;6x!j20zAeE8z43-n_-#diSoa*VH$GI<#r@{g^xaK!1=D#)hR1Gw};{Co# z$tg4!)*E?v!;y-~1@h4Sadp&Zi!k+P+Y7F9)mg84(dh6e_b#4NnRSf#gLAO>qL~lv zukUMZ&1F;aJV~yl?L^Q1I&0uYB@6~0-`KF@&MNF}ve@q^ia)Vz++YE)20jdi%Pj!t z+ll_!PQz@z&xdHc(4<09r3UtYE*=WQa{oEGh9A99J=x>la&lYae+eVVMG73aDZLa= z_DoGFyX`gE8BIa688+c+^ZOh-nN9+Z_$r^0dW(rqe3f9eMe#zC{JCZhJEGsQ5maa9 z25$Amp$)dB^!(2jHRxh`9Rp9*9oz41dw z-;ev*jB;-kw_@Y>rChvsnXMtyH3Ij!RBw&i6?D?-)3OpnTy6FCsqq&ezN?s>uH`Ry zxg|BtSH1#Nne0AKVSz@F@5wyIf>MIyfH9R=i0W~qLB9DjpT){rdLHfwS>ziBs-yG-|--jXN`igwfPHeL^n8~=DUF@0!r;uGWCiNOG7>Htsr~lC~H&w0KHxm zp?p-KkpZOullvTeJ8!H15c7VGI;=vuA+hcwdl%&~$Qu5yJCMvb$SBvqNi(=W(#|{F zeuUbFn?gSf9h4I|Hn{l@t|CA@ z$WyP*_}mtyReGx!YQCb$22J0s4*{&BkA8EL2c__;x5?$yCTWn*a+3c9{2F!~>C0x( zWFY~wGZi8beZM-&AGQ&ssC7>yzAF5!HD^jpWcqf*0696cMu7ys6pQ(9KlUP#$V*+Azmw~nslOe9R(gn_cWkRz4h zt^Pc-e-dTWF^`|24)%jff)y`Nd7Y}mUxCA~S}XU6i+4?q3PUToH!rAQo11m6Uv)H6 zYBI?4f{Dlb9`0MG%L<0Ws@_4!#O1rE-hPegzi1-35`wEU1V0{9T%CheEeb*DgNx!x z=(e?~?#fU4S~a}(h{dVUu8+_SF4WkYS2q(}^j>exTJ`glckACbxT4>{HHIEj;{$Nt zhv2{dbq*`Q(PotbRFz!0Zusl6ehfXOB>?H^rFL7+K8g%r(lXhVNdw&8CR_sJ6IGUM?5l^l)bujPKX zE&snfT`M@vfrQaMq-PK&?wBp_4bv zeJfY6ItIjR)t%6Pq*)IBgSPr}*48gBR1U#a8ooX4<9(s8TN#|as`U@w;gzqPlkUBo zlxQYh{I1mJ;b{OMi0nfCBj~z`nr$|(A0C(oKv%GQ8ubDEp=f( zsDc%)$`ILewsJn*td6FBuC5W*u;g-Hz0U!Qm=AE1;;D4In@0`KI3;d2^R;OBlQami zW8Y|gCpxE&?OriR(-@A)b0O^;wk50XHD<;0$GDHzxJZRB6<1C5A}YV&xu=c+Nc`E^ zwKMVyABWir2{AR!kUyk~8wgXk;j zff-;|Do3V;|8})_ zCwsImT<7@UNJ#*?qih`Re0$t?q9&ILo*Z|il5=b@3}OR)%SWcab9}1?Dez>;d$OD# z{Qv!Or`TO(+lqVYO7#Z#w{1Bag}r^XW5o&h<~;|Z^SHUUoOaW!-@bWQ`R*$U8id!3Bn*??@;)v~c8~#o7tbD<-Ro5HXagN%= zLDUwTbcl08sv5Loh*%~li*Y;{8E`P=d+!PT=y1LW%yvkYre@kX2Ygmqz$1&>H*l7R zDRBJ_iA`k1u#%WwPc-JO^R)A$3HI|=uHYLgR`pb+9S<5(3J(yr~A&5K{lx;Q|rb6Bap+Y5} zp5E=YHyMQnCn&L*)OtfB3qwoh$8E6ynaYCL#V0!teUq7QN)M7%GIF;SZyI#d7KXxI zsTTn@dH_kUGpd6oc*HFv#|R zIv;NEs~204q4@_8;bfWDx__M>w0&0MeCvMS$AlUm)-Otk_0yYtVos~iJg`V`0#)QA z?FRbJHL?)=1z6R*sCocQ^FB)2APVY<`)0v>mx8!}nOc%B=II%!%gY5Z|2mT68@HdJ z3P4Q|W#zt|`3W8C3BeZLUC9t_Jk*#WlejdSS`WtM^twZ^A=I)!`rTgQk}5bDJsV;m z?Ev7!x9fR4b=PwC>Ga7^!QFCJ!8ftl`+O{E^LbO%IyxoPusNzOth?ALcdp;&S$w$k zPN$_0bJXp^TfW|hzSz**e%^IYla>dxY*i$!a4jAjHt2T9$m+IPf`+NENIVgeUYfBt z6K0bn3+q!3T)OjHuzxs6hWnWNT{bCR;+u5UJlfU5o#7EmdZAeJEsgbcYh%RZl&+V? zDG;97Y+9#RI*M_u19=kl>37+cixb|95O(~47}|dAs>(AOA;!$5D=b;KoNh3#_VyNU2kGgDM%!Oh|Aflj5ZHbK`wD7LC)iFc z55zEYtti}u8h&6C30IUFi}TP!(d&_w`>Gek(fPee1zomSugKW_!Mz9ME)KNnU~y-} zx#~B4a5mA&a^<_n-buU|s8VGk(!7BBTvCbKqe=?ESc_;^ zmuK{m^bjFlZ44#nV@(_yy9!fOAQ)~-tQi{hQ*|t(z8QOgGt2AGU1iQ5dxRfi>sx?- zciPBNxVq&IDt^+D5o>Y>C(#HeW4(T}&i7eQS( zamfXw5l9qZGFlF>0mXmPPBK3ev|^A?F;;wZdHR!xe9_8Gx!_|_o5SdWU_KKLJ~$Mn zk`0aUa8`u0LOl64=x`z6jtxOg12&-d!ez`E^dyj6mrAGK?gYuk&#m_e!B9iTh8W;! zg8HW1c{*d?vE9Q7?jD;LUR#Ln9`)VqSUx$89wH)#4rGc-hbGpLwq=Xf6sDX$6eKb- z=~#Mm&YFI5>_yP7m}k4)n%5Td_vTk-bS*A;)dX58Skb0FPcw^?G9P;{`6~{MwP0=T zS6Ognvwch&E?!}>(%9BuijO-sE`g>BBZT(RaVdS2Hlv4fg8Yh4avNW!co;N8r)YA4b5*>wh1JGm zgHdaPfRQkS2A9kP^9NX+OPKB18PwgD(AflK?&_y&M2Q;`K^IOkaoSu0fa6FRBMUD> z6*QhT?}%o7twH(0(*;K#lmmD^c-v19qFsQRPJvaVF#&XMpb}@hchklo)+CY`24w{H z_qgH%W9{tGN5>Z@b7W-$2Knm2(RPNRsaT>6)G?(}4oJ&lpd_5|{5FolbD^|q_w`ph zopO(%Kti@-^o;_++&~0$K*XF@{U{&~Ke5ZB;Sg!|$cpI@^g@Ujy$}qt@=m@<1Wd<8 z@wL!^FXtYKChv%TMn5eEuGn??vYOa`x~4U2dDz!hZrBIBqIX*StK{(zsOzIK7GmRo z@}@Rm?hZ0a9}G*l1a*#-p^Pk1Hf0UuGKyuB2Ak;)I;aks)^Kw^<@3S!j3$EhmB|It z0q>9TL>p#Yhz2j4?F5FYIHwW8I54Nyk9`H!;cB>YkmryEjbqNhfm9lJ0|}dNKDE=i z{ebm%-tD{X0&d~_*`q=cnq-h*>s>O5T&FF#S<1#-XGczLY0OV%>hIN8X$x%Er3y%| z>b=SljJHYvp%=Bi$d%(_8jL|O@^--cTV3FHFBXh~0(ApoCCBW6oHYoagl_bFiTQtW z+@nyytRKjb`04k{p0Fwu1blYuTmTgK)kl2y!$T-eSDhnaf3os_ORxM^P;+^xxbGim ze!gPHZaC%2^ViB< z+-Vo<-Y1E^kU*6R9xU`K=>NRxi0C1I5C4B>44;b{zkkJDWyhUiw!qfbTB?|C=xS|JTG&*EHsE|q|?mxK)CN~u{uD`@H!9Nj&)T#AP~qL;MRg;9d}en1qLWE%Fz5Yi}MBc z&DwSQ_XPkZmMCa;l}$&wp_ot}eH0s8R6cTGYdjq=u7ljqpy1KlHun1v1DD}eMFR*LpMi)^pkcjt508U# zLbD@{oUwp)hE{UoH;D^xVcsau={MVAA%YxX09oLX8$hL?`$)7=%(HOy(p}ykF7~+# zEieEPgf|nKEm=olHc>(wGQ?A_M%)Ao(GHP=rgqw#R{q*=o=`}YA zRj6_p8hYA|k~$R7=S3*2rUTRv)V-@L-37Yxp?u(&Lu$q)4N`Sd^LRX*7|?H5%>*A*xnU;EQ)7ZrR4EXL zkh9iDmy0VoEqmkkYgJa=n}fjdm^OjJPxlT(qi9f$ndPkF3#eSAA+n;<0Wv86mW!|{ zj4U}K_Q30H8{k|ztG@7(KL(=b8OjHc`1fk~H4ZtscAdn&asb!FL~Y7`*(cw0)Ozo6 z0JfAz%->>+N9Y_JKV@KglnM<+j|Dgsdfzrn--{-N4u)Njgb?sca}gvRjxTLI&UM3H z{r6)7fI2f}7wCm>wjTr_8LsgenB9w7t{9$?cIG4KjfsG|?;^NfQtfSM2>3w=>hIa~ z%4-lB0qpgjb0O!20th|><-k7m(2NJ$6I|tBws}!qjr#nH8IVyaUOz8#^|gSr;HKpQ zT!7^EA1r#gdS?_=0y_ewiJ*jm&Uz6-FGL}N3{#zX=RFJ20`=_5rUFcy(`F^FUH6oJ zyMSTV1~rl70lc@f1t(Zyps!XK09a$Bm%A+h&ke$zn5vzh(TnZyX3xGH=*QQoo`UoOfg;Ly(u|x8kIS6D^t;J1%ZvSomyZU zfPY7CO|k>#NP#h2pst!G<5M(Tx%wPp0Tej9=*uMs(g2iuQnC!7_0e{!Lqeok>FnZ5 zEzD#I)lbLxCv5uwZ%SY8f2hoP2>A{gSSk8pkqDAYBS2(ib~+cdJY~L{O#+Z)CCOEL zIRP}{Ph?p3+?s;w`hhJ(cUXC(WWi5+OzI5d(R#NC2EqDFUp2Md@eqY2S07de1G98UJr^v}{T_sJGkZLk zrM)#WIE5gQ{12&tnVMiPN^HBA_oBr1v`{lYflVI`so+OiZ4|%hn7;wm=V<_C;kJ&L zrX&JWMVbwy^%b%DK(ek5B;z`83QW)gaiv9>R}K%xp*yD9F<6M|yIU1Gw_Qu0fUs$3 zkjdmeGSuFn5@u#>d?kNi86@_7$yW!8#k;4wQN3Uw1=8W#LY8ZwBMR`V5p7JA6B%HT z`W$ZkN8JDbBKJ1Ra}YqNO$}uJ9!~)(Hbbz-v_krQ#7jK09&Zk{=gdHjLXbTi*XG5K zD&$B3axr}Oh>>z~4>+QpX{dFG^!VT;1k8>AocoFjYrU)QaixK4D@*@}iNCmYX9ni$ zkxT9~+Xx9pd=5mC6dS6(K!GeZh-C9y`b3zx`%z7}q z+Rzd-uMpn+mp9{E_w-o_h7TGsSqJqe%|O`~XW0edC!JMp_t;{cCw{*hB)DDg&fs#- z3`7Xm^VSWQ|7NS1jArp+2mUaWvuMjGzDo zPodwA#;geA47TQP24HJ4E|Bx5o1j{Zrr*kb>g@a5I#$U*7;l<0lB2PF2z3R<$ybp% zL(4h<6f$w>;haerl>yMs*t%SQjy|&iO0UOs&vVXsC;@DzS!v~G&a7v@mitW!t$v;E zH+1lqFn<@F{?gs=;$pd9rs|u4|9{gmfAEb_$(LO=`MHW*8s$z;%;!fFqg^e|){$U2 zja$idFLJ)UlnTkH&N8Pz*QtWAn#SBf`YCeBOa^|(^p(kS1&J-x#clFGPM?VU5D)fJ zxiI0yWB2}kEcuqE_L>~ZiovDv5o3>rpC~Mhk-z~G2ys`wY9h?UKYJh6^cYX=n zDXMazGP>x*rhf{|e|Ure*rL>#hgv`U@{2qLUj!*8erb((K;->R#Qw2EDIhnkKXT|_ ziK^x8s0QQ)A{T;`IrlE>ZUIqEowWqV=iUOHZK|0}Pel&j2{L#4!C#|mbUD4znsv~` zH)Hy^(?2iy{SR`6;II3`A8~x{uY>((t9~8q7u}NkrLkX8iC-G~#a8{z@AbnIEd>VnbOF-E!7iFWq7D*dG9L4Tm>!jm@ zUhzss9nspu`TD;GDgh@9kdN^Qx8r!s<;V8$qt2lECM^kJDX*sh{fd!Y^fDZ*S3B@5 zOgSA}Ii7-zfq3K1#%9~}DWgJP)%%BNrb>J3cHOfJEBJjivHT!Wun+~6bsr4hx^-*! zpqWcaG6X$3hP3Iv_60;H@!FWjXEtpXAso{CZl}L`_A2cy#=DMLJh5|swHDhSG;Q^<3S!mOeY?cEsB%Y6QbsC zI-2g^*P9pwY8Yd?Pn@)9{tv_$Q)Q*SR&ouffv^1&Dtm{L5YAeIp&9@`mxj)Mbx*r2 zy{uLO+g>Y%$2#LoU;qIv@~K|^k1ziIhg`%e#Vg8|jG_d>g$bb}H7tli+X)nyl7v*3 zro&|cHOmF4a<*qWYK=OmnyrUWQSb7Y?Ue)9`RbsWW4$uKh){;R)x*nX=lGNl36DT5 z5GpaaI~2f*J!hBWz0j&7Ye71^m6Ko$^zZq*z*$lm6gWeq5GoD)0Rt;7G>-~``~nrC zvxtE>AxMW)0lhry}Bi3s>2A`7m;F6 z0)_%u3%Fu}0o8WGIX!jyu=|@WMg>&HvB99JBD^~(G<$UvgBl}WUU_0E)*~~9Q&j`GaxSyL!C%;5cC3c84|oAZGvU{)IdU z{0EiRc32*JjfJOo51K}aJ_-Uv-9>GnFN6q&1+9n4)QF-3R04ob99S!X{=ZWK3>5f$ ztLKT9nZ5@|VgeOnZ;e1+6-kS&;pp#kD~mE;zK8stM7L!|oNOo%UD z>2q#%_n;GPs+!+5-r?S=CF#|Mm{WkU`c^+^ zCK9JM0JQ)>6hJmWQPT2bnDF3nXea<+gds{nV-Fa1&+aLjx}!Mv_2h0^wOVMivK!nFg3jM-mek+f(a`D(AB1eJfcc*H~z-0FcKGBScmi+9&~~}HyWnRLC%siuL!0pKx8@5P%^h~K8b}r zb**Y*mHWK~Flm6Amz$e=Wss>+@1;rzW+MGT=DPffkRW*zNER-HxPv4xQQ-+JYLYI2 zb!zPwgGLS%rBrkY@}L3GAJu!;qp#kJZl}R8@elWfSjd*D(F^=GZE%=Hfn_I1Z1%V5 zu?vxm!WrPN-CGWOy&BVC)4h9yn$;#jsyi)TVri79ojtxhKLv)TH@cGw2{BL#k)(F1 zM%c%RVu7+WZeq!*0BR-`Xo!q)<Nh>s+1oGc5k482OFXDO~4RDX_ zKANLrwNYdu)MVs8RB z?uGgukW$avKTZuq|_aG+2}$a9dvtwLdBOdwd1(>`3v~aQ#|P0 zq?h+#{l)LO8&h4RN8#5l3bg3}ZaMD?s2!%IT=@w~3%Q)mCqeUUEv|^9Pkug$Z#+(T z<;r~P>RGR^oT_}FXAvk+(UqM;g6gwR_z%2(l5K?cuxzYmE_^ho!S1ek<;tFSmF>RJ zHwujsxjA~#e?k#pFjje*)D`x%Y_mDGF^d&Xq{rVJJ&BsRrEB3^y%WG<^&E_MIDjtA_xiAYAz(``72}NMS&8hWIu8=tG;EY8V6EzI_mXU6V)1nB z&C_ZV=LCxg0lSQgC(<({vTOmx=A@>k*5IhNguw18S1zd0rPgRv;;1=>2S2019%IWi zuB<Q5|aoh#Y9aGBFiCl zjU8Q)P4xqzY)?$?nY7@j--`j0pcxe+2unuEIpA^OPM7P>Xs;0kq3Z~gr;n|xMC?F; zt}Sfw1wr3^jeG$xS@A;&)1h_+;@1rvp72@~iMOjSwuIx7ZyO+#l6x_aeZbhHIebji z-d*}olTkf2kWPRv4{`OYL3bWhE2yR-D}G-F&qT#Fk%n{BUE!q0@hu+FG* zCK~5?#iEIFDOihkyFKB#crPMStnegMXz^h3(}IGkS(DmR-p4qVTGmkl%}G+h9kY!& z^K@IqmFX3_k7n-Z2RuC@x}i+meI0As<&^w=e?!5k?Im^<3vXmybRLWlwYB?4R1wND zV^KqXfP3*nzYpTCDTv~cfprWq;F9gh+^Q5FRa#FywUm{-`J(piWbaycgm!Diu(+)K zHs|YSpm|u%j%`V!I^Fc736o=8lrpN#5PQnAvjW z$_B&`DuzS|YLG`eXk~)=GZx|m%*qW%A7`PaJv3+@V%S=$yWkp1O0wcbhyEmQE5TkF zHUrJ3c=snVYZIfu+o4?&eF#aXEF7Gepu^YQM+U|IM9Jy7Du3VFWo>YjY8xSH(lxtK ziWFyENA+`X9WKTA2L|N_Xgh0+DK)2Mc;R2XQ-D!;w4@aJ12ml|21@u|SqhGLLgj?#^`*1Wny% z9;4*8OQG6Scj>B@1ys0XA0#D8Ou?=`+M}C^?4U;brA1NowNfWqWasX;cEaaAxb$aO z&ZXZ>`D^lB3gV*zvSV^lO1y1o3w{P2Kh zkNM_7E1~vAxedxtCH*59-X59?<|d|0N}wc}8sHOu#t`Kg-{&vr*+j z<5<+Km^B-XX~Ntmq3-0JwMkN4Z$ufI}at8O-oEZTQwgr2uU zcgEK&=yftyZt#YHA?NawMS__1S4ImoA!~EtV4K zCBYc%9z3hZrD8fbW%b!@y^Sn~?iZmsJyE+X-<^K>S&Y^gL^XhG4Uo5Q+9jgT`{_J( zIf0kA7S0v_{Tjw=i+hGgRFKM_sOBg^_;K z9JNQW*9CNvxt%8Sx=yi|UYtMctGb(+ie60BajtaMe!AMsBa?UjAZ$c+CMee9-Jz6g zXEZu|o^4(uq;H})InoSqForf;9_eSlqjeTldbKchuQ(jX_0+4Row!z2y3M#O0gYjg zfP7K)mNSJNpnNTAA`Y&ZtI~D;rgJylp~j$a!r9f|3?{pBT=n-3(>9Z1%3WG^HXt~p zc28)C@y(w?c%{=9dk$i~^2i`|>A4`!$|5di;iRs6w!YVITxp^-*KW`=qqE}w`bgN&fg(qFOuFpaj7jmFS+*5q!dy?tLomuY8_VegcVCd z46#L~qXi-CgV3&Vg-SVlOwzK=sB#^Yj}Oz!4lz4`^T2gGdz9ISs~J`y*24^|B2H^5 zHF*Y|H=sW5+AzASZQUE+^hWKZIt%3_V+(D0fwWuk4yZ^>)h8eFxs2&77c3MIcyqyP zpSG5t#Yrc6wrbr~j2QjRl2%G6WYav>H+k(fbfL0gjUPf&q2|qs6c{P02wpcF^nh`q zV(EzNO$)n6LzA}1@!hhtsLx{P)+6xrVnx~Zp}R#Lwr!cmFe7R8V>_Z9E!jJ8SKjzr z{XsS;v6D#lD8(S)MJY!r!nc zDIK8vWH6u~3?-bOf>cdixFwRosS&XU$ zcQ4gO=I{r1HFh#`FjGx+a^?+hcrOd!2%Fi8u0^ zfoW>uVzO3#(?VjD(+*iF%%&6Y-`raAOp1_d>BvPz@rVhdiRr-h>GQFs+YI9;gHjYk z1#TS4KS%2;c-fVAgyiAXwkTe`XggHRYBlt-be`FRT#TGCtRP-KVa^a=H{r)3cJub} zAs1e`xX%Q|jI2xb^Hl3(PirjIxLN$Wt99jMz)TvpW2B+Q|MzR}AEymGO5BW#Xm0%tP9^Pqjk4rSOdS zq<-;3?ys_6O~q z?{q{R)z-e#KDsBJWV~BEI(GjmOjtM&0ngU^t@={D*s$6eLlr4TmT*p?Ao(TUT?Ean zLHClLc;~grJ~`6A;#|?xhpGu}bn>5W%0AD>Vj|W`&EZ;3?Y!qap55ZU+Zc{|H${zzi*lJ!ia)tGdGpoQ6s(JB>q|3)_VM;8gqc>PGX33b)QRU( zYzeZS1J~~sV-;oL>sZb&Ed}+d9lEiHbOt`-DczbB-P+70E zyXbQTfxHSQX(>&GRm6-$e2l~#l?~7rq>3{$N7Z9*^6}C#?8H?YmGMwuV66K1hLz+B zkHjpt6~tGPeM%FY4pCGK^NXY0vziCSGqY0s_%nwhgADxT3NNQ5n|myi~O3kD?M zw9OpFXfftRlL|_-22bx;M;~%Un!>}@8u$~ex2l4?V>*r5{_dS(#Du?*@{Sz)j4WLG z4KX~x=516GtlzN~$Vr}Z7DU6v!o!p%#Vnm`RS&C5!{vK^2V@?bu2>eA<-C>>YB&F* zyVX=mpr51de^5Q&`KE^$Ae1eBHowmlQH**x9tOg_ zw+70ywKMem^E}!^le#-|#BUn>aUl8QBD$4+Ok1efeR)ndxp=AD5S zJI@Kri%_)U7iXK(P}iKopYcX`Mgk6Fm`{yj|FP48nx95(M5U(94Ge_K<71n~9MY3% zDbcsG+1?Uf=Z%W;ONmj;CZ+RjL1pzBW`qDPZt^DMwuo1a^;R@?wzo_-WurTy#)gFv zV{Pzvzn94l!Seh02%nd4FMsS!#n_6LqDqzsUT=GjG?%oSig$S<9QL-4V1KhOidVkh z`phecEaW3Or?YlUnTS6c&XB#2zuf4OWj`W zcP~3WJmuEu1NJe9aw$}&{`kfm7mehh#k77w3_mw%*fSSnlM0rQOm+~NFDjC#ZsfT^ zE?y)Z%&AqB0Vi@fI7q5xk&RZum*~wKB+w`QSWS=ebt}k68sp}SOpXNT7fHq0jV;Zo z5@=EHNrZ4Ps|;EQ7-+mI}s0{MjW!-BU35cD;mM&V%lTXrLV1{ z@bY%{YfbIK&Zr^FZD~7FqP(@yBdXZgaU3@RLE4fUUz5}wV@;LoD&&vx#=j*QbeE#8 zeAX6Bs%XYI^rr}a!XPG&+NaHj%2ST<`<*ST`0VeDn16t%D`YjM7*vo0I&J+s_h?#? zxPzEOqD9i6PCwjPhmxdLRd7;00pXXPQgG~O!eKVJ>VAAT45gvU=Gz(Yt!KuEk_x7ViovSiL1_HYR-}ei=tdy1qFGq;ogF& zggYhKDW~c|u{BX|g7_?E|1mRy1=$reXc$Ez{!mshac@4O(j1BCz8UUQ-a3C3q3b9$ z+TK*FP@dSj!G?THYlv>FvmVxXw17IPb}7yO{MJ%7i$Xlu1GAURe2J8L@xjxnJNp6S zDs|dpeJ2i#qj3EF{-g=p!m71&ZpUH!{zY{Xgo>1RA8!auB1-rLl95KuO!&r1m0P{? z)a>X@WocP0u46@J&xOw1d*vs3M)Z&t5C+F}qvdZMWm6=YAZV_re$=u>f35^?*~}0} z$0|1mp?5nndJ{b;8fGAoBDzn9l!Gn@Af2%X0Jari`KyEdqct(yXY%_KmOs5@8a=|YHdak%k@wP74X zxiS>;qNxTeOCafZikPi;@14!Sb?~(BIQp(MZ62!{->K9e9yI&R8^-Tj zweqb`QTMbHEO_#~l)m=aeX?wD56BldQ`m-Sy`zg*l1<(3ur$0QGTh7F^!Mr#yFi~K~h<10G$zvaCk6YX#sKJ;=dIfkO&Tv3@O)b8#3aGduxEKo#d z?PVI_knXjrKEX5hv%PVO_By;GdVaXXg3sEIVjR867W7EH{zbyexZ~Msynm)n)tBnR z8|&sx6MPZ_J~5tYEIykrRZUa6nL>>ACMd-~0KRZG+~V@^mK@)cDp_nJjWfOM$5_o&v`#o%<@{K&OF72yjVJ}hh9=U6c_uk z`>t!>(u9aNU`z_>JXyB;B<$jyVD1kFGf(j@=$5Dt+A0ZJ{TlcjU_+BNp^bv(H@laiH9heWe1^-db^#cJI~($fx#3+c}D5pky$S@RwTNSB#Z2^ z{j8sHY|t-jym_an-nCquLE@yIdK_Gb<$=ZZx{oGDIr=5%NR73!pI=BN5SQxbMq-hc zlKL#TN<~S^*p1;NXFETxrzQOqkRcE!k;lX?$+Y4wTjFZ&Ic|J1>U>>kZsh<%;qs!h z-Fzb7M4+1Fg9xJ5Tu_YFTSOYppbKW=(6uxh^@o2~y=7-iRnu&gM6*&S!vbwm4MmmN zn%U3vQFR8TZ#La@WoomEHmzFwhIxGiyvY5IH^P*a`t z(Up>UFzL)bZjCc@oJk(N44+$y2oIPOvxtu+rbs<=DPT`4G*b%OuZA5;V_!T(d*&BU zQGCB{mdMjNi&7&|U~qN;R>ywUyx9?bh!wb90>nSOy>r8*WSl){*Xph3UJS0>!Q3hN z?ylc@z0MKpRMdIo>Kej0&ymUU^T+kHT6Sz~ik(VPtHwo^MupZbIp$uph}qn9!|Hk! zOJnX~%wl7`y~vdahqKmcHj}eQ?%t2>ic;UMvE-Do`p^7;^iL^Cr{GcoMj1XG_Fz&y z+mxql(@uqbiKLG5Pg@;UBOEeE4e`T1mCV#xH$+ob%8uu0lkMYkMc#^0(@WKt(1N?W*Oc*52dV z$(`*50!VPz?5HsJ;Zt0xaP|W}W_FAgBDgP$cCA+$f?Lzj&}Dkg2ya%+i<_qiL|(_S zgfB0FTyLwM_HKWK{7%-0d|Vg(|(s zUVk>15N${H$sXDLt}B2`UtRtGxT6Llw3O?Mn^TfC|6(01*SCDv|2qlR_^Z<3O- zXc=oNT@Oc%)s}QzbL*$Ln{&OY-%_wExDb$uTl%XwZrVM?1_NeT-REB()yqo2jL!u> zRXldz;R5e(;i+0Zi_9?%;woMPWyKB*HcYe*0##xn8mrH(*VF4BT~H_2POF~ioG;&$ zYiVJ1lyP*1Yk7JGCH3os4_dtT`F zyfKmFzoPK&{)jiA3-q4m4v|>p>`qqbNN?~8NF#E`Emj;z%^E$=F9K96unjmi7?)JC zfuq(DVYa7p$k>hZ**sd05;dppNG2GEfyp=X&(7gZZUyc9)6wI6;W&0lkR6MG1KwK)vmY}nNqb_>=Qy6aG;gcKgeVbvEY`bt3%*^AFoe;{lK!vu&NV8j{9og# zm7`Wx&QxRnyj1Eqzw!J&s)}Fug!1o&5!T%d7cOK=EZ> z85s3dxsN1+gAz?=86cxOfzVx%O?J>}NKvNOQ-5#OPQ~DU8~|1cef#m4a??wPkuj7( z+zQ-#@+_xOZC=fNxtu_LkfwM_70FM(N=_Cy9vtLO-C%_m0W8B(X)96JmGGg76RB9Z~Z8hH*L&1ECWR5z9Pj>Y3PZTm|5G;cr*RM%xm%G&AAL?Q17QN+R)eg zAJ6tht}XLrK9rPnc?C)|d&q*!1%$Db2^DdD08bJQm!HJ#p(-e0=*?MT4;$h=vVXNX zN=U*q(p2T=*nvTzkg^?^u)S?-f^&Xg-XlXCob_yJx14PBG4HXBx@B9V8!M;Xk`vFq_g-3AM# ztqJ9W^xvYZzOg=yfI=q8L-BnylZ;x@K(L-Z!%cysBcM_TuzkDgcCcy!W?jCMyWB2H zIcR#cFd)%Z=!Jd*U406bIu_rwLXVrSCRp8_kYR6w0{H1F(cx;2xbS3ePcO|WKa8vo}BjL-8vz6AM z=@FV8Uwz{krfvVE=m7RRzU;ky)LkH&5>PZc^8o_lz-b!p%9&_8k6Eh3JZikUR|i(- zWAqsjO}33LjLFh6&>$dAWF+A$vBx z*8t=P0@ z(?kcyUqryyoiD*{MMMj_L$AP%mSg)X_4Jc_K!zdvggrQMX7!vu6K(%E1lr>Ww@bb* zDLQP6^zMC*Y^?N&F;VOwvpyQY^`87ruW9RatmdXU4J9>B!)YzTI08))6u^lsu|7E3 z*S*s2y4pn*$3u$=Yb#j1c)1EtyyJc@sE~r(_S#1@l&y*10Ifc)WIeI5pE}RM{?C1C zZek@CGG{9WBMZCYW#i7(B!s8*b5fr(gTl{$i-?AOlLGuLZ^lQJNFD~d#Ajxt=)m<# zQt?KYsz^J?kr#F(V!LXq@jz;IF+Y$L8Vx&}vf9mdWLo0@l}GS8G#cqIoNlmma0uW? zQMG)oLKDj=t@7^GZAvX3QpqnnHI)d1=8naX#)E8S!oZTyX!7MVoeJ{6YXi7!kAWN+ zxS8z#TvRpw6F(j@lfvuJC2t9DoJ_RO_4E(uZJ>+|nxGkS)^q0$I(XqIFXFGGy;KfY z98e`|Kt-IzC!uzqf}(D&=ry~o@VsNhuV?M=%V_UGnS6T!8!aLS-)GYp1=+SpRxhX_Su7qt8<$s#A5~F zvkX^Clt`izLRjovpsH7>7dyz(uz-{FV$-E`w>Ze9TAp&HAWEsZJPqKUj}01HhF!bk zd7s;l1{{vpEI4o?EjKpovjk@e!@}d1&1)qRfB? zDPrT>F}}^nNY$EJjdu^jW<48;s)DK}y4mH>`>SxMAGjqmmqqXiu4BsmPBB!ogul8S zu+Hev^}j!*uH_7^;cggC3CMZZR5hctg=N)K)1u0ycTF``4rSaOr-N+@B4uPPp3ux`0VJ+rUz#Zb@h1D3 zHO-H;0e*G(eDMP3EBz=>coOXToRj>_3qp*?wKc8bn28!%!d#c?=x!Y05wj&J0uP5{ zZ}mMSYzk&UkvU25IC5lpg}oB4RlteGuM)+7@Gm73+B5%n4D1|F?gIkiA#$W7WuIMPR5hU11Yi9-UN27Gchou z`?DqORbS5{lRU^eAzZI7#Ayb2-bT79BQ@xI3L6>adrxNdr-HhsvQS6t=x|@mp>xde z_?^gc7N;*Qe)gmd87Nor=Du;hj+Ereji+(HhHQN zQ#3gMNly2xI!z!MvR85fq4BT_dC4d8QSMGm*wLC_gh(PS5+LwXpfaN1BmOjBNvK7J2sfk?hF0Wt zecf^^J_GRsNr~O(Xx}>fYZw4oR9vs)S0L&zNzTqK>F{p3Z3kq`r{BM05S#q0wZ7-K z9}66;u}&|**h8mafIf=Gh-!Kw>Z!^Ok?EX0>D?%lH+TY$&U%3C;2rn#7ZmvDrjGa; zor&*|s%Ux#YOrTo0HjSSsO|?hbb#8#wcYXL2c3|_yfi@MBaL8TQOW4>hh!`J+g9EU z_Vmhkf~)6TKC0EH=T$0McXMO|lRK8kfATLaZPEs%nyP$GDr||FSx%{jLRyMJc{c}9 zk}6w~!yxsIY1PR;;_G~=3L%OCb;TBY%Mz&=%)%YX!*ff%pcaHur^^L~tv;8Df5F&u ztj`&{ogWiHe#h)(4@qOkejsO=&(dW!REf!bfV8Wfz~y1|qiXn41W%fp*zc3H4UO6R z9G%?)S#q+-5~j$4$mKoClx7=h{?b(-5q!FiHQE=wW=!CAec~WPUjV4`)19x)K2DCz z51RlQ6URN^HWPIEnSeIGY!K_V^ywzW1S8vfWZJ{r<9C4&o250KMx!G*iwEv(`7mI8 zRB-vj(j{c!ZZ*6E6wG?Vn_E7#^eGK&0P6swQAcJ`mSG_-sr3v+vG0e0%aQ)bF_gSK zD5VjW5jLT9_odI2k;>6{qCi12TIfCKbowm}`c;6_YtWv7jaNvf!E>5HUNO)DOeE@o z9L!+>->{qMZUB9Sb-7=Z>=sp>PcVQ&1u<%v?MAAvBPZ$s>(*I+U-(G>~) zTg|0n<+4VkE@Pq(*2VBOF}Y)=iFutDBw!_wMqN*UEp;Yn^Yq?F67<%85+)ZVQ@zHl zW`Y}8c`tRwIaKpR5gWTb+a2bPCor*PKUj(ZYAr=<;xz3Wiiu!=@3UqqOujVMxRWZ??nB|JF zH1+@9%tLPcCrYFQWL_Xs{VVaY^O%tRXC%M8w?l9$Z_0 zPG4s1{Co4jE9xy`iZy2XK_EoWnxZ_!e+=cU$e%}5A`lU&i;bb^N3eq_j# z_t~R-KjU~%`VIA$ArvBD;5&=lxj0z-%t$Jkn)Wi>BuP@Y)i}!z{^j#Lv_xf+p$mHN7)>`y=|{D!eJh{gQJf&#BLvS`}FEq&tbqDvRL2YRuv08vamPW^TT{) zSmwKX^mV+VD>M-0f4Q0~}_@{a@*Za6R zm)_=P1)GIkwF{oRP@0@T)`87#1F(#L2X7MPNVB2Ffo(^@zcen7Ul-c*^xP%VHG9>7 zJ$rVW9i#HCeC;^@xsK27Fm#lyg^vIBBUo1;joP1VX0O2A^__mZK0AmXH_)32MFXwO^CT$x=Z6JmCka~95y^mk;Rp0>uZ1w%lFAcb+_$AV^W4=&uJryY z4;`g{ZU1D0`kZ0LnbkEm03X)vX=lzJ%=UR=pe-c=X4~|edRvI=h>hz9+I+dB>_*8h z^6#ysx!I9VP+vf9;vMswj)hJm5u`g8{>=T-hgk4+JpkGXeaiqt!gGs4p!DLu&-)+E zCMVWsZa4ltOm-x6rmtUWIKwSizhvth0SS)V}iy8O@GD_AW?LI!H~ zf7%KDZN5m*_I)?8^&j7QwIe~lZfPj@AKffgZ@-2BH_bm?>;In*r_-R9Q@;@WjJytb OdH?8pyy|FJ)_(!>xrb5! literal 0 HcmV?d00001 diff --git a/docs/solutions/pose.md b/docs/solutions/pose.md index 96e10c81e..c2c962937 100644 --- a/docs/solutions/pose.md +++ b/docs/solutions/pose.md @@ -79,19 +79,32 @@ to visualize its associated subgraphs, please see ## Pose Estimation Quality To evaluate the quality of our [models](./models.md#pose) against other -well-performing publicly available solutions, we use a validation dataset, -consisting of 1k images with diverse Yoga, HIIT, and Dance postures. Each image +well-performing publicly available solutions, we use three different validation +datasets, representing different verticals: Yoga, Dance and HIIT. Each image contains only a single person located 2-4 meters from the camera. To be consistent with other solutions, we perform evaluation only for 17 keypoints from [COCO topology](https://cocodataset.org/#keypoints-2020). -Method | [mAP](https://cocodataset.org/#keypoints-eval) | [PCK@0.2](https://github.com/cbsudux/Human-Pose-Estimation-101) | [FPS](https://en.wikipedia.org/wiki/Frame_rate), Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | [FPS](https://en.wikipedia.org/wiki/Frame_rate), MacBook Pro (15-inch, 2017) ------------------------------------------------------------------------------------------------------ | ---------------------------------------------: | --------------------------------------------------------------: | ------------------------------------------------------------------------------------------------------------------------------: | ---------------------------------------------------------------------------: -BlazePose.Lite | 49.1 | 91.7 | 49 | 40 -BlazePose.Full | 64.5 | 95.8 | 40 | 37 -BlazePose.Heavy | 70.9 | 97.0 | 19 | 26 -[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose) | 57.6 | 93.1 | N/A | N/A -[Apple Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 37.0 | 85.3 | N/A | N/A +Method | Yoga
[`mAP`] | Yoga
[`PCK@0.2`] | Dance
[`mAP`] | Dance
[`PCK@0.2`] | HIIT
[`mAP`] | HIIT
[`PCK@0.2`] +----------------------------------------------------------------------------------------------------- | -----------------: | ---------------------: | ------------------: | ----------------------: | -----------------: | ---------------------: +BlazePose.Heavy | 68.1 | **96.4** | 73.0 | **97.2** | 74.0 | **97.5** +BlazePose.Full | 62.6 | **95.5** | 67.4 | **96.3** | 68.0 | **95.7** +BlazePose.Lite | 45.0 | **90.2** | 53.6 | **92.5** | 53.8 | **93.5** +[AlphaPose.ResNet50](https://github.com/MVIG-SJTU/AlphaPose) | 63.4 | **96.0** | 57.8 | **95.5** | 63.4 | **96.0** +[Apple.Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8 | **82.7** | 36.4 | **91.4** | 44.5 | **88.6** + +![pose_tracking_pck_chart.png](../images/mobile/pose_tracking_pck_chart.png) | +:--------------------------------------------------------------------------: | +*Fig 2. Quality evaluation in [`PCK@0.2`].* | + +We designed our models specifically for live perception use cases, so all of +them work in real-time on the majority of modern devices. + +Method | Latency
Pixel 3 [TFLite GPU](https://www.tensorflow.org/lite/performance/gpu_advanced) | Latency
MacBook Pro (15-inch 2017) +--------------- | -------------------------------------------------------------------------------------------: | ---------------------------------------: +BlazePose.Heavy | 53 ms | 38 ms +BlazePose.Full | 25 ms | 27 ms +BlazePose.Lite | 20 ms | 25 ms ## Models @@ -109,7 +122,7 @@ hip midpoints. ![pose_tracking_detector_vitruvian_man.png](../images/mobile/pose_tracking_detector_vitruvian_man.png) | :----------------------------------------------------------------------------------------------------: | -*Fig 2. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* | +*Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* | ### Pose Landmark Model (BlazePose GHUM 3D) @@ -124,7 +137,7 @@ this [paper](https://arxiv.org/abs/2006.10204) and ![pose_tracking_full_body_landmarks.png](../images/mobile/pose_tracking_full_body_landmarks.png) | :----------------------------------------------------------------------------------------------: | -*Fig 3. 33 pose landmarks.* | +*Fig 4. 33 pose landmarks.* | ## Solution APIs @@ -384,3 +397,6 @@ on how to build MediaPipe examples. * [Models and model cards](./models.md#pose) * [Web demo](https://code.mediapipe.dev/codepen/pose) * [Python Colab](https://mediapipe.page.link/pose_py_colab) + +[`mAP`]: https://cocodataset.org/#keypoints-eval +[`PCK@0.2`]: https\://github.com/cbsudux/Human-Pose-Estimation-101 diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 425c349dc..0c9dbcd99 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -233,6 +233,22 @@ cc_test( ], ) +cc_library( + name = "concatenate_vector_calculator_hdr", + hdrs = ["concatenate_vector_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + ":concatenate_vector_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + cc_library( name = "concatenate_vector_calculator", srcs = ["concatenate_vector_calculator.cc"], diff --git a/mediapipe/calculators/core/default_side_packet_calculator.cc b/mediapipe/calculators/core/default_side_packet_calculator.cc index 6485d9bff..145d06389 100644 --- a/mediapipe/calculators/core/default_side_packet_calculator.cc +++ b/mediapipe/calculators/core/default_side_packet_calculator.cc @@ -71,7 +71,8 @@ absl::Status DefaultSidePacketCalculator::GetContract(CalculatorContract* cc) { if (cc->InputSidePackets().HasTag(kOptionalValueTag)) { cc->InputSidePackets() .Tag(kOptionalValueTag) - .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag)); + .SetSameAs(&cc->InputSidePackets().Tag(kDefaultValueTag)) + .Optional(); } RET_CHECK(cc->OutputSidePackets().HasTag(kValueTag)); diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index e94fb7ec7..c92b9a2ab 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -410,7 +410,9 @@ cc_library( srcs = ["image_properties_calculator.cc"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework/api2:node", "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", diff --git a/mediapipe/calculators/image/image_properties_calculator.cc b/mediapipe/calculators/image/image_properties_calculator.cc index 5fbd64012..59011804e 100644 --- a/mediapipe/calculators/image/image_properties_calculator.cc +++ b/mediapipe/calculators/image/image_properties_calculator.cc @@ -12,25 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image_frame.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gpu_buffer.h" #endif // !MEDIAPIPE_DISABLE_GPU -namespace { -constexpr char kImageFrameTag[] = "IMAGE"; -constexpr char kGpuBufferTag[] = "IMAGE_GPU"; -} // namespace - namespace mediapipe { +namespace api2 { + +#if MEDIAPIPE_DISABLE_GPU +// Just a placeholder to not have to depend on mediapipe::GpuBuffer. +using GpuBuffer = AnyType; +#else +using GpuBuffer = mediapipe::GpuBuffer; +#endif // MEDIAPIPE_DISABLE_GPU // Extracts image properties from the input image and outputs the properties. // Currently only supports image size. // Input: // One of the following: -// IMAGE: An ImageFrame +// IMAGE: An Image or ImageFrame (for backward compatibility with existing +// graphs that use IMAGE for ImageFrame input) +// IMAGE_CPU: An ImageFrame // IMAGE_GPU: A GpuBuffer // // Output: @@ -42,59 +49,64 @@ namespace mediapipe { // input_stream: "IMAGE:image" // output_stream: "SIZE:size" // } -class ImagePropertiesCalculator : public CalculatorBase { +class ImagePropertiesCalculator : public Node { public: - static absl::Status GetContract(CalculatorContract* cc) { - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^ - cc->Inputs().HasTag(kGpuBufferTag)); - if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - } -#if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag)) { - cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>(); - } -#endif // !MEDIAPIPE_DISABLE_GPU + static constexpr Input< + OneOf>::Optional kIn{"IMAGE"}; + // IMAGE_CPU, dedicated to ImageFrame input, is only needed in some top-level + // graphs for the Python Solution APIs to figure out the type of input stream + // without running into ambiguities from IMAGE. + // TODO: Remove IMAGE_CPU once Python Solution APIs adopt Image. + static constexpr Input::Optional kInCpu{"IMAGE_CPU"}; + static constexpr Input::Optional kInGpu{"IMAGE_GPU"}; + static constexpr Output> kOut{"SIZE"}; - if (cc->Outputs().HasTag("SIZE")) { - cc->Outputs().Tag("SIZE").Set>(); - } + MEDIAPIPE_NODE_CONTRACT(kIn, kInCpu, kInGpu, kOut); - return absl::OkStatus(); - } + static absl::Status UpdateContract(CalculatorContract* cc) { + RET_CHECK_EQ(kIn(cc).IsConnected() + kInCpu(cc).IsConnected() + + kInGpu(cc).IsConnected(), + 1) + << "One and only one of IMAGE, IMAGE_CPU and IMAGE_GPU input is " + "expected."; - absl::Status Open(CalculatorContext* cc) override { - cc->SetOffset(TimestampDiff(0)); return absl::OkStatus(); } absl::Status Process(CalculatorContext* cc) override { - int width; - int height; + std::pair size; - if (cc->Inputs().HasTag(kImageFrameTag) && - !cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { - const auto& image = cc->Inputs().Tag(kImageFrameTag).Get(); - width = image.Width(); - height = image.Height(); + if (kIn(cc).IsConnected()) { + kIn(cc).Visit( + [&size](const mediapipe::Image& value) { + size.first = value.width(); + size.second = value.height(); + }, + [&size](const mediapipe::ImageFrame& value) { + size.first = value.Width(); + size.second = value.Height(); + }); + } + if (kInCpu(cc).IsConnected()) { + const auto& image = *kInCpu(cc); + size.first = image.Width(); + size.second = image.Height(); } #if !MEDIAPIPE_DISABLE_GPU - if (cc->Inputs().HasTag(kGpuBufferTag) && - !cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) { - const auto& image = - cc->Inputs().Tag(kGpuBufferTag).Get(); - width = image.width(); - height = image.height(); + if (kInGpu(cc).IsConnected()) { + const auto& image = *kInGpu(cc); + size.first = image.width(); + size.second = image.height(); } #endif // !MEDIAPIPE_DISABLE_GPU - cc->Outputs().Tag("SIZE").AddPacket( - MakePacket>(width, height) - .At(cc->InputTimestamp())); + kOut(cc).Send(size); return absl::OkStatus(); } }; -REGISTER_CALCULATOR(ImagePropertiesCalculator); +MEDIAPIPE_REGISTER_NODE(ImagePropertiesCalculator); + +} // namespace api2 } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 59e4646ea..2234787c9 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -585,6 +585,7 @@ cc_library( ], "//conditions:default": [], }), + visibility = ["//visibility:public"], deps = [ ":image_to_tensor_utils", "//mediapipe/framework/formats:image", diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc index c6c9a19f4..1c27f282a 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc @@ -312,7 +312,7 @@ class GlProcessor : public ImageToTensorConverter { return absl::OkStatus(); })); - return tensor; + return std::move(tensor); } ~GlProcessor() override { diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc index 565dd85b9..1f86e1ced 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc @@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter { tflite::gpu::HW(output_dims.height, output_dims.width), command_buffer, buffer_view.buffer())); [command_buffer commit]; - return tensor; + return std::move(tensor); } } diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc index b8d1b0a8b..04a4bbd97 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_opencv.cc @@ -103,7 +103,7 @@ class OpenCvProcessor : public ImageToTensorConverter { GetValueRangeTransformation(kInputImageRangeMin, kInputImageRangeMax, range_min, range_max)); transformed.convertTo(dst, CV_32FC3, transform.scale, transform.offset); - return tensor; + return std::move(tensor); } private: diff --git a/mediapipe/calculators/util/landmarks_smoothing_calculator.cc b/mediapipe/calculators/util/landmarks_smoothing_calculator.cc index 38bdb9d04..fb2310610 100644 --- a/mediapipe/calculators/util/landmarks_smoothing_calculator.cc +++ b/mediapipe/calculators/util/landmarks_smoothing_calculator.cc @@ -205,11 +205,12 @@ class VelocityFilter : public LandmarksFilter { class OneEuroFilterImpl : public LandmarksFilter { public: OneEuroFilterImpl(double frequency, double min_cutoff, double beta, - double derivate_cutoff) + double derivate_cutoff, float min_allowed_object_scale) : frequency_(frequency), min_cutoff_(min_cutoff), beta_(beta), - derivate_cutoff_(derivate_cutoff) {} + derivate_cutoff_(derivate_cutoff), + min_allowed_object_scale_(min_allowed_object_scale) {} absl::Status Reset() override { x_filters_.clear(); @@ -224,15 +225,25 @@ class OneEuroFilterImpl : public LandmarksFilter { // Initialize filters once. MP_RETURN_IF_ERROR(InitializeFiltersIfEmpty(in_landmarks.landmark_size())); + const float object_scale = GetObjectScale(in_landmarks); + if (object_scale < min_allowed_object_scale_) { + *out_landmarks = in_landmarks; + return absl::OkStatus(); + } + const float value_scale = 1.0f / object_scale; + // Filter landmarks. Every axis of every landmark is filtered separately. for (int i = 0; i < in_landmarks.landmark_size(); ++i) { const auto& in_landmark = in_landmarks.landmark(i); auto* out_landmark = out_landmarks->add_landmark(); *out_landmark = in_landmark; - out_landmark->set_x(x_filters_[i].Apply(timestamp, in_landmark.x())); - out_landmark->set_y(y_filters_[i].Apply(timestamp, in_landmark.y())); - out_landmark->set_z(z_filters_[i].Apply(timestamp, in_landmark.z())); + out_landmark->set_x( + x_filters_[i].Apply(timestamp, value_scale, in_landmark.x())); + out_landmark->set_y( + y_filters_[i].Apply(timestamp, value_scale, in_landmark.y())); + out_landmark->set_z( + z_filters_[i].Apply(timestamp, value_scale, in_landmark.z())); } return absl::OkStatus(); @@ -265,6 +276,7 @@ class OneEuroFilterImpl : public LandmarksFilter { double min_cutoff_; double beta_; double derivate_cutoff_; + double min_allowed_object_scale_; std::vector x_filters_; std::vector y_filters_; @@ -344,7 +356,8 @@ absl::Status LandmarksSmoothingCalculator::Open(CalculatorContext* cc) { options.one_euro_filter().frequency(), options.one_euro_filter().min_cutoff(), options.one_euro_filter().beta(), - options.one_euro_filter().derivate_cutoff()); + options.one_euro_filter().derivate_cutoff(), + options.one_euro_filter().min_allowed_object_scale()); } else { RET_CHECK_FAIL() << "Landmarks filter is either not specified or not supported"; diff --git a/mediapipe/calculators/util/landmarks_smoothing_calculator.proto b/mediapipe/calculators/util/landmarks_smoothing_calculator.proto index 2466fafe6..7699287c9 100644 --- a/mediapipe/calculators/util/landmarks_smoothing_calculator.proto +++ b/mediapipe/calculators/util/landmarks_smoothing_calculator.proto @@ -50,9 +50,9 @@ message LandmarksSmoothingCalculatorOptions { // For the details of the filter implementation and the procedure of its // configuration please check http://cristal.univ-lille.fr/~casiez/1euro/ message OneEuroFilter { - // Frequency of incomming frames defined in seconds. Used only if can't be - // calculated from provided events (e.g. on the very first frame). - optional float frequency = 1 [default = 0.033]; + // Frequency of incomming frames defined in frames per seconds. Used only if + // can't be calculated from provided events (e.g. on the very first frame). + optional float frequency = 1 [default = 30.0]; // Minimum cutoff frequency. Start by tuning this parameter while keeping // `beta = 0` to reduce jittering to the desired level. 1Hz (the default @@ -68,6 +68,10 @@ message LandmarksSmoothingCalculatorOptions { // algorithm, but can be tuned to further smooth the speed (i.e. derivate) // on the object. optional float derivate_cutoff = 4 [default = 1.0]; + + // If calculated object scale is less than given value smoothing will be + // disabled and landmarks will be returned as is. + optional float min_allowed_object_scale = 5 [default = 1e-6]; } oneof filter_options { diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc index 08d9704e5..59b21d574 100644 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc +++ b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc @@ -77,10 +77,12 @@ class RefineLandmarksFromHeatmapCalculatorImpl const auto& options = cc->Options(); - ASSIGN_OR_RETURN(auto out_lms, RefineLandmarksFromHeatMap( - in_lms, hm_raw, hm_tensor.shape().dims, - options.kernel_size(), - options.min_confidence_to_refine())); + ASSIGN_OR_RETURN( + auto out_lms, + RefineLandmarksFromHeatMap( + in_lms, hm_raw, hm_tensor.shape().dims, options.kernel_size(), + options.min_confidence_to_refine(), options.refine_presence(), + options.refine_visibility())); kOutLandmarks(cc).Send(std::move(out_lms)); return absl::OkStatus(); @@ -104,7 +106,8 @@ class RefineLandmarksFromHeatmapCalculatorImpl absl::StatusOr RefineLandmarksFromHeatMap( const mediapipe::NormalizedLandmarkList& in_lms, const float* heatmap_raw_data, const std::vector& heatmap_dims, - int kernel_size, float min_confidence_to_refine) { + int kernel_size, float min_confidence_to_refine, bool refine_presence, + bool refine_visibility) { ASSIGN_OR_RETURN(auto hm_dims, GetHwcFromDims(heatmap_dims)); auto [hm_height, hm_width, hm_channels] = hm_dims; @@ -136,7 +139,7 @@ absl::StatusOr RefineLandmarksFromHeatMap( float sum = 0; float weighted_col = 0; float weighted_row = 0; - float max_value = 0; + float max_confidence_value = 0; // Main loop. Go over kernel and calculate weighted sum of coordinates, // sum of weights and max weights. @@ -150,15 +153,33 @@ absl::StatusOr RefineLandmarksFromHeatMap( // options. float confidence = Sigmoid(heatmap_raw_data[idx]); sum += confidence; - max_value = std::max(max_value, confidence); + max_confidence_value = std::max(max_confidence_value, confidence); weighted_col += col * confidence; weighted_row += row * confidence; } } - if (max_value >= min_confidence_to_refine && sum > 0) { + if (max_confidence_value >= min_confidence_to_refine && sum > 0) { out_lms.mutable_landmark(lm_index)->set_x(weighted_col / hm_width / sum); out_lms.mutable_landmark(lm_index)->set_y(weighted_row / hm_height / sum); } + if (refine_presence && sum > 0 && + out_lms.landmark(lm_index).has_presence()) { + // We assume confidence in heatmaps describes landmark presence. + // If landmark is not confident in heatmaps, probably it is not present. + const float presence = out_lms.landmark(lm_index).presence(); + const float new_presence = std::min(presence, max_confidence_value); + out_lms.mutable_landmark(lm_index)->set_presence(new_presence); + } + if (refine_visibility && sum > 0 && + out_lms.landmark(lm_index).has_visibility()) { + // We assume confidence in heatmaps describes landmark presence. + // As visibility = (not occluded but still present) -> that mean that if + // landmark is not present, it is not visible as well. + // I.e. visibility confidence cannot be bigger than presence confidence. + const float visibility = out_lms.landmark(lm_index).visibility(); + const float new_visibility = std::min(visibility, max_confidence_value); + out_lms.mutable_landmark(lm_index)->set_visibility(new_visibility); + } } return out_lms; } diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h index 9656347e1..3985196bc 100644 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h +++ b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.h @@ -43,7 +43,8 @@ class RefineLandmarksFromHeatmapCalculator : public NodeIntf { absl::StatusOr RefineLandmarksFromHeatMap( const mediapipe::NormalizedLandmarkList& in_lms, const float* heatmap_raw_data, const std::vector& heatmap_dims, - int kernel_size, float min_confidence_to_refine); + int kernel_size, float min_confidence_to_refine, bool refine_presence, + bool refine_visibility); } // namespace mediapipe diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto index 1f8ff04b6..eebcaed0b 100644 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto +++ b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.proto @@ -24,4 +24,6 @@ message RefineLandmarksFromHeatmapCalculatorOptions { } optional int32 kernel_size = 1 [default = 9]; optional float min_confidence_to_refine = 2 [default = 0.5]; + optional bool refine_presence = 3 [default = false]; + optional bool refine_visibility = 4 [default = false]; } diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc index 83afacbbc..d484b401f 100644 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc +++ b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator_test.cc @@ -70,8 +70,8 @@ TEST(RefineLandmarksFromHeatmapTest, Smoke) { z, z, z}; // clang-format on - auto ret_or_error = RefineLandmarksFromHeatMap(vec_to_lms({{0.5, 0.5}}), - hm.data(), {3, 3, 1}, 3, 0.1); + auto ret_or_error = RefineLandmarksFromHeatMap( + vec_to_lms({{0.5, 0.5}}), hm.data(), {3, 3, 1}, 3, 0.1, true, true); MP_EXPECT_OK(ret_or_error); EXPECT_THAT(lms_to_vec(*ret_or_error), ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)))); @@ -94,7 +94,7 @@ TEST(RefineLandmarksFromHeatmapTest, MultiLayer) { auto ret_or_error = RefineLandmarksFromHeatMap( vec_to_lms({{0.5, 0.5}, {0.5, 0.5}, {0.5, 0.5}}), hm.data(), {3, 3, 3}, 3, - 0.1); + 0.1, true, true); MP_EXPECT_OK(ret_or_error); EXPECT_THAT(lms_to_vec(*ret_or_error), ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)), @@ -119,7 +119,7 @@ TEST(RefineLandmarksFromHeatmapTest, KeepIfNotSure) { auto ret_or_error = RefineLandmarksFromHeatMap( vec_to_lms({{0.5, 0.5}, {0.5, 0.5}, {0.5, 0.5}}), hm.data(), {3, 3, 3}, 3, - 0.6); + 0.6, true, true); MP_EXPECT_OK(ret_or_error); EXPECT_THAT(lms_to_vec(*ret_or_error), ElementsAre(Pair(FloatEq(0.5), FloatEq(0.5)), @@ -140,8 +140,9 @@ TEST(RefineLandmarksFromHeatmapTest, Border) { z, z, 0}, 3, 3, 2); // clang-format on - auto ret_or_error = RefineLandmarksFromHeatMap( - vec_to_lms({{0.0, 0.0}, {0.9, 0.9}}), hm.data(), {3, 3, 2}, 3, 0.1); + auto ret_or_error = + RefineLandmarksFromHeatMap(vec_to_lms({{0.0, 0.0}, {0.9, 0.9}}), + hm.data(), {3, 3, 2}, 3, 0.1, true, true); MP_EXPECT_OK(ret_or_error); EXPECT_THAT(lms_to_vec(*ret_or_error), ElementsAre(Pair(FloatEq(0), FloatEq(1 / 3.)), diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index 747a4eda8..f74e09fc6 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -1638,6 +1638,8 @@ cc_test( ":calculator_contract_test_cc_proto", ":calculator_framework", ":graph_validation", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:default_side_packet_calculator", "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework:packet_generator_cc_proto", diff --git a/mediapipe/framework/formats/image.h b/mediapipe/framework/formats/image.h index bfddefacf..1cde8e057 100644 --- a/mediapipe/framework/formats/image.h +++ b/mediapipe/framework/formats/image.h @@ -236,7 +236,8 @@ inline int Image::channels() const { inline int Image::step() const { if (use_gpu_) - return width() * ImageFrame::ByteDepthForFormat(image_format()); + return width() * channels() * + ImageFrame::ByteDepthForFormat(image_format()); else return image_frame_->WidthStep(); } diff --git a/mediapipe/framework/graph_validation_test.cc b/mediapipe/framework/graph_validation_test.cc index cf8223d83..c1ffa07de 100644 --- a/mediapipe/framework/graph_validation_test.cc +++ b/mediapipe/framework/graph_validation_test.cc @@ -499,5 +499,55 @@ TEST(GraphValidationTest, OptionalInputsForGraph) { MP_EXPECT_OK(graph_1.WaitUntilDone()); } +// Shows a calculator graph and DefaultSidePacketCalculator running with and +// without one optional side packet. +TEST(GraphValidationTest, DefaultOptionalInputsForGraph) { + // A subgraph defining one optional input-side-packet. + auto config_1 = ParseTextProtoOrDie(R"pb( + type: "PassThroughGraph" + input_side_packet: "side_input_0" + output_side_packet: "OUTPUT:output_0" + node { + calculator: "ConstantSidePacketCalculator" + options: { + [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { + packet { int_value: 2 } + } + } + output_side_packet: "PACKET:int_packet" + } + node { + calculator: "DefaultSidePacketCalculator" + input_side_packet: "OPTIONAL_VALUE:side_input_0" + input_side_packet: "DEFAULT_VALUE:int_packet" + output_side_packet: "VALUE:side_output_0" + } + )pb"); + GraphValidation validation_1; + MP_EXPECT_OK(validation_1.Validate({config_1}, {})); + CalculatorGraph graph_1; + MP_EXPECT_OK(graph_1.Initialize({config_1}, {})); + + // Run the graph specifying the optional side packet. + std::map side_packets; + side_packets.insert({"side_input_0", MakePacket(33)}); + MP_EXPECT_OK(graph_1.StartRun(side_packets)); + MP_EXPECT_OK(graph_1.CloseAllPacketSources()); + MP_EXPECT_OK(graph_1.WaitUntilDone()); + + // The specified side packet value is used. + auto side_packet_0 = graph_1.GetOutputSidePacket("side_output_0"); + EXPECT_EQ(side_packet_0->Get(), 33); + + // Run the graph omitting the optional inputs. + MP_EXPECT_OK(graph_1.StartRun({})); + MP_EXPECT_OK(graph_1.CloseAllPacketSources()); + MP_EXPECT_OK(graph_1.WaitUntilDone()); + + // The default side packet value is used. + side_packet_0 = graph_1.GetOutputSidePacket("side_output_0"); + EXPECT_EQ(side_packet_0->Get(), 2); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/profiler/graph_profiler.cc b/mediapipe/framework/profiler/graph_profiler.cc index eb7d80c62..5512a772f 100644 --- a/mediapipe/framework/profiler/graph_profiler.cc +++ b/mediapipe/framework/profiler/graph_profiler.cc @@ -604,7 +604,6 @@ absl::Status GraphProfiler::CaptureProfile(GraphProfile* result) { *result->mutable_calculator_profiles()->Add() = std::move(p); } this->Reset(); - AssignNodeNames(result); return status; } diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 7beabd152..890889a18 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -681,6 +681,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "//mediapipe/framework/tool:switch_container_cc_proto", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -705,6 +706,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "//mediapipe/framework/tool:switch_container_cc_proto", "@com_google_absl//absl/strings", ], alwayslink = 1, diff --git a/mediapipe/framework/tool/switch_container.cc b/mediapipe/framework/tool/switch_container.cc index f91275be7..d78cdfcef 100644 --- a/mediapipe/framework/tool/switch_container.cc +++ b/mediapipe/framework/tool/switch_container.cc @@ -82,6 +82,8 @@ CalculatorGraphConfig::Node* BuildDemuxNode( CalculatorGraphConfig* config) { CalculatorGraphConfig::Node* result = config->add_node(); *result->mutable_calculator() = "SwitchDemuxCalculator"; + *result->mutable_input_stream_handler()->mutable_input_stream_handler() = + "ImmediateInputStreamHandler"; return result; } @@ -91,9 +93,42 @@ CalculatorGraphConfig::Node* BuildMuxNode( CalculatorGraphConfig* config) { CalculatorGraphConfig::Node* result = config->add_node(); *result->mutable_calculator() = "SwitchMuxCalculator"; + *result->mutable_input_stream_handler()->mutable_input_stream_handler() = + "ImmediateInputStreamHandler"; return result; } +// Copies options from one node to another. +void CopyOptions(const CalculatorGraphConfig::Node& source, + CalculatorGraphConfig::Node* dest) { + if (source.has_options()) { + *dest->mutable_options() = source.options(); + } + *dest->mutable_node_options() = source.node_options(); +} + +// Clears options that are consumed by the container and not forwarded. +void ClearContainerOptions(SwitchContainerOptions* result) { + result->clear_contained_node(); +} + +// Clears options that are consumed by the container and not forwarded. +void ClearContainerOptions(CalculatorGraphConfig::Node* dest) { + if (dest->has_options() && + dest->mutable_options()->HasExtension(SwitchContainerOptions::ext)) { + ClearContainerOptions( + dest->mutable_options()->MutableExtension(SwitchContainerOptions::ext)); + } + for (google::protobuf::Any& a : *dest->mutable_node_options()) { + if (a.Is()) { + SwitchContainerOptions extension; + a.UnpackTo(&extension); + ClearContainerOptions(&extension); + a.PackFrom(extension); + } + } +} + // Returns an unused name similar to a specified name. std::string UniqueName(std::string name, std::set* names) { CHECK(names != nullptr); @@ -199,12 +234,16 @@ absl::StatusOr SwitchContainer::GetConfig( // Add a graph node for the demux, mux. auto demux = BuildDemuxNode(input_tags, &config); + CopyOptions(container_node, demux); + ClearContainerOptions(demux); demux->add_input_stream("SELECT:gate_select"); demux->add_input_stream("ENABLE:gate_enable"); demux->add_input_side_packet("SELECT:gate_select"); demux->add_input_side_packet("ENABLE:gate_enable"); auto mux = BuildMuxNode(output_tags, &config); + CopyOptions(container_node, mux); + ClearContainerOptions(mux); mux->add_input_stream("SELECT:gate_select"); mux->add_input_stream("ENABLE:gate_enable"); mux->add_input_side_packet("SELECT:gate_select"); diff --git a/mediapipe/framework/tool/switch_container_test.cc b/mediapipe/framework/tool/switch_container_test.cc index 9e21a934f..09d5ef2ac 100644 --- a/mediapipe/framework/tool/switch_container_test.cc +++ b/mediapipe/framework/tool/switch_container_test.cc @@ -225,6 +225,12 @@ TEST(SwitchContainerTest, ApplyToSubnodes) { input_stream: "foo" output_stream: "C0__:switchcontainer__c0__foo" output_stream: "C1__:switchcontainer__c1__foo" + options { + [mediapipe.SwitchContainerOptions.ext] {} + } + input_stream_handler { + input_stream_handler: "ImmediateInputStreamHandler" + } } node { name: "switchcontainer__TripleIntCalculator" @@ -245,6 +251,12 @@ TEST(SwitchContainerTest, ApplyToSubnodes) { input_stream: "C0__:switchcontainer__c0__bar" input_stream: "C1__:switchcontainer__c1__bar" output_stream: "bar" + options { + [mediapipe.SwitchContainerOptions.ext] {} + } + input_stream_handler { + input_stream_handler: "ImmediateInputStreamHandler" + } } node { calculator: "PassThroughCalculator" @@ -270,6 +282,75 @@ TEST(SwitchContainerTest, RunsWithSubnodes) { RunTestContainer(supergraph); } +// Shows the SwitchContainer does not allow input_stream_handler overwrite. +TEST(SwitchContainerTest, ValidateInputStreamHandler) { + EXPECT_TRUE(SubgraphRegistry::IsRegistered("SwitchContainer")); + CalculatorGraph graph; + CalculatorGraphConfig supergraph = SideSubnodeContainerExample(); + *supergraph.mutable_input_stream_handler()->mutable_input_stream_handler() = + "DefaultInputStreamHandler"; + MP_ASSERT_OK(graph.Initialize(supergraph, {})); + CalculatorGraphConfig expected_graph = mediapipe::ParseTextProtoOrDie< + CalculatorGraphConfig>(R"pb( + node { + name: "switchcontainer__SwitchDemuxCalculator" + calculator: "SwitchDemuxCalculator" + input_side_packet: "ENABLE:enable" + input_side_packet: "foo" + output_side_packet: "C0__:switchcontainer__c0__foo" + output_side_packet: "C1__:switchcontainer__c1__foo" + options { + [mediapipe.SwitchContainerOptions.ext] {} + } + input_stream_handler { + input_stream_handler: "ImmediateInputStreamHandler" + } + } + node { + name: "switchcontainer__TripleIntCalculator" + calculator: "TripleIntCalculator" + input_side_packet: "switchcontainer__c0__foo" + output_side_packet: "switchcontainer__c0__bar" + input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" } + } + node { + name: "switchcontainer__PassThroughCalculator" + calculator: "PassThroughCalculator" + input_side_packet: "switchcontainer__c1__foo" + output_side_packet: "switchcontainer__c1__bar" + input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" } + } + node { + name: "switchcontainer__SwitchMuxCalculator" + calculator: "SwitchMuxCalculator" + input_side_packet: "ENABLE:enable" + input_side_packet: "C0__:switchcontainer__c0__bar" + input_side_packet: "C1__:switchcontainer__c1__bar" + output_side_packet: "bar" + options { + [mediapipe.SwitchContainerOptions.ext] {} + } + input_stream_handler { + input_stream_handler: "ImmediateInputStreamHandler" + } + } + node { + calculator: "PassThroughCalculator" + input_side_packet: "foo" + input_side_packet: "bar" + output_side_packet: "output_foo" + output_side_packet: "output_bar" + input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" } + } + input_stream_handler { input_stream_handler: "DefaultInputStreamHandler" } + executor {} + input_side_packet: "foo" + input_side_packet: "enable" + output_side_packet: "output_bar" + )pb"); + EXPECT_THAT(graph.Config(), mediapipe::EqualsProto(expected_graph)); +} + // Shows the SwitchContainer container applied to a pair of simple subnodes. TEST(SwitchContainerTest, ApplyToSideSubnodes) { EXPECT_TRUE(SubgraphRegistry::IsRegistered("SwitchContainer")); @@ -286,6 +367,12 @@ TEST(SwitchContainerTest, ApplyToSideSubnodes) { input_side_packet: "foo" output_side_packet: "C0__:switchcontainer__c0__foo" output_side_packet: "C1__:switchcontainer__c1__foo" + options { + [mediapipe.SwitchContainerOptions.ext] {} + } + input_stream_handler { + input_stream_handler: "ImmediateInputStreamHandler" + } } node { name: "switchcontainer__TripleIntCalculator" @@ -306,6 +393,12 @@ TEST(SwitchContainerTest, ApplyToSideSubnodes) { input_side_packet: "C0__:switchcontainer__c0__bar" input_side_packet: "C1__:switchcontainer__c1__bar" output_side_packet: "bar" + options { + [mediapipe.SwitchContainerOptions.ext] {} + } + input_stream_handler { + input_stream_handler: "ImmediateInputStreamHandler" + } } node { calculator: "PassThroughCalculator" diff --git a/mediapipe/framework/tool/switch_demux_calculator.cc b/mediapipe/framework/tool/switch_demux_calculator.cc index 35f9cc0a0..46e6c358e 100644 --- a/mediapipe/framework/tool/switch_demux_calculator.cc +++ b/mediapipe/framework/tool/switch_demux_calculator.cc @@ -70,17 +70,11 @@ REGISTER_CALCULATOR(SwitchDemuxCalculator); absl::Status SwitchDemuxCalculator::GetContract(CalculatorContract* cc) { // Allow any one of kSelectTag, kEnableTag. - if (cc->Inputs().HasTag(kSelectTag)) { - cc->Inputs().Tag(kSelectTag).Set(); - } else if (cc->Inputs().HasTag(kEnableTag)) { - cc->Inputs().Tag(kEnableTag).Set(); - } + cc->Inputs().Tag(kSelectTag).Set().Optional(); + cc->Inputs().Tag(kEnableTag).Set().Optional(); // Allow any one of kSelectTag, kEnableTag. - if (cc->InputSidePackets().HasTag(kSelectTag)) { - cc->InputSidePackets().Tag(kSelectTag).Set(); - } else if (cc->InputSidePackets().HasTag(kEnableTag)) { - cc->InputSidePackets().Tag(kEnableTag).Set(); - } + cc->InputSidePackets().Tag(kSelectTag).Set().Optional(); + cc->InputSidePackets().Tag(kEnableTag).Set().Optional(); // Set the types for all output channels to corresponding input types. std::set channel_tags = ChannelTags(cc->Outputs().TagMap()); diff --git a/mediapipe/framework/tool/switch_mux_calculator.cc b/mediapipe/framework/tool/switch_mux_calculator.cc index dd120a2ed..ffa611239 100644 --- a/mediapipe/framework/tool/switch_mux_calculator.cc +++ b/mediapipe/framework/tool/switch_mux_calculator.cc @@ -73,17 +73,11 @@ REGISTER_CALCULATOR(SwitchMuxCalculator); absl::Status SwitchMuxCalculator::GetContract(CalculatorContract* cc) { // Allow any one of kSelectTag, kEnableTag. - if (cc->Inputs().HasTag(kSelectTag)) { - cc->Inputs().Tag(kSelectTag).Set(); - } else if (cc->Inputs().HasTag(kEnableTag)) { - cc->Inputs().Tag(kEnableTag).Set(); - } + cc->Inputs().Tag(kSelectTag).Set().Optional(); + cc->Inputs().Tag(kEnableTag).Set().Optional(); // Allow any one of kSelectTag, kEnableTag. - if (cc->InputSidePackets().HasTag(kSelectTag)) { - cc->InputSidePackets().Tag(kSelectTag).Set(); - } else if (cc->InputSidePackets().HasTag(kEnableTag)) { - cc->InputSidePackets().Tag(kEnableTag).Set(); - } + cc->InputSidePackets().Tag(kSelectTag).Set().Optional(); + cc->InputSidePackets().Tag(kEnableTag).Set().Optional(); // Set the types for all input channels to corresponding output types. std::set channel_tags = ChannelTags(cc->Inputs().TagMap()); diff --git a/mediapipe/graphs/holistic_tracking/BUILD b/mediapipe/graphs/holistic_tracking/BUILD index 4d5a69439..14290e32f 100644 --- a/mediapipe/graphs/holistic_tracking/BUILD +++ b/mediapipe/graphs/holistic_tracking/BUILD @@ -44,7 +44,6 @@ cc_library( name = "holistic_tracking_gpu_deps", deps = [ ":holistic_tracking_to_render_data", - "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", @@ -63,7 +62,6 @@ cc_library( name = "holistic_tracking_cpu_graph_deps", deps = [ ":holistic_tracking_to_render_data", - "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:annotation_overlay_calculator", diff --git a/mediapipe/graphs/holistic_tracking/holistic_tracking_cpu.pbtxt b/mediapipe/graphs/holistic_tracking/holistic_tracking_cpu.pbtxt index 088bf3e9c..fead24569 100644 --- a/mediapipe/graphs/holistic_tracking/holistic_tracking_cpu.pbtxt +++ b/mediapipe/graphs/holistic_tracking/holistic_tracking_cpu.pbtxt @@ -36,23 +36,9 @@ node { } } -node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:0:model_complexity" - output_side_packet: "PACKET:1:smooth_landmarks" - node_options: { - [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { - packet { int_value: 1 } - packet { bool_value: true } - } - } -} - node { calculator: "HolisticLandmarkCpu" input_stream: "IMAGE:throttled_input_video" - input_side_packet: "MODEL_COMPLEXITY:model_complexity" - input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" output_stream: "POSE_LANDMARKS:pose_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" diff --git a/mediapipe/graphs/holistic_tracking/holistic_tracking_gpu.pbtxt b/mediapipe/graphs/holistic_tracking/holistic_tracking_gpu.pbtxt index a4e2da01e..dc85be423 100644 --- a/mediapipe/graphs/holistic_tracking/holistic_tracking_gpu.pbtxt +++ b/mediapipe/graphs/holistic_tracking/holistic_tracking_gpu.pbtxt @@ -36,23 +36,9 @@ node { } } -node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:0:model_complexity" - output_side_packet: "PACKET:1:smooth_landmarks" - node_options: { - [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { - packet { int_value: 1 } - packet { bool_value: true } - } - } -} - node { calculator: "HolisticLandmarkGpu" input_stream: "IMAGE:throttled_input_video" - input_side_packet: "MODEL_COMPLEXITY:model_complexity" - input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" output_stream: "POSE_LANDMARKS:pose_landmarks" output_stream: "POSE_ROI:pose_roi" output_stream: "POSE_DETECTION:pose_detection" diff --git a/mediapipe/graphs/pose_tracking/BUILD b/mediapipe/graphs/pose_tracking/BUILD index 53d5ef5e2..383d09ae2 100644 --- a/mediapipe/graphs/pose_tracking/BUILD +++ b/mediapipe/graphs/pose_tracking/BUILD @@ -24,10 +24,7 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "pose_tracking_gpu_deps", deps = [ - "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/calculators/image:image_properties_calculator", - "//mediapipe/calculators/util:landmarks_smoothing_calculator", "//mediapipe/graphs/pose_tracking/subgraphs:pose_renderer_gpu", "//mediapipe/modules/pose_landmark:pose_landmark_gpu", ], @@ -43,10 +40,7 @@ mediapipe_binary_graph( cc_library( name = "pose_tracking_cpu_deps", deps = [ - "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/calculators/image:image_properties_calculator", - "//mediapipe/calculators/util:landmarks_smoothing_calculator", "//mediapipe/graphs/pose_tracking/subgraphs:pose_renderer_cpu", "//mediapipe/modules/pose_landmark:pose_landmark_cpu", ], diff --git a/mediapipe/graphs/pose_tracking/pose_tracking_cpu.pbtxt b/mediapipe/graphs/pose_tracking/pose_tracking_cpu.pbtxt index 380e9e04c..3d21c51ea 100644 --- a/mediapipe/graphs/pose_tracking/pose_tracking_cpu.pbtxt +++ b/mediapipe/graphs/pose_tracking/pose_tracking_cpu.pbtxt @@ -29,54 +29,20 @@ node { output_stream: "throttled_input_video" } -node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:model_complexity" - node_options: { - [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { - packet { int_value: 1 } - } - } -} - # Subgraph that detects poses and corresponding landmarks. node { calculator: "PoseLandmarkCpu" - input_side_packet: "MODEL_COMPLEXITY:model_complexity" input_stream: "IMAGE:throttled_input_video" output_stream: "LANDMARKS:pose_landmarks" output_stream: "DETECTION:pose_detection" output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks" } -# Calculates size of the image. -node { - calculator: "ImagePropertiesCalculator" - input_stream: "IMAGE:throttled_input_video" - output_stream: "SIZE:image_size" -} - -# Smoothes pose landmarks in order to reduce jitter. -node { - calculator: "LandmarksSmoothingCalculator" - input_stream: "NORM_LANDMARKS:pose_landmarks" - input_stream: "IMAGE_SIZE:image_size" - output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_smoothed" - node_options: { - [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] { - velocity_filter: { - window_size: 5 - velocity_scale: 10.0 - } - } - } -} - # Subgraph that renders pose-landmark annotation onto the input image. node { calculator: "PoseRendererCpu" input_stream: "IMAGE:throttled_input_video" - input_stream: "LANDMARKS:pose_landmarks_smoothed" + input_stream: "LANDMARKS:pose_landmarks" input_stream: "ROI:roi_from_landmarks" input_stream: "DETECTION:pose_detection" output_stream: "IMAGE:output_video" diff --git a/mediapipe/graphs/pose_tracking/pose_tracking_gpu.pbtxt b/mediapipe/graphs/pose_tracking/pose_tracking_gpu.pbtxt index c47e76944..df2253034 100644 --- a/mediapipe/graphs/pose_tracking/pose_tracking_gpu.pbtxt +++ b/mediapipe/graphs/pose_tracking/pose_tracking_gpu.pbtxt @@ -29,54 +29,20 @@ node { output_stream: "throttled_input_video" } -node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:model_complexity" - node_options: { - [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { - packet { int_value: 1 } - } - } -} - # Subgraph that detects poses and corresponding landmarks. node { calculator: "PoseLandmarkGpu" - input_side_packet: "MODEL_COMPLEXITY:model_complexity" input_stream: "IMAGE:throttled_input_video" output_stream: "LANDMARKS:pose_landmarks" output_stream: "DETECTION:pose_detection" output_stream: "ROI_FROM_LANDMARKS:roi_from_landmarks" } -# Calculates size of the image. -node { - calculator: "ImagePropertiesCalculator" - input_stream: "IMAGE_GPU:throttled_input_video" - output_stream: "SIZE:image_size" -} - -# Smoothes pose landmarks in order to reduce jitter. -node { - calculator: "LandmarksSmoothingCalculator" - input_stream: "NORM_LANDMARKS:pose_landmarks" - input_stream: "IMAGE_SIZE:image_size" - output_stream: "NORM_FILTERED_LANDMARKS:pose_landmarks_smoothed" - node_options: { - [type.googleapis.com/mediapipe.LandmarksSmoothingCalculatorOptions] { - velocity_filter: { - window_size: 5 - velocity_scale: 10.0 - } - } - } -} - # Subgraph that renders pose-landmark annotation onto the input image. node { calculator: "PoseRendererGpu" input_stream: "IMAGE:throttled_input_video" - input_stream: "LANDMARKS:pose_landmarks_smoothed" + input_stream: "LANDMARKS:pose_landmarks" input_stream: "ROI:roi_from_landmarks" input_stream: "DETECTION:pose_detection" output_stream: "IMAGE:output_video" diff --git a/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt b/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt index 97e9f1441..fbbdaa098 100644 --- a/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt +++ b/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt @@ -154,7 +154,7 @@ node { # Extracts image size. node { calculator: "ImagePropertiesCalculator" - input_stream: "IMAGE:image" + input_stream: "IMAGE_CPU:image" output_stream: "SIZE:image_size" } diff --git a/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt b/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt index 835951376..fa1d5c260 100644 --- a/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt +++ b/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt @@ -52,7 +52,7 @@ input_stream: "IMAGE:image" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # Whether to filter landmarks across different input images to reduce jitter. diff --git a/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt b/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt index 21cf8d881..1f6fa63d7 100644 --- a/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt +++ b/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt @@ -52,7 +52,7 @@ input_stream: "IMAGE:image" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # Whether to filter landmarks across different input images to reduce jitter. diff --git a/mediapipe/modules/objectron/objectron_cpu.pbtxt b/mediapipe/modules/objectron/objectron_cpu.pbtxt index 6f8fbade5..834c56464 100644 --- a/mediapipe/modules/objectron/objectron_cpu.pbtxt +++ b/mediapipe/modules/objectron/objectron_cpu.pbtxt @@ -113,7 +113,7 @@ node { # Extracts image size from the input images. node { calculator: "ImagePropertiesCalculator" - input_stream: "IMAGE:image" + input_stream: "IMAGE_CPU:image" output_stream: "SIZE:image_size" } diff --git a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt index c4527f95c..d98ad4a4a 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_cpu.pbtxt @@ -28,7 +28,7 @@ input_stream: "ROI:roi" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # Pose landmarks within the given ROI. (NormalizedLandmarkList) diff --git a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt index 0ffa50f77..7cb87d0e1 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_by_roi_gpu.pbtxt @@ -28,7 +28,7 @@ input_stream: "ROI:roi" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # Pose landmarks within the given ROI. (NormalizedLandmarkList) diff --git a/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt index 78513ca70..e90f2961a 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_cpu.pbtxt @@ -29,12 +29,12 @@ type: "PoseLandmarkCpu" input_stream: "IMAGE:image" # Whether to filter landmarks across different input images to reduce jitter. -# If unspecified, functions as set to false. (bool) +# If unspecified, functions as set to true. (bool) input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # Pose landmarks within the given ROI. (NormalizedLandmarkList) @@ -117,7 +117,7 @@ node: { # Calculates size of the image. node { calculator: "ImagePropertiesCalculator" - input_stream: "IMAGE:image" + input_stream: "IMAGE_CPU:image" output_stream: "SIZE:image_size" } diff --git a/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt index b81c58c0b..6f777ed5e 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_filtering.pbtxt @@ -14,7 +14,7 @@ type: "PoseLandmarkFiltering" -# Whether to enable filtering. If unspecified, functions as not enabled. (bool) +# Whether to enable filtering. If unspecified, functions as enabled. (bool) input_side_packet: "ENABLE:enable" # Size of the image (width & height) where the landmarks are estimated from. @@ -37,6 +37,7 @@ node { output_stream: "NORM_FILTERED_LANDMARKS:filtered_visibility" options: { [mediapipe.SwitchContainerOptions.ext] { + enable: true contained_node: { calculator: "VisibilitySmoothingCalculator" options: { @@ -68,6 +69,7 @@ node { output_stream: "NORM_FILTERED_LANDMARKS:filtered_landmarks" options: { [mediapipe.SwitchContainerOptions.ext] { + enable: true contained_node: { calculator: "LandmarksSmoothingCalculator" options: { @@ -80,9 +82,16 @@ node { calculator: "LandmarksSmoothingCalculator" options: { [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { - velocity_filter: { - window_size: 5 - velocity_scale: 10.0 + one_euro_filter { + # Min cutoff 0.1 results into ~ 0.02 alpha in landmark EMA filter + # when landmark is static. + min_cutoff: 0.1 + # Beta 40.0 in combintation with min_cutoff 0.1 results into ~0.8 + # alpha in landmark EMA filter when landmark is moving fast. + beta: 40.0 + # Derivative cutoff 1.0 results into ~0.17 alpha in landmark + # velocity EMA filter. + derivate_cutoff: 1.0 } } } @@ -93,29 +102,13 @@ node { # Smoothes pose landmark visibilities to reduce jitter. node { - calculator: "SwitchContainer" - input_side_packet: "ENABLE:enable" + calculator: "VisibilitySmoothingCalculator" input_stream: "NORM_LANDMARKS:aux_landmarks" output_stream: "NORM_FILTERED_LANDMARKS:filtered_aux_visibility" options: { - [mediapipe.SwitchContainerOptions.ext] { - contained_node: { - calculator: "VisibilitySmoothingCalculator" - options: { - [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { - no_filter: {} - } - } - } - contained_node: { - calculator: "VisibilitySmoothingCalculator" - options: { - [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { - low_pass_filter { - alpha: 0.1 - } - } - } + [mediapipe.VisibilitySmoothingCalculatorOptions.ext] { + low_pass_filter { + alpha: 0.1 } } } @@ -123,31 +116,26 @@ node { # Smoothes auxiliary landmarks to reduce jitter. node { - calculator: "SwitchContainer" - input_side_packet: "ENABLE:enable" + calculator: "LandmarksSmoothingCalculator" input_stream: "NORM_LANDMARKS:filtered_aux_visibility" input_stream: "IMAGE_SIZE:image_size" output_stream: "NORM_FILTERED_LANDMARKS:filtered_aux_landmarks" options: { - [mediapipe.SwitchContainerOptions.ext] { - contained_node: { - calculator: "LandmarksSmoothingCalculator" - options: { - [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { - no_filter: {} - } - } - } - contained_node: { - calculator: "LandmarksSmoothingCalculator" - options: { - [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { - velocity_filter: { - window_size: 5 - velocity_scale: 10.0 - } - } - } + [mediapipe.LandmarksSmoothingCalculatorOptions.ext] { + # Auxiliary landmarks are smoothed heavier than main landmarks to + # make ROI crop for pose landmarks prediction very stable when + # object is not moving but responsive enough in case of sudden + # movements. + one_euro_filter { + # Min cutoff 0.01 results into ~ 0.002 alpha in landmark EMA + # filter when landmark is static. + min_cutoff: 0.01 + # Beta 1.0 in combintation with min_cutoff 0.01 results into ~0.2 + # alpha in landmark EMA filter when landmark is moving fast. + beta: 1.0 + # Derivative cutoff 1.0 results into ~0.17 alpha in landmark + # velocity EMA filter. + derivate_cutoff: 1.0 } } } diff --git a/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt index 4acd5dc59..c4397376e 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_gpu.pbtxt @@ -29,12 +29,12 @@ type: "PoseLandmarkGpu" input_stream: "IMAGE:image" # Whether to filter landmarks across different input images to reduce jitter. -# If unspecified, functions as set to false. (bool) +# If unspecified, functions as set to true. (bool) input_side_packet: "SMOOTH_LANDMARKS:smooth_landmarks" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # Pose landmarks within the given ROI. (NormalizedLandmarkList) diff --git a/mediapipe/modules/pose_landmark/pose_landmark_model_loader.pbtxt b/mediapipe/modules/pose_landmark/pose_landmark_model_loader.pbtxt index d5a912b6d..ce7036ebb 100644 --- a/mediapipe/modules/pose_landmark/pose_landmark_model_loader.pbtxt +++ b/mediapipe/modules/pose_landmark/pose_landmark_model_loader.pbtxt @@ -4,7 +4,7 @@ type: "PoseLandmarkModelLoader" # Complexity of the pose landmark model: 0, 1 or 2. Landmark accuracy as well as # inference latency generally go up with the model complexity. If unspecified, -# functions as set to 0. (int) +# functions as set to 1. (int) input_side_packet: "MODEL_COMPLEXITY:model_complexity" # TF Lite model represented as a FlatBuffer. @@ -18,6 +18,7 @@ node { output_side_packet: "PACKET:model_path" options: { [mediapipe.SwitchContainerOptions.ext] { + select: 1 contained_node: { calculator: "ConstantSidePacketCalculator" options: { diff --git a/mediapipe/python/solutions/download_utils.py b/mediapipe/python/solutions/download_utils.py new file mode 100644 index 000000000..3b69074b0 --- /dev/null +++ b/mediapipe/python/solutions/download_utils.py @@ -0,0 +1,37 @@ +# Copyright 2021 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe Downloading utils.""" + +import os +import shutil +import urllib.request + +_OSS_URL_PREFIX = 'https://github.com/google/mediapipe/raw/master/' + + +def download_oss_model(model_path: str): + """Downloads the oss model from the MediaPipe GitHub repo if it doesn't exist in the package.""" + + mp_root_path = os.sep.join(os.path.abspath(__file__).split(os.sep)[:-4]) + model_abspath = os.path.join(mp_root_path, model_path) + if os.path.exists(model_abspath): + return + model_url = _OSS_URL_PREFIX + model_path + print('Downloading model to ' + model_abspath) + with urllib.request.urlopen(model_url) as response, open(model_abspath, + 'wb') as out_file: + if response.code != 200: + raise ConnectionError('Cannot download ' + model_path + + ' from the MediaPipe Github repo.') + shutil.copyfileobj(response, out_file) diff --git a/mediapipe/python/solutions/hands.py b/mediapipe/python/solutions/hands.py index a4bd035ab..15760ed75 100644 --- a/mediapipe/python/solutions/hands.py +++ b/mediapipe/python/solutions/hands.py @@ -44,7 +44,7 @@ class HandLandmark(enum.IntEnum): WRIST = 0 THUMB_CMC = 1 THUMB_MCP = 2 - THUMB_DIP = 3 + THUMB_IP = 3 THUMB_TIP = 4 INDEX_FINGER_MCP = 5 INDEX_FINGER_PIP = 6 @@ -68,8 +68,8 @@ BINARYPB_FILE_PATH = 'mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu HAND_CONNECTIONS = frozenset([ (HandLandmark.WRIST, HandLandmark.THUMB_CMC), (HandLandmark.THUMB_CMC, HandLandmark.THUMB_MCP), - (HandLandmark.THUMB_MCP, HandLandmark.THUMB_DIP), - (HandLandmark.THUMB_DIP, HandLandmark.THUMB_TIP), + (HandLandmark.THUMB_MCP, HandLandmark.THUMB_IP), + (HandLandmark.THUMB_IP, HandLandmark.THUMB_TIP), (HandLandmark.WRIST, HandLandmark.INDEX_FINGER_MCP), (HandLandmark.INDEX_FINGER_MCP, HandLandmark.INDEX_FINGER_PIP), (HandLandmark.INDEX_FINGER_PIP, HandLandmark.INDEX_FINGER_DIP), diff --git a/mediapipe/python/solutions/holistic.py b/mediapipe/python/solutions/holistic.py index ac204e243..64b63ab4f 100644 --- a/mediapipe/python/solutions/holistic.py +++ b/mediapipe/python/solutions/holistic.py @@ -1,4 +1,4 @@ -# Copyright 2020 The MediaPipe Authors. +# Copyright 2020-2021 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ from typing import NamedTuple import numpy as np from mediapipe.calculators.core import constant_side_packet_calculator_pb2 +# The following imports are needed because python pb2 silently discards +# unknown protobuf fields. # pylint: disable=unused-import from mediapipe.calculators.core import gate_calculator_pb2 from mediapipe.calculators.core import split_vector_calculator_pb2 @@ -32,9 +34,12 @@ from mediapipe.calculators.util import landmark_projection_calculator_pb2 from mediapipe.calculators.util import local_file_contents_calculator_pb2 from mediapipe.calculators.util import non_max_suppression_calculator_pb2 from mediapipe.calculators.util import rect_transformation_calculator_pb2 +from mediapipe.framework.tool import switch_container_pb2 from mediapipe.modules.holistic_landmark.calculators import roi_tracking_calculator_pb2 # pylint: enable=unused-import + from mediapipe.python.solution_base import SolutionBase +from mediapipe.python.solutions import download_utils # pylint: disable=unused-import from mediapipe.python.solutions.face_mesh import FACE_CONNECTIONS from mediapipe.python.solutions.hands import HAND_CONNECTIONS @@ -46,6 +51,17 @@ from mediapipe.python.solutions.pose import PoseLandmark BINARYPB_FILE_PATH = 'mediapipe/modules/holistic_landmark/holistic_landmark_cpu.binarypb' +def _download_oss_pose_landmark_model(model_complexity): + """Downloads the pose landmark lite/heavy model from the MediaPipe Github repo if it doesn't exist in the package.""" + + if model_complexity == 0: + download_utils.download_oss_model( + 'mediapipe/modules/pose_landmark/pose_landmark_lite.tflite') + elif model_complexity == 2: + download_utils.download_oss_model( + 'mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite') + + class Holistic(SolutionBase): """MediaPipe Holistic. @@ -81,6 +97,7 @@ class Holistic(SolutionBase): pose landmarks to be considered tracked successfully. See details in https://solutions.mediapipe.dev/holistic#min_tracking_confidence. """ + _download_oss_pose_landmark_model(model_complexity) super().__init__( binary_graph_path=BINARYPB_FILE_PATH, side_inputs={ diff --git a/mediapipe/python/solutions/objectron.py b/mediapipe/python/solutions/objectron.py index 9681b645a..195c2b8c7 100644 --- a/mediapipe/python/solutions/objectron.py +++ b/mediapipe/python/solutions/objectron.py @@ -1,4 +1,4 @@ -# Copyright 2020 The MediaPipe Authors. +# Copyright 2020-2021 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,10 +15,7 @@ """MediaPipe Objectron.""" import enum -import os -import shutil from typing import List, Tuple, NamedTuple, Optional -import urllib.request import attr import numpy as np @@ -48,6 +45,7 @@ from mediapipe.modules.objectron.calculators import frame_annotation_to_rect_cal from mediapipe.modules.objectron.calculators import lift_2d_frame_annotation_to_3d_calculator_pb2 # pylint: enable=unused-import from mediapipe.python.solution_base import SolutionBase +from mediapipe.python.solutions import download_utils class BoxLandmark(enum.IntEnum): @@ -92,23 +90,6 @@ BOX_CONNECTIONS = frozenset([ (BoxLandmark.FRONT_BOTTOM_RIGHT, BoxLandmark.FRONT_TOP_RIGHT), (BoxLandmark.BACK_TOP_RIGHT, BoxLandmark.FRONT_TOP_RIGHT), ]) -_OSS_URL_PREFIX = 'https://github.com/google/mediapipe/raw/master/' - - -def _download_oss_model(model_path: str): - """Download the objectron oss model from GitHub if it doesn't exist in the package.""" - - mp_root_path = os.sep.join(os.path.abspath(__file__).split(os.sep)[:-4]) - model_abspath = os.path.join(mp_root_path, model_path) - if os.path.exists(model_abspath): - return - model_url = _OSS_URL_PREFIX + model_path - with urllib.request.urlopen(model_url) as response, open(model_abspath, - 'wb') as out_file: - if response.code != 200: - raise ConnectionError('Cannot download ' + model_path + - ' from the MediaPipe Github repo.') - shutil.copyfileobj(response, out_file) @attr.s(auto_attribs=True) @@ -152,10 +133,19 @@ _MODEL_DICT = { } +def _download_oss_objectron_models(objectron_model: str): + """Downloads the objectron models from the MediaPipe Github repo if they don't exist in the package.""" + + download_utils.download_oss_model( + 'mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite' + ) + download_utils.download_oss_model(objectron_model) + + def get_model_by_name(name: str) -> ObjectronModel: if name not in _MODEL_DICT: raise ValueError(f'{name} is not a valid model name for Objectron.') - _download_oss_model(_MODEL_DICT[name].model_path) + _download_oss_objectron_models(_MODEL_DICT[name].model_path) return _MODEL_DICT[name] diff --git a/mediapipe/python/solutions/pose.py b/mediapipe/python/solutions/pose.py index 47d2d87f6..e25fe626c 100644 --- a/mediapipe/python/solutions/pose.py +++ b/mediapipe/python/solutions/pose.py @@ -1,4 +1,4 @@ -# Copyright 2020 The MediaPipe Authors. +# Copyright 2020-2021 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,6 +20,8 @@ from typing import NamedTuple import numpy as np from mediapipe.calculators.core import constant_side_packet_calculator_pb2 +# The following imports are needed because python pb2 silently discards +# unknown protobuf fields. # pylint: disable=unused-import from mediapipe.calculators.core import gate_calculator_pb2 from mediapipe.calculators.core import split_vector_calculator_pb2 @@ -37,8 +39,11 @@ from mediapipe.calculators.util import non_max_suppression_calculator_pb2 from mediapipe.calculators.util import rect_transformation_calculator_pb2 from mediapipe.calculators.util import thresholding_calculator_pb2 from mediapipe.calculators.util import visibility_smoothing_calculator_pb2 +from mediapipe.framework.tool import switch_container_pb2 # pylint: enable=unused-import + from mediapipe.python.solution_base import SolutionBase +from mediapipe.python.solutions import download_utils class PoseLandmark(enum.IntEnum): @@ -117,6 +122,17 @@ POSE_CONNECTIONS = frozenset([ ]) +def _download_oss_pose_landmark_model(model_complexity): + """Downloads the pose landmark lite/heavy model from the MediaPipe Github repo if it doesn't exist in the package.""" + + if model_complexity == 0: + download_utils.download_oss_model( + 'mediapipe/modules/pose_landmark/pose_landmark_lite.tflite') + elif model_complexity == 2: + download_utils.download_oss_model( + 'mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite') + + class Pose(SolutionBase): """MediaPipe Pose. @@ -151,6 +167,7 @@ class Pose(SolutionBase): pose landmarks to be considered tracked successfully. See details in https://solutions.mediapipe.dev/pose#min_tracking_confidence. """ + _download_oss_pose_landmark_model(model_complexity) super().__init__( binary_graph_path=BINARYPB_FILE_PATH, side_inputs={ diff --git a/mediapipe/util/filtering/one_euro_filter.cc b/mediapipe/util/filtering/one_euro_filter.cc index c2451c6dc..154236991 100644 --- a/mediapipe/util/filtering/one_euro_filter.cc +++ b/mediapipe/util/filtering/one_euro_filter.cc @@ -21,7 +21,8 @@ OneEuroFilter::OneEuroFilter(double frequency, double min_cutoff, double beta, last_time_ = 0; } -double OneEuroFilter::Apply(absl::Duration timestamp, double value) { +double OneEuroFilter::Apply(absl::Duration timestamp, double value_scale, + double value) { int64_t new_timestamp = absl::ToInt64Nanoseconds(timestamp); if (last_time_ >= new_timestamp) { // Results are unpredictable in this case, so nothing to do but @@ -39,7 +40,7 @@ double OneEuroFilter::Apply(absl::Duration timestamp, double value) { // estimate the current variation per second double dvalue = x_->HasLastRawValue() - ? (value - x_->LastRawValue()) * frequency_ + ? (value - x_->LastRawValue()) * value_scale * frequency_ : 0.0; // FIXME: 0.0 or value? double edvalue = dx_->ApplyWithAlpha(dvalue, GetAlpha(derivate_cutoff_)); // use it to update the cutoff frequency diff --git a/mediapipe/util/filtering/one_euro_filter.h b/mediapipe/util/filtering/one_euro_filter.h index 0d4dd2916..54d84f409 100644 --- a/mediapipe/util/filtering/one_euro_filter.h +++ b/mediapipe/util/filtering/one_euro_filter.h @@ -13,7 +13,7 @@ class OneEuroFilter { OneEuroFilter(double frequency, double min_cutoff, double beta, double derivate_cutoff); - double Apply(absl::Duration timestamp, double value); + double Apply(absl::Duration timestamp, double value_scale, double value); private: double GetAlpha(double cutoff); diff --git a/setup.py b/setup.py index c19ecf992..81569b34d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -"""Copyright 2020 The MediaPipe Authors. +"""Copyright 2020-2021 The MediaPipe Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -436,9 +436,9 @@ setuptools.setup( 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3 :: Only', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Artificial Intelligence',