Internal change
PiperOrigin-RevId: 521909998
This commit is contained in:
parent
9554836145
commit
f8b2aa0633
|
@ -125,8 +125,10 @@ objc_library(
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
deps = [
|
deps = [
|
||||||
"//third_party/apple_frameworks:AVFoundation",
|
"//third_party/apple_frameworks:AVFoundation",
|
||||||
|
"//third_party/apple_frameworks:CoreAudio",
|
||||||
"//third_party/apple_frameworks:CoreVideo",
|
"//third_party/apple_frameworks:CoreVideo",
|
||||||
"//third_party/apple_frameworks:Foundation",
|
"//third_party/apple_frameworks:Foundation",
|
||||||
|
"//third_party/apple_frameworks:MediaToolbox",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -13,8 +13,11 @@
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#import <AVFoundation/AVFoundation.h>
|
#import <AVFoundation/AVFoundation.h>
|
||||||
|
#import <CoreAudio/CoreAudioTypes.h>
|
||||||
#import <Foundation/Foundation.h>
|
#import <Foundation/Foundation.h>
|
||||||
|
|
||||||
|
NS_ASSUME_NONNULL_BEGIN
|
||||||
|
|
||||||
@class MPPInputSource;
|
@class MPPInputSource;
|
||||||
|
|
||||||
/// A delegate that can receive frames from a source.
|
/// A delegate that can receive frames from a source.
|
||||||
|
@ -31,7 +34,7 @@
|
||||||
timestamp:(CMTime)timestamp
|
timestamp:(CMTime)timestamp
|
||||||
fromSource:(MPPInputSource*)source;
|
fromSource:(MPPInputSource*)source;
|
||||||
|
|
||||||
// Provides the delegate with a new depth frame data
|
// Provides the delegate with new depth frame data.
|
||||||
@optional
|
@optional
|
||||||
- (void)processDepthData:(AVDepthData*)depthData
|
- (void)processDepthData:(AVDepthData*)depthData
|
||||||
timestamp:(CMTime)timestamp
|
timestamp:(CMTime)timestamp
|
||||||
|
@ -40,6 +43,23 @@
|
||||||
@optional
|
@optional
|
||||||
- (void)videoDidPlayToEnd:(CMTime)timestamp;
|
- (void)videoDidPlayToEnd:(CMTime)timestamp;
|
||||||
|
|
||||||
|
// Provides the delegate with the format of the audio track to be played.
|
||||||
|
@optional
|
||||||
|
- (void)willStartPlayingAudioWithFormat:(const AudioStreamBasicDescription*)format
|
||||||
|
fromSource:(MPPInputSource*)source;
|
||||||
|
|
||||||
|
// Lets the delegate know that there is no audio track despite audio playback
|
||||||
|
// having been requested (or that audio playback failed for other reasons).
|
||||||
|
@optional
|
||||||
|
- (void)noAudioAvailableFromSource:(MPPInputSource*)source;
|
||||||
|
|
||||||
|
// Provides the delegate with a new audio packet.
|
||||||
|
@optional
|
||||||
|
- (void)processAudioPacket:(const AudioBufferList*)audioPacket
|
||||||
|
numFrames:(CMItemCount)numFrames
|
||||||
|
timestamp:(CMTime)timestamp
|
||||||
|
fromSource:(MPPInputSource*)source;
|
||||||
|
|
||||||
@end
|
@end
|
||||||
|
|
||||||
/// Abstract class for a video source.
|
/// Abstract class for a video source.
|
||||||
|
@ -68,3 +88,5 @@
|
||||||
- (void)stop;
|
- (void)stop;
|
||||||
|
|
||||||
@end
|
@end
|
||||||
|
|
||||||
|
NS_ASSUME_NONNULL_END
|
||||||
|
|
|
@ -18,7 +18,15 @@
|
||||||
/// Not meant for batch processing of video.
|
/// Not meant for batch processing of video.
|
||||||
@interface MPPPlayerInputSource : MPPInputSource
|
@interface MPPPlayerInputSource : MPPInputSource
|
||||||
|
|
||||||
/// Designated initializer.
|
/// Initializes the video source with optional audio processing.
|
||||||
|
///
|
||||||
|
/// @param video The video asset to play.
|
||||||
|
/// @param audioProcessingEnabled If set, indicates that the (first) audio track
|
||||||
|
/// should be processed if it exists, and the corresponding methods for
|
||||||
|
/// audio will be invoked on the @c delegate.
|
||||||
|
- (instancetype)initWithAVAsset:(AVAsset*)video audioProcessingEnabled:(BOOL)audioProcessingEnabled;
|
||||||
|
|
||||||
|
/// Initializes the video source to process @c video with audio processing disabled.
|
||||||
- (instancetype)initWithAVAsset:(AVAsset*)video;
|
- (instancetype)initWithAVAsset:(AVAsset*)video;
|
||||||
|
|
||||||
/// Skip into video @c time from beginning (time 0), within error of +/- tolerance to closest time.
|
/// Skip into video @c time from beginning (time 0), within error of +/- tolerance to closest time.
|
||||||
|
|
|
@ -13,11 +13,13 @@
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
#import <CoreVideo/CoreVideo.h>
|
#import <CoreVideo/CoreVideo.h>
|
||||||
|
#import <MediaToolbox/MediaToolbox.h>
|
||||||
|
|
||||||
#import "MPPPlayerInputSource.h"
|
#import "MPPPlayerInputSource.h"
|
||||||
#if !TARGET_OS_OSX
|
#if !TARGET_OS_OSX
|
||||||
#import "mediapipe/objc/MPPDisplayLinkWeakTarget.h"
|
#import "mediapipe/objc/MPPDisplayLinkWeakTarget.h"
|
||||||
#endif
|
#endif
|
||||||
|
#import "mediapipe/objc/MPPInputSource.h"
|
||||||
|
|
||||||
@implementation MPPPlayerInputSource {
|
@implementation MPPPlayerInputSource {
|
||||||
AVAsset* _video;
|
AVAsset* _video;
|
||||||
|
@ -35,7 +37,53 @@
|
||||||
BOOL _playing;
|
BOOL _playing;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void InitAudio(MTAudioProcessingTapRef tap, void* clientInfo, void** tapStorageOut) {
|
||||||
|
// `clientInfo` comes as a user-defined argument through
|
||||||
|
// `MTAudioProcessingTapCallbacks`; we pass our `MPPPlayerInputSource`
|
||||||
|
// there. Tap processing functions allow for user-defined "storage" - we just
|
||||||
|
// treat our input source as such.
|
||||||
|
*tapStorageOut = clientInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
void PrepareAudio(MTAudioProcessingTapRef tap, CMItemCount maxFrames,
|
||||||
|
const AudioStreamBasicDescription* audioFormat) {
|
||||||
|
// See `InitAudio`.
|
||||||
|
MPPPlayerInputSource* source =
|
||||||
|
(__bridge MPPPlayerInputSource*)MTAudioProcessingTapGetStorage(tap);
|
||||||
|
if ([source.delegate respondsToSelector:@selector(willStartPlayingAudioWithFormat:fromSource:)]) {
|
||||||
|
[source.delegate willStartPlayingAudioWithFormat:audioFormat fromSource:source];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void ProcessAudio(MTAudioProcessingTapRef tap, CMItemCount numberFrames,
|
||||||
|
MTAudioProcessingTapFlags flags, AudioBufferList* bufferListInOut,
|
||||||
|
CMItemCount* numberFramesOut, MTAudioProcessingTapFlags* flagsOut) {
|
||||||
|
CMTimeRange timeRange;
|
||||||
|
OSStatus status = MTAudioProcessingTapGetSourceAudio(tap, numberFrames, bufferListInOut, flagsOut,
|
||||||
|
&timeRange, numberFramesOut);
|
||||||
|
if (status != 0) {
|
||||||
|
NSLog(@"Error from GetSourceAudio: %ld", (long)status);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// See `InitAudio`.
|
||||||
|
MPPPlayerInputSource* source =
|
||||||
|
(__bridge MPPPlayerInputSource*)MTAudioProcessingTapGetStorage(tap);
|
||||||
|
if ([source.delegate respondsToSelector:@selector(processAudioPacket:
|
||||||
|
numFrames:timestamp:fromSource:)]) {
|
||||||
|
[source.delegate processAudioPacket:bufferListInOut
|
||||||
|
numFrames:numberFrames
|
||||||
|
timestamp:timeRange.start
|
||||||
|
fromSource:source];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
- (instancetype)initWithAVAsset:(AVAsset*)video {
|
- (instancetype)initWithAVAsset:(AVAsset*)video {
|
||||||
|
return [self initWithAVAsset:video audioProcessingEnabled:NO];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (instancetype)initWithAVAsset:(AVAsset*)video
|
||||||
|
audioProcessingEnabled:(BOOL)audioProcessingEnabled {
|
||||||
self = [super init];
|
self = [super init];
|
||||||
if (self) {
|
if (self) {
|
||||||
_video = video;
|
_video = video;
|
||||||
|
@ -67,6 +115,11 @@
|
||||||
CVDisplayLinkStop(_videoDisplayLink);
|
CVDisplayLinkStop(_videoDisplayLink);
|
||||||
CVDisplayLinkSetOutputCallback(_videoDisplayLink, renderCallback, (__bridge void*)self);
|
CVDisplayLinkSetOutputCallback(_videoDisplayLink, renderCallback, (__bridge void*)self);
|
||||||
#endif // TARGET_OS_OSX
|
#endif // TARGET_OS_OSX
|
||||||
|
|
||||||
|
if (audioProcessingEnabled) {
|
||||||
|
[self setupAudioPlayback];
|
||||||
|
}
|
||||||
|
|
||||||
_videoPlayer = [AVPlayer playerWithPlayerItem:_videoItem];
|
_videoPlayer = [AVPlayer playerWithPlayerItem:_videoItem];
|
||||||
_videoPlayer.actionAtItemEnd = AVPlayerActionAtItemEndNone;
|
_videoPlayer.actionAtItemEnd = AVPlayerActionAtItemEndNone;
|
||||||
NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
|
NSNotificationCenter* center = [NSNotificationCenter defaultCenter];
|
||||||
|
@ -88,6 +141,47 @@
|
||||||
return self;
|
return self;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
- (void)setupAudioPlayback {
|
||||||
|
bool have_audio = false;
|
||||||
|
NSArray<AVAssetTrack*>* audioTracks =
|
||||||
|
[_video tracksWithMediaCharacteristic:AVMediaCharacteristicAudible];
|
||||||
|
if (audioTracks.count != 0) {
|
||||||
|
// We always limit ourselves to the first audio track if there are
|
||||||
|
// multiple (which is a rarity) - note that it can still be e.g. stereo.
|
||||||
|
AVAssetTrack* audioTrack = audioTracks[0];
|
||||||
|
MTAudioProcessingTapCallbacks audioCallbacks;
|
||||||
|
audioCallbacks.version = kMTAudioProcessingTapCallbacksVersion_0;
|
||||||
|
audioCallbacks.clientInfo = (__bridge void*)(self);
|
||||||
|
audioCallbacks.init = InitAudio;
|
||||||
|
audioCallbacks.prepare = PrepareAudio;
|
||||||
|
audioCallbacks.process = ProcessAudio;
|
||||||
|
audioCallbacks.unprepare = NULL;
|
||||||
|
audioCallbacks.finalize = NULL;
|
||||||
|
|
||||||
|
MTAudioProcessingTapRef audioTap;
|
||||||
|
OSStatus status =
|
||||||
|
MTAudioProcessingTapCreate(kCFAllocatorDefault, &audioCallbacks,
|
||||||
|
kMTAudioProcessingTapCreationFlag_PreEffects, &audioTap);
|
||||||
|
if (status == noErr && audioTap != NULL) {
|
||||||
|
AVMutableAudioMixInputParameters* audioMixInputParams =
|
||||||
|
[AVMutableAudioMixInputParameters audioMixInputParametersWithTrack:audioTrack];
|
||||||
|
audioMixInputParams.audioTapProcessor = audioTap;
|
||||||
|
CFRelease(audioTap);
|
||||||
|
|
||||||
|
AVMutableAudioMix* audioMix = [AVMutableAudioMix audioMix];
|
||||||
|
|
||||||
|
audioMix.inputParameters = @[ audioMixInputParams ];
|
||||||
|
_videoItem.audioMix = audioMix;
|
||||||
|
have_audio = true;
|
||||||
|
} else {
|
||||||
|
NSLog(@"Error %ld when trying to create the audio processing tap", (long)status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!have_audio && [self.delegate respondsToSelector:@selector(noAudioAvailableFromSource:)]) {
|
||||||
|
[self.delegate noAudioAvailableFromSource:self];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
- (void)start {
|
- (void)start {
|
||||||
[_videoPlayer play];
|
[_videoPlayer play];
|
||||||
_playing = YES;
|
_playing = YES;
|
||||||
|
|
Loading…
Reference in New Issue
Block a user