cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rewrite audio buffer conversion in Eigen primitives

Browse Source 
PiperOrigin-RevId: 520717550
This commit is contained in:
MediaPipe Team 2023-03-30 12:49:53 -07:00 committed by Copybara-Service
parent 984073bf73
commit 99ba7dd787
2 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mediapipe/objc/BUILD
View File

@ -204,6 +204,7 @@ objc_library(
"//third_party/apple_frameworks:AVFoundation", "//third_party/apple_frameworks:AVFoundation",
"//third_party/apple_frameworks:CoreAudio", "//third_party/apple_frameworks:CoreAudio",
"//third_party/apple_frameworks:CoreMedia", "//third_party/apple_frameworks:CoreMedia",
"@eigen_archive//:eigen3",
], ],
) )

58
mediapipe/objc/DrishtiAudioUtil.mm
View File

@ -16,50 +16,55 @@
#include <limits> #include <limits>
#include "third_party/eigen3/Eigen/Core"
namespace { namespace {
using Eigen::Index;
using Eigen::Map;
using Eigen::VectorXf;
using VectorXi16 = Eigen::Vector<SInt16, Eigen::Dynamic>;
// `float` is 32-bit. // `float` is 32-bit.
static_assert(std::numeric_limits<float>::is_iec559); static_assert(std::numeric_limits<float>::is_iec559);
using float32_t = float;
template <typename SampleDataType> // Reads an array of `size` elements of type `float` at `samples` and writes it into `target`,
float GetSample(const void* data, int index); // which is an Eigen expression compatible with a `VectorXf` of size `size`.
template <typename OutputVector>
template <> void CopyBufferToFloatVector(const float* samples, CMItemCount size, OutputVector target) {
float GetSample<float32_t>(const void* data, int index) { target = Map<const VectorXf>(samples, static_cast<Index>(size));
return reinterpret_cast<const float32_t*>(data)[index];
}; };
template <> // Reads an array of `size` elements of type `SInt16` at `samples` and writes it into `target`,
float GetSample<SInt16>(const void* data, int index) { // which is an Eigen expression compatible with a `VectorXf` of size `size`.
template <typename OutputVector>
void CopyBufferToFloatVector(const SInt16* samples, CMItemCount size, OutputVector target) {
// Convert to the [-1, 1] range. // Convert to the [-1, 1] range.
return static_cast<float>(reinterpret_cast<const SInt16*>(data)[index]) / constexpr float kRangeMax = static_cast<float>(std::numeric_limits<SInt16>::max());
static_cast<float>(std::numeric_limits<SInt16>::max()); target = Map<const VectorXi16>(samples, static_cast<Index>(size)).cast<float>() / kRangeMax;
}; };
template <typename SampleDataType> template <typename SampleDataType>
std::unique_ptr<mediapipe::Matrix> MakeMatrix(const AudioBuffer* buffers, int channels, std::unique_ptr<mediapipe::Matrix> MakeMatrix(const AudioBuffer* buffers, CMItemCount channels,
CMItemCount frames, bool interleaved) { CMItemCount frames, bool interleaved) {
// Create the matrix and fill it accordingly. Its dimensions are `channels x frames`. // Create the matrix and fill it accordingly. Its dimensions are `channels x frames`.
auto matrix = std::make_unique<mediapipe::Matrix>(channels, frames); auto matrix = std::make_unique<mediapipe::Matrix>(channels, frames);
// Split the case of interleaved and non-interleaved samples (see // Split the cases of interleaved and non-interleaved samples (see
// https://developer.apple.com/documentation/coremedia/1489723-cmsamplebuffercreate#discussion) // https://developer.apple.com/documentation/coremedia/1489723-cmsamplebuffercreate#discussion)
// - however, the resulting operations coincide when `channels == 1`. // - however, the resulting operations coincide when `channels == 1`.
if (interleaved) { if (interleaved) {
// A single buffer contains interleaved samples for all the channels {L, R, L, R, L, R, ...}. // A single buffer contains interleaved samples for all the channels {L, R, L, R, L, R, ...}.
const void* samples = buffers[0].mData; // This corresponds to Eigen's default column-major matrix layout.
for (int channel = 0; channel < channels; ++channel) { const SampleDataType* samples = reinterpret_cast<const SampleDataType*>(buffers[0].mData);
for (int frame = 0; frame < frames; ++frame) { CopyBufferToFloatVector(/*samples=*/samples, /*size=*/channels * frames,
(*matrix)(channel, frame) = GetSample<SampleDataType>(samples, channels * frame + channel); /*target=*/matrix->reshaped());
}
}
} else { } else {
// Non-interleaved audio: each channel's samples are stored in a separate buffer: // Non-interleaved audio: each channel's samples are stored in a separate buffer:
// {{L, L, L, L, ...}, {R, R, R, R, ...}}. // {{L, L, L, L, ...}, {R, R, R, R, ...}}.
for (int channel = 0; channel < channels; ++channel) { for (CMItemCount channel = 0; channel < channels; ++channel) {
const void* samples = buffers[channel].mData; const SampleDataType* samples =
for (int frame = 0; frame < frames; ++frame) { reinterpret_cast<const SampleDataType*>(buffers[channel].mData);
(*matrix)(channel, frame) = GetSample<SampleDataType>(samples, frame); CopyBufferToFloatVector(/*samples=*/samples, /*size=*/frames,
} /*target=*/matrix->row(static_cast<Index>(channel)));
} }
} }
return matrix; return matrix;
@ -89,13 +94,12 @@ absl::StatusOr<std::unique_ptr<mediapipe::Matrix>> MediaPipeConvertAudioBufferLi
if ((streamHeader->mFormatFlags & kAudioFormatFlagIsFloat) && if ((streamHeader->mFormatFlags & kAudioFormatFlagIsFloat) &&
streamHeader->mBitsPerChannel == 32) { streamHeader->mBitsPerChannel == 32) {
return MakeMatrix<float32_t>(audioBufferList->mBuffers, numChannels, numFrames, return MakeMatrix<float>(audioBufferList->mBuffers, numChannels, numFrames, isAudioInterleaved);
isAudioInterleaved);
} }
if ((streamHeader->mFormatFlags & kAudioFormatFlagIsSignedInteger) && if ((streamHeader->mFormatFlags & kAudioFormatFlagIsSignedInteger) &&
streamHeader->mBitsPerChannel == 16) { streamHeader->mBitsPerChannel == 16) {
return MakeMatrix<SInt16>(audioBufferList->mBuffers, numChannels, numFrames, return MakeMatrix<SInt16>(audioBufferList->mBuffers, numChannels, numFrames,
isAudioInterleaved); isAudioInterleaved);
} }
return absl::InternalError("Incompatible audio sample storage format"); return absl::InternalError("Unsupported audio sample storage format");
} }