Skip to content

Commit 817b581

Browse files
copybara-githubcopybara-github
committed
Merge pull request #5397 from priankakariatyml:ios-audio-task-runner
PiperOrigin-RevId: 632546454
2 parents 8a2ffeb + 45928e8 commit 817b581

File tree

11 files changed

+523
-3
lines changed

11 files changed

+523
-3
lines changed

‎mediapipe/tasks/ios/audio/core/BUILD

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,48 @@ objc_library(
7878
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
7979
],
8080
)
81+
82+
objc_library(
83+
name = "MPPAudioPacketCreator",
84+
srcs = ["sources/MPPAudioPacketCreator.mm"],
85+
hdrs = ["sources/MPPAudioPacketCreator.h"],
86+
copts = [
87+
"-ObjC++",
88+
"-std=c++17",
89+
],
90+
deps = [
91+
":MPPAudioData",
92+
"//mediapipe/framework:packet",
93+
"//mediapipe/framework:timestamp",
94+
"//mediapipe/framework/formats:matrix",
95+
"//mediapipe/tasks/ios/common:MPPCommon",
96+
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
97+
],
98+
)
99+
100+
objc_library(
101+
name = "MPPAudioRunningMode",
102+
hdrs = ["sources/MPPAudioRunningMode.h"],
103+
)
104+
105+
objc_library(
106+
name = "MPPAudioTaskRunner",
107+
srcs = ["sources/MPPAudioTaskRunner.mm"],
108+
hdrs = ["sources/MPPAudioTaskRunner.h"],
109+
copts = [
110+
"-ObjC++",
111+
"-std=c++17",
112+
],
113+
deps = [
114+
":MPPAudioData",
115+
":MPPAudioPacketCreator",
116+
":MPPAudioRunningMode",
117+
"//mediapipe/framework:packet",
118+
"//mediapipe/tasks/ios/common:MPPCommon",
119+
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
120+
"//mediapipe/tasks/ios/common/utils:NSStringHelpers",
121+
"//mediapipe/tasks/ios/core:MPPPacketCreator",
122+
"//mediapipe/tasks/ios/core:MPPTaskInfo",
123+
"//mediapipe/tasks/ios/core:MPPTaskRunner",
124+
],
125+
)

‎mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ NS_SWIFT_NAME(AudioData)
3434
@interface MPPAudioData : NSObject
3535

3636
/** Audio format specifying the number of channels and sample rate supported. */
37-
@property(nonatomic, readonly) MPPAudioDataFormat *audioFormat;
37+
@property(nonatomic, readonly) MPPAudioDataFormat *format;
3838

3939
/**
4040
* A copy of all the internal buffer elements in order with the most recent elements appearing at

‎mediapipe/tasks/ios/audio/core/sources/MPPAudioData.m

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ @implementation MPPAudioData {
2424
- (instancetype)initWithFormat:(MPPAudioDataFormat *)format sampleCount:(NSUInteger)sampleCount {
2525
self = [super init];
2626
if (self) {
27-
_audioFormat = format;
27+
_format = format;
2828

2929
const NSInteger length = sampleCount * format.channelCount;
3030
_ringBuffer = [[MPPFloatRingBuffer alloc] initWithLength:length];
@@ -40,7 +40,7 @@ - (BOOL)loadBuffer:(MPPFloatBuffer *)buffer
4040
}
4141

4242
- (BOOL)loadAudioRecord:(MPPAudioRecord *)audioRecord error:(NSError **)error {
43-
if (![audioRecord.audioDataFormat isEqual:self.audioFormat]) {
43+
if (![audioRecord.audioDataFormat isEqual:self.format]) {
4444
[MPPCommonUtils createCustomError:error
4545
withCode:MPPTasksErrorCodeInvalidArgumentError
4646
description:@"The provided audio record has incompatible audio format"];

‎mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright 2024 The MediaPipe Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#import <Foundation/Foundation.h>
16+
17+
#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h"
18+
19+
#include "mediapipe/framework/packet.h"
20+
21+
/**
22+
* This class helps create various kinds of packets for MediaPipe Audio Tasks.
23+
*/
24+
@interface MPPAudioPacketCreator : NSObject
25+
26+
/**
27+
* Creates a MediapPipe Packet wrapping the buffer of a `MPPAudioData` that can be send to a graph.
28+
*
29+
* @param audioData The audio data of type `MPPAudioData` to send to the MediaPipe graph.
30+
* @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
31+
* error will be saved.
32+
*
33+
* @return The MediaPipe packet containing the buffer of the given audio data. An empty packet is
34+
* returned if an error occurred during the conversion.
35+
*/
36+
+ (mediapipe::Packet)createPacketWithAudioData:(MPPAudioData *)audioData error:(NSError **)error;
37+
38+
/**
39+
* Creates a MediapPipe Packet wrapping the buffer of a `MPPAudioData` that can be send to a graph
40+
* at the specified timestamp.
41+
*
42+
* @param audioData The audio data of type `MPPAudioData` to send to the MediaPipe graph.
43+
* @param timestampInMilliseconds The timestamp (in milliseconds) to assign to the packet.
44+
* @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
45+
* error will be saved.
46+
*
47+
* @return The MediaPipe packet containing the buffer of the given audio data. An empty packet is
48+
* returned if an error occurred during the conversion.
49+
*/
50+
+ (mediapipe::Packet)createPacketWithAudioData:(MPPAudioData *)audioData
51+
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
52+
error:(NSError **)error;
53+
54+
@end

‎mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.mm

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Copyright 2024 The MediaPipe Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioPacketCreator.h"
16+
17+
#import "mediapipe/tasks/ios/common/sources/MPPCommon.h"
18+
#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h"
19+
20+
#include "mediapipe/framework/formats/matrix.h"
21+
#include "mediapipe/framework/timestamp.h"
22+
23+
static const NSUInteger kMicrosecondsPerMillisecond = 1000;
24+
25+
namespace {
26+
using ::mediapipe::Adopt;
27+
using ::mediapipe::Matrix;
28+
using ::mediapipe::Packet;
29+
using ::mediapipe::Timestamp;
30+
} // namespace
31+
32+
@implementation MPPAudioPacketCreator
33+
34+
+ (Packet)createPacketWithAudioData:(MPPAudioData *)audioData error:(NSError **)error {
35+
std::unique_ptr<Matrix> matrix = [MPPAudioPacketCreator createMatrixWithAudioData:audioData
36+
error:error];
37+
if (!matrix) {
38+
return Packet();
39+
}
40+
41+
return mediapipe::Adopt(matrix.release());
42+
}
43+
44+
+ (Packet)createPacketWithAudioData:(MPPAudioData *)audioData
45+
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
46+
error:(NSError **)error {
47+
std::unique_ptr<Matrix> matrix = [MPPAudioPacketCreator createMatrixWithAudioData:audioData
48+
error:error];
49+
if (!matrix) {
50+
return Packet();
51+
}
52+
return Adopt(matrix.release())
53+
.At(Timestamp(int64_t(timestampInMilliseconds * kMicrosecondsPerMillisecond)));
54+
}
55+
56+
+ (std::unique_ptr<Matrix>)createMatrixWithAudioData:(MPPAudioData *)audioData
57+
error:(NSError **)error {
58+
MPPFloatBuffer *audioDataBuffer = audioData.buffer;
59+
if (!audioDataBuffer.data) {
60+
[MPPCommonUtils createCustomError:error
61+
withCode:MPPTasksErrorCodeInvalidArgumentError
62+
description:@"Audio data buffer cannot be nil."];
63+
return nullptr;
64+
}
65+
66+
NSUInteger rowCount = audioData.format.channelCount;
67+
NSUInteger colCount = audioData.bufferLength;
68+
69+
std::unique_ptr<mediapipe::Matrix> matrix(new mediapipe::Matrix(rowCount, colCount));
70+
// iOS is always little-endian. Hence, data can be copied directly.
71+
memcpy(matrix->data(), audioDataBuffer.data, rowCount * colCount * sizeof(float));
72+
73+
return matrix;
74+
}
75+
76+
@end

‎mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2023 The MediaPipe Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#import <Foundation/Foundation.h>
16+
17+
NS_ASSUME_NONNULL_BEGIN
18+
19+
/**
20+
* MediaPipe audio task running mode. A MediaPipe audio task can be run with three different
21+
* modes: image, video and live stream.
22+
*/
23+
typedef NS_ENUM(NSUInteger, MPPAudioRunningMode) {
24+
25+
/** The mode for running a mediapipe audio task on independent audio clips. */
26+
MPPAudioRunningModeAudioClips NS_SWIFT_NAME(audioClips),
27+
28+
/**
29+
* The mode for running a mediapipe audio task on an audio stream, such as from a microphone.
30+
*/
31+
MPPAudioRunningModeAudioStream NS_SWIFT_NAME(audioStream),
32+
33+
} NS_SWIFT_NAME(RunningMode); // In Swift `RunningMode` can be resolved as
34+
// `MediaPipeTasksAudio.RunningMode` when used alongside the other
35+
// task libraries.
36+
37+
NS_INLINE NSString *MPPAudioRunningModeDisplayName(MPPAudioRunningMode runningMode) {
38+
switch (runningMode) {
39+
case MPPAudioRunningModeAudioClips:
40+
return @"Audio Clips";
41+
case MPPAudioRunningModeAudioStream:
42+
return @"Audio Stream";
43+
default:
44+
return nil;
45+
}
46+
}
47+
48+
NS_ASSUME_NONNULL_END

‎mediapipe/tasks/ios/audio/core/sources/MPPAudioTaskRunner.h

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Copyright 2023 The MediaPipe Authors.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#import <Foundation/Foundation.h>
16+
#import <UIKit/UIKit.h>
17+
18+
#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioData.h"
19+
#import "mediapipe/tasks/ios/audio/core/sources/MPPAudioRunningMode.h"
20+
#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h"
21+
22+
#include "mediapipe/framework/packet.h"
23+
24+
NS_ASSUME_NONNULL_BEGIN
25+
26+
/**
27+
* This class is used to create and call appropriate methods on the C++ Task Runner to initialize,
28+
* execute and terminate any MediaPipe audio task.
29+
*/
30+
@interface MPPAudioTaskRunner : MPPTaskRunner
31+
32+
/**
33+
* Initializes a new `MPPAudioTaskRunner` with the given task info, audio running mode, packets
34+
* callback, audio input and sample rate stream names. Make sure that the packets callback is set
35+
* properly based on the audio task's running mode. In case of audio stream running mode, a C++
36+
* packets callback that is intended to deliver inference results must be provided. In audio clips
37+
* mode, packets callback must be set to nil.
38+
*
39+
* @param taskInfo A `MPPTaskInfo` initialized by the task.
40+
* @param runningMode MediaPipe audio task running mode.
41+
* @param packetsCallback An optional C++ callback function that takes a list of output packets as
42+
* the input argument. If provided, the callback must in turn call the block provided by the user in
43+
* the appropriate task options. Make sure that the packets callback is set properly based on the
44+
* audio task's running mode. In case of audio stream running mode, a C++ packets callback that is
45+
* intended to deliver inference results must be provided. In audio clips running mode, packets
46+
* callback must be set to nil.
47+
* @param audioInputStreamName Name of the audio input stream of the task.
48+
* @param sampleRatInputStreamName Name of the sample rate input stream of the task.
49+
*
50+
* @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
51+
* error will be saved.
52+
*
53+
* @return An instance of `MPPAudioTaskRunner` initialized with the given task info, running mode,
54+
* packets callback, audio input and sample rate stream names.
55+
*/
56+
57+
- (nullable instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo
58+
runningMode:(MPPAudioRunningMode)runningMode
59+
packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback
60+
audioInputStreamName:(NSString *)imageInputStreamName
61+
sampleRateInputStreamName:(nullable NSString *)normRectInputStreamName
62+
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
63+
64+
/**
65+
* A synchronous method to invoke the C++ task runner to process standalone audio clip inputs. The
66+
* call blocks the current thread until a failure status or a successful result is returned.
67+
*
68+
*
69+
* @param audioClip An audio clip input of type `MPPAudioData` to the task.
70+
* @param error Pointer to the memory location where errors if any should be
71+
* saved. If @c NULL, no error will be saved.
72+
*
73+
* @return An optional `PacketMap` containing pairs of output stream name and data packet.
74+
*/
75+
- (std::optional<mediapipe::tasks::core::PacketMap>)processAudioClip:(MPPAudioData *)audioClip
76+
error:(NSError **)error;
77+
78+
- (instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo
79+
packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback
80+
error:(NSError **)error NS_UNAVAILABLE;
81+
82+
- (instancetype)init NS_UNAVAILABLE;
83+
84+
+ (instancetype)new NS_UNAVAILABLE;
85+
86+
@end
87+
88+
NS_ASSUME_NONNULL_END

0 commit comments

Comments
 (0)