Skip to content

Commit 771ce4a

Browse files
nift4nift4
committed
expand format support for audio processors used by DefaultAudioSink
This allows future extensions such as direct 24 bit / 32 bit (int) playback in DefaultAudioSink, and is a step towards Issue: #1931 Run audio processors possibly discarding data first to avoid wasting CPU on data that'll be discarded anyway.
1 parent 1918a25 commit 771ce4a

File tree

8 files changed

+309
-11
lines changed

8 files changed

+309
-11
lines changed

‎libraries/common/src/main/java/androidx/media3/common/util/Util.java

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,10 @@
117117
import java.lang.reflect.Method;
118118
import java.math.BigDecimal;
119119
import java.math.RoundingMode;
120+
import java.nio.BufferOverflowException;
120121
import java.nio.ByteBuffer;
121122
import java.nio.ByteOrder;
123+
import java.nio.ReadOnlyBufferException;
122124
import java.nio.charset.StandardCharsets;
123125
import java.util.ArrayDeque;
124126
import java.util.ArrayList;
@@ -3421,6 +3423,54 @@ public static long getNowUnixTimeMs(long elapsedRealtimeEpochOffsetMs) {
34213423
: SystemClock.elapsedRealtime() + elapsedRealtimeEpochOffsetMs;
34223424
}
34233425

3426+
/**
3427+
* Absolute <i>get</i> method for reading a sign-extended 24-bit integer value.
3428+
*
3429+
* <p>Reads three bytes at the given index, composing them into a 24-bit integer value according
3430+
* to the current byte order.
3431+
*
3432+
* @param buf The buffer to operate on
3433+
* @param index The index from which the bytes will be read
3434+
* @return The 24-bit integer value at the given index
3435+
* @throws IndexOutOfBoundsException If {@code index} is negative or not smaller than the buffer's
3436+
* limit, minus two
3437+
*/
3438+
public static int getInt24(ByteBuffer buf, int index) {
3439+
byte component1 = buf.get(buf.order() == ByteOrder.BIG_ENDIAN ? index : index + 2);
3440+
byte component2 = buf.get(index + 1);
3441+
byte component3 = buf.get(buf.order() == ByteOrder.BIG_ENDIAN ? index + 2 : index);
3442+
return ((component1 << 24) & 0xff000000
3443+
| (component2 << 16) & 0xff0000
3444+
| (component3 << 8) & 0xff00)
3445+
>> 8;
3446+
}
3447+
3448+
/**
3449+
* Relative <i>put</i> method for writing a 24-bit integer value.
3450+
*
3451+
* <p>Writes three bytes containing the given int value, in the current byte order, into this
3452+
* buffer at the current position, and then increments the position by three.
3453+
*
3454+
* @param val The short value to be written
3455+
* @param buf The buffer to operate on
3456+
* @throws BufferOverflowException If there are fewer than two bytes remaining in this buffer
3457+
* @throws ReadOnlyBufferException If this buffer is read-only
3458+
* @throws IllegalArgumentException If the value is out of range for a 24-bit integer
3459+
*/
3460+
public static void putInt24(ByteBuffer buf, int val) {
3461+
if ((val & ~0xffffff) != 0 && (val & ~0x7fffff) != 0xff800000) {
3462+
throw new IllegalArgumentException("value out of range: " + Integer.toHexString(val));
3463+
}
3464+
byte component1 =
3465+
buf.order() == ByteOrder.BIG_ENDIAN ? (byte) ((val & 0xFF0000) >> 16) : (byte) (val & 0xFF);
3466+
byte component2 = (byte) ((val & 0xFF00) >> 8);
3467+
byte component3 =
3468+
buf.order() == ByteOrder.BIG_ENDIAN ? (byte) (val & 0xFF) : (byte) ((val & 0xFF0000) >> 16);
3469+
buf.put(component1);
3470+
buf.put(component2);
3471+
buf.put(component3);
3472+
}
3473+
34243474
/**
34253475
* Moves the elements starting at {@code fromIndex} to {@code newFromIndex}.
34263476
*

‎libraries/common/src/test/java/androidx/media3/common/util/UtilTest.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@
2121
import static androidx.media3.common.util.Util.contentHashCode;
2222
import static androidx.media3.common.util.Util.escapeFileName;
2323
import static androidx.media3.common.util.Util.getCodecsOfType;
24+
import static androidx.media3.common.util.Util.getInt24;
2425
import static androidx.media3.common.util.Util.getStringForTime;
2526
import static androidx.media3.common.util.Util.gzip;
2627
import static androidx.media3.common.util.Util.maxValue;
2728
import static androidx.media3.common.util.Util.minValue;
2829
import static androidx.media3.common.util.Util.parseXsDateTime;
2930
import static androidx.media3.common.util.Util.parseXsDuration;
31+
import static androidx.media3.common.util.Util.putInt24;
3032
import static androidx.media3.common.util.Util.unescapeFileName;
3133
import static androidx.media3.test.utils.TestUtil.buildTestData;
3234
import static androidx.media3.test.utils.TestUtil.buildTestString;
@@ -1626,6 +1628,52 @@ public void contentHashCode_sparseArraysWithDifferentContent_returnsDifferentCon
16261628
assertThat(contentHashCode(sparseArray1)).isNotEqualTo(contentHashCode(sparseArray2));
16271629
}
16281630

1631+
@Test
1632+
public void putInt24_littleEndian() {
1633+
ByteBuffer buf = ByteBuffer.allocateDirect(2 * 3).order(ByteOrder.LITTLE_ENDIAN);
1634+
assertThrows(IllegalArgumentException.class, () -> putInt24(buf, 0xFF000000));
1635+
assertThrows(IllegalArgumentException.class, () -> putInt24(buf, 0x8FFFFFFF));
1636+
assertThrows(IllegalArgumentException.class, () -> putInt24(buf, 0x01FFFFFF));
1637+
putInt24(buf, -1);
1638+
putInt24(buf, 0x123456);
1639+
buf.rewind();
1640+
assertThat(createByteArray(buf))
1641+
.isEqualTo(new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, 0x56, 0x34, 0x12});
1642+
assertThat(getInt24(buf, 0)).isEqualTo(0xffffffff);
1643+
assertThat(getInt24(buf, 0) & 0xffffff).isEqualTo(0xffffff);
1644+
assertThat(getInt24(buf, 0)).isEqualTo(-1);
1645+
assertThat(getInt24(buf, 3)).isEqualTo(0x00123456);
1646+
1647+
buf.rewind();
1648+
putInt24(buf, 0xff0001);
1649+
putInt24(buf, 0x00ff02);
1650+
assertThat(getInt24(buf, 0) & 0xffffff).isEqualTo(0xff0001);
1651+
assertThat(getInt24(buf, 3) & 0xffffff).isEqualTo(0x00ff02);
1652+
}
1653+
1654+
@Test
1655+
public void putInt24_bigEndian() {
1656+
ByteBuffer buf = ByteBuffer.allocateDirect(2 * 3).order(ByteOrder.BIG_ENDIAN);
1657+
assertThrows(IllegalArgumentException.class, () -> putInt24(buf, 0xFF000000));
1658+
assertThrows(IllegalArgumentException.class, () -> putInt24(buf, 0x8FFFFFFF));
1659+
assertThrows(IllegalArgumentException.class, () -> putInt24(buf, 0x01FFFFFF));
1660+
putInt24(buf, -1);
1661+
putInt24(buf, 0x123456);
1662+
buf.rewind();
1663+
assertThat(createByteArray(buf))
1664+
.isEqualTo(new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, 0x12, 0x34, 0x56});
1665+
assertThat(getInt24(buf, 0)).isEqualTo(0xffffffff);
1666+
assertThat(getInt24(buf, 0) & 0xffffff).isEqualTo(0xffffff);
1667+
assertThat(getInt24(buf, 0)).isEqualTo(-1);
1668+
assertThat(getInt24(buf, 3)).isEqualTo(0x00123456);
1669+
1670+
buf.rewind();
1671+
putInt24(buf, 0xff0001);
1672+
putInt24(buf, 0x00ff02);
1673+
assertThat(getInt24(buf, 0) & 0xffffff).isEqualTo(0xff0001);
1674+
assertThat(getInt24(buf, 3) & 0xffffff).isEqualTo(0x00ff02);
1675+
}
1676+
16291677
private static void assertEscapeUnescapeFileName(String fileName, String escapedFileName) {
16301678
assertThat(escapeFileName(fileName)).isEqualTo(escapedFileName);
16311679
assertThat(unescapeFileName(escapedFileName)).isEqualTo(fileName);

‎libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ChannelMappingAudioProcessor.java

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import androidx.media3.common.audio.AudioProcessor;
2424
import androidx.media3.common.audio.BaseAudioProcessor;
2525
import androidx.media3.common.util.Assertions;
26+
import androidx.media3.common.util.Util;
2627
import java.nio.ByteBuffer;
2728
import java.util.Arrays;
2829

@@ -56,8 +57,7 @@ public AudioFormat onConfigure(AudioFormat inputAudioFormat)
5657
return AudioFormat.NOT_SET;
5758
}
5859

59-
if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT
60-
&& inputAudioFormat.encoding != C.ENCODING_PCM_FLOAT) {
60+
if (!Util.isEncodingLinearPcm(inputAudioFormat.encoding)) {
6161
throw new UnhandledAudioFormatException(inputAudioFormat);
6262
}
6363

@@ -91,9 +91,21 @@ public void queueInput(ByteBuffer inputBuffer) {
9191
for (int channelIndex : outputChannels) {
9292
int inputIndex = position + getByteDepth(inputAudioFormat.encoding) * channelIndex;
9393
switch (inputAudioFormat.encoding) {
94+
case C.ENCODING_PCM_8BIT:
95+
buffer.put(inputBuffer.get(inputIndex));
96+
break;
9497
case C.ENCODING_PCM_16BIT:
98+
case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
9599
buffer.putShort(inputBuffer.getShort(inputIndex));
96100
break;
101+
case C.ENCODING_PCM_24BIT:
102+
case C.ENCODING_PCM_24BIT_BIG_ENDIAN:
103+
Util.putInt24(buffer, Util.getInt24(inputBuffer, inputIndex));
104+
break;
105+
case C.ENCODING_PCM_32BIT:
106+
case C.ENCODING_PCM_32BIT_BIG_ENDIAN:
107+
buffer.putInt(inputBuffer.getInt(inputIndex));
108+
break;
97109
case C.ENCODING_PCM_FLOAT:
98110
buffer.putFloat(inputBuffer.getFloat(inputIndex));
99111
break;

‎libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,7 @@ public DefaultAudioSink build() {
521521
private final TrimmingAudioProcessor trimmingAudioProcessor;
522522
private final ImmutableList<AudioProcessor> toIntPcmAvailableAudioProcessors;
523523
private final ImmutableList<AudioProcessor> toFloatPcmAvailableAudioProcessors;
524+
private final ImmutableList<AudioProcessor> forPreProcessingAvailableAudioProcessors;
524525
private final AudioTrackPositionTracker audioTrackPositionTracker;
525526
private final ArrayDeque<MediaPositionParameters> mediaPositionParametersCheckpoints;
526527
private final boolean preferAudioTrackPlaybackParams;
@@ -599,12 +600,10 @@ private DefaultAudioSink(Builder builder) {
599600
audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener());
600601
channelMappingAudioProcessor = new ChannelMappingAudioProcessor();
601602
trimmingAudioProcessor = new TrimmingAudioProcessor();
602-
toIntPcmAvailableAudioProcessors =
603-
ImmutableList.of(
604-
new ToInt16PcmAudioProcessor(), channelMappingAudioProcessor, trimmingAudioProcessor);
605-
toFloatPcmAvailableAudioProcessors =
606-
ImmutableList.of(
607-
new ToFloatPcmAudioProcessor(), channelMappingAudioProcessor, trimmingAudioProcessor);
603+
toIntPcmAvailableAudioProcessors = ImmutableList.of(new ToInt16PcmAudioProcessor());
604+
toFloatPcmAvailableAudioProcessors = ImmutableList.of(new ToFloatPcmAudioProcessor());
605+
forPreProcessingAvailableAudioProcessors =
606+
ImmutableList.of(trimmingAudioProcessor, channelMappingAudioProcessor);
608607
volume = 1f;
609608
audioSessionId = C.AUDIO_SESSION_ID_UNSET;
610609
auxEffectInfo = new AuxEffectInfo(AuxEffectInfo.NO_AUX_EFFECT_ID, 0f);
@@ -702,6 +701,7 @@ public void configure(Format inputFormat, int specifiedBufferSize, @Nullable int
702701
inputPcmFrameSize = Util.getPcmFrameSize(inputFormat.pcmEncoding, inputFormat.channelCount);
703702

704703
ImmutableList.Builder<AudioProcessor> pipelineProcessors = new ImmutableList.Builder<>();
704+
pipelineProcessors.addAll(forPreProcessingAvailableAudioProcessors);
705705
if (shouldUseFloatOutput(inputFormat.pcmEncoding)) {
706706
pipelineProcessors.addAll(toFloatPcmAvailableAudioProcessors);
707707
} else {

‎libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/TrimmingAudioProcessor.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import static java.lang.Math.min;
1919

20-
import androidx.media3.common.C;
2120
import androidx.media3.common.Format;
2221
import androidx.media3.common.audio.BaseAudioProcessor;
2322
import androidx.media3.common.util.Util;
@@ -78,8 +77,7 @@ public long getDurationAfterProcessorApplied(long durationUs) {
7877
@Override
7978
public AudioFormat onConfigure(AudioFormat inputAudioFormat)
8079
throws UnhandledAudioFormatException {
81-
if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT
82-
&& inputAudioFormat.encoding != C.ENCODING_PCM_FLOAT) {
80+
if (!Util.isEncodingLinearPcm(inputAudioFormat.encoding)) {
8381
throw new UnhandledAudioFormatException(inputAudioFormat);
8482
}
8583
reconfigurationPending = true;

‎libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/ChannelMappingAudioProcessorTest.java

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,12 @@
1515
*/
1616
package androidx.media3.exoplayer.audio;
1717

18+
import static androidx.media3.test.utils.TestUtil.createByteArray;
1819
import static androidx.media3.test.utils.TestUtil.createByteBuffer;
1920
import static androidx.media3.test.utils.TestUtil.createFloatArray;
21+
import static androidx.media3.test.utils.TestUtil.createInt24Array;
22+
import static androidx.media3.test.utils.TestUtil.createInt24ByteBuffer;
23+
import static androidx.media3.test.utils.TestUtil.createIntArray;
2024
import static androidx.media3.test.utils.TestUtil.createShortArray;
2125
import static com.google.common.truth.Truth.assertThat;
2226

@@ -36,10 +40,22 @@ public class ChannelMappingAudioProcessorTest {
3640
new AudioFormat(
3741
/* sampleRate= */ 44100, /* channelCount= */ 3, /* encoding= */ C.ENCODING_PCM_FLOAT);
3842

43+
private static final AudioFormat PCM_32BIT_STEREO_FORMAT =
44+
new AudioFormat(
45+
/* sampleRate= */ 44100, /* channelCount= */ 2, /* encoding= */ C.ENCODING_PCM_32BIT);
46+
47+
private static final AudioFormat PCM_24BIT_STEREO_FORMAT =
48+
new AudioFormat(
49+
/* sampleRate= */ 44100, /* channelCount= */ 2, /* encoding= */ C.ENCODING_PCM_24BIT);
50+
3951
private static final AudioFormat PCM_16BIT_STEREO_FORMAT =
4052
new AudioFormat(
4153
/* sampleRate= */ 44100, /* channelCount= */ 2, /* encoding= */ C.ENCODING_PCM_16BIT);
4254

55+
private static final AudioFormat PCM_8BIT_STEREO_FORMAT =
56+
new AudioFormat(
57+
/* sampleRate= */ 44100, /* channelCount= */ 2, /* encoding= */ C.ENCODING_PCM_8BIT);
58+
4359
@Test
4460
public void channelMap_withPcmFloatSamples_mapsOutputCorrectly()
4561
throws AudioProcessor.UnhandledAudioFormatException {
@@ -53,6 +69,32 @@ public void channelMap_withPcmFloatSamples_mapsOutputCorrectly()
5369
assertThat(output).isEqualTo(new float[] {3f, 2f, 1f, 6f, 5f, 4f});
5470
}
5571

72+
@Test
73+
public void channelMap_withPcm32Samples_mapsOutputCorrectly()
74+
throws AudioProcessor.UnhandledAudioFormatException {
75+
ChannelMappingAudioProcessor processor = new ChannelMappingAudioProcessor();
76+
processor.setChannelMap(new int[] {1, 0});
77+
processor.configure(PCM_32BIT_STEREO_FORMAT);
78+
processor.flush();
79+
80+
processor.queueInput(createByteBuffer(new int[] {1, 2, 3, 4, 5, 6}));
81+
int[] output = createIntArray(processor.getOutput());
82+
assertThat(output).isEqualTo(new int[] {2, 1, 4, 3, 6, 5});
83+
}
84+
85+
@Test
86+
public void channelMap_withPcm24Samples_mapsOutputCorrectly()
87+
throws AudioProcessor.UnhandledAudioFormatException {
88+
ChannelMappingAudioProcessor processor = new ChannelMappingAudioProcessor();
89+
processor.setChannelMap(new int[] {1, 0});
90+
processor.configure(PCM_24BIT_STEREO_FORMAT);
91+
processor.flush();
92+
93+
processor.queueInput(createInt24ByteBuffer(new int[] {0xff0001, 0x00ff02, 0x0300ff, 0x40ff00}));
94+
int[] output = createInt24Array(processor.getOutput());
95+
assertThat(output).isEqualTo(new int[] {0x00ff02, 0xff0001, 0x40ff00, 0x0300ff});
96+
}
97+
5698
@Test
5799
public void channelMap_withPcm16Samples_mapsOutputCorrectly()
58100
throws AudioProcessor.UnhandledAudioFormatException {
@@ -66,6 +108,19 @@ public void channelMap_withPcm16Samples_mapsOutputCorrectly()
66108
assertThat(output).isEqualTo(new short[] {2, 1, 4, 3, 6, 5});
67109
}
68110

111+
@Test
112+
public void channelMap_withPcm8Samples_mapsOutputCorrectly()
113+
throws AudioProcessor.UnhandledAudioFormatException {
114+
ChannelMappingAudioProcessor processor = new ChannelMappingAudioProcessor();
115+
processor.setChannelMap(new int[] {1, 0});
116+
processor.configure(PCM_8BIT_STEREO_FORMAT);
117+
processor.flush();
118+
119+
processor.queueInput(createByteBuffer(new byte[] {1, 2, 3, 4, 5, 6}));
120+
byte[] output = createByteArray(processor.getOutput());
121+
assertThat(output).isEqualTo(new byte[] {2, 1, 4, 3, 6, 5});
122+
}
123+
69124
@Test
70125
public void channelMap_withMoreOutputChannels_duplicatesSamples()
71126
throws AudioProcessor.UnhandledAudioFormatException {

0 commit comments

Comments
 (0)