From 9455f59e71e0c9db3d07cbb32addf55bccc0fca3 Mon Sep 17 00:00:00 2001 From: lawadr <3211473+lawadr@users.noreply.github.com> Date: Wed, 11 Oct 2023 22:48:32 +0100 Subject: [PATCH 1/8] Fix playback of AAC in TS files When the channel config is 0 in the ADTS header, delay the submission of the format to the output until a Program Config Element (PCE) is found and appended to the format's initialization data. This reproduces the initialization data payload that is submitted by the same AAC stream when inside other containers such as MP4 or MKV. --- .../media3/extractor/ts/AdtsReader.java | 102 +++++++++++++++++- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java index de80f047d6c..b35af3b2fd4 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java @@ -50,6 +50,7 @@ public final class AdtsReader implements ElementaryStreamReader { private static final int STATE_READING_ID3_HEADER = 2; private static final int STATE_READING_ADTS_HEADER = 3; private static final int STATE_READING_SAMPLE = 4; + private static final int STATE_READING_AAC_PCE = 5; private static final int HEADER_SIZE = 5; private static final int CRC_SIZE = 2; @@ -66,6 +67,9 @@ public final class AdtsReader implements ElementaryStreamReader { private static final byte[] ID3_IDENTIFIER = {'I', 'D', '3'}; private static final int VERSION_UNSET = -1; + private static final int AAC_PCE_MIN_SIZE = 6; + private static final int AAC_PCE_MAX_SIZE = 49; + private final boolean exposeId3; private final ParsableBitArray adtsScratch; private final ParsableByteArray id3HeaderBuffer; @@ -94,6 +98,10 @@ public final class AdtsReader implements ElementaryStreamReader { private long sampleDurationUs; private int sampleSize; + // Used when reading the AAC PCE. + @Nullable private Format pendingOutputFormat; + @Nullable private ParsableBitArray pceBuffer; + // Used when reading the samples. private long timeUs; @@ -182,6 +190,11 @@ public void consume(ParsableByteArray data) throws ParserException { parseAdtsHeader(); } break; + case STATE_READING_AAC_PCE: + if (continueRead(data, pceBuffer.data, pceBuffer.data.length)) { + readAacProgramConfigElement(); + } + break; case STATE_READING_SAMPLE: readSample(data); break; @@ -272,6 +285,17 @@ private void setCheckingAdtsHeaderState() { bytesRead = 0; } + /** Sets the state to STATE_READING_AAC_PCE. */ + private void setReadingAacPceState( + TrackOutput outputToUse, long currentSampleDuration, int sampleSize) { + state = STATE_READING_AAC_PCE; + bytesRead = 0; + currentOutput = outputToUse; + this.currentSampleDuration = currentSampleDuration; + this.sampleSize = sampleSize; + pceBuffer = new ParsableBitArray(new byte[min(sampleSize, AAC_PCE_MAX_SIZE)]); + } + /** * Locates the next sample start, advancing the position to the byte that immediately follows * identifier. If a sample was not located, the position is advanced to the limit. @@ -515,8 +539,14 @@ private void parseAdtsHeader() throws ParserException { // In this class a sample is an access unit, but the MediaFormat sample rate specifies the // number of PCM audio samples per second. sampleDurationUs = (C.MICROS_PER_SECOND * 1024) / format.sampleRate; - output.format(format); - hasOutputFormat = true; + + if (channelConfig == 0) { + // Delay format submission until the AAC PCE is found and appended to audio specific config. + pendingOutputFormat = format; + } else { + output.format(format); + hasOutputFormat = true; + } } else { adtsScratch.skipBits(10); } @@ -527,7 +557,73 @@ private void parseAdtsHeader() throws ParserException { sampleSize -= CRC_SIZE; } - setReadingSampleState(output, sampleDurationUs, 0, sampleSize); + if (!hasOutputFormat && sampleSize >= AAC_PCE_MIN_SIZE) { + // As sample can fit a PCE, try reading it. + setReadingAacPceState(output, sampleDurationUs, sampleSize); + } else { + setReadingSampleState(output, sampleDurationUs, 0, sampleSize); + } + } + + @RequiresNonNull({"pendingOutputFormat", "pceBuffer"}) + void readAacProgramConfigElement() { + if (pceBuffer.readBits(3) == 5 /* PCE tag */) { + pceBuffer.skipBits(10); // Element instance tag, profile, sample frequency index + + int extraBits = 0; // Number of bits beyond amount to be parsed. + extraBits += pceBuffer.readBits(4) * 5; // Front channel elements + extraBits += pceBuffer.readBits(4) * 5; // Side channel elements + extraBits += pceBuffer.readBits(4) * 5; // Back channel elements + extraBits += pceBuffer.readBits(2) * 4; // LFE channel elements + extraBits += pceBuffer.readBits(3) * 4; // Data elements + extraBits += pceBuffer.readBits(4) * 5; // Coupling channel elements + + if (pceBuffer.readBit()) { + pceBuffer.skipBits(4); // Mono mixdown + } + + if (pceBuffer.readBit()) { + pceBuffer.skipBits(4); // Stereo mixdown + } + + if (pceBuffer.readBit()) { + pceBuffer.skipBits(3); // Matrix mixdown + } + + // Calculate total PCE size including initial PCE tag and a zero length comment. + int pceSize = (pceBuffer.getPosition() + extraBits + 7) / 8 + 1; + + if (sampleSize >= pceSize) { + // Calculate PCE size excluding initial PCE tag, alignment bits and a zero length comment. + int numPceBits = pceBuffer.getPosition() + extraBits - 3 /* PCE tag */; + + // Append PCE to format's audio specific config. + byte[] oldConfig = pendingOutputFormat.initializationData.get(0); + + int configSize = oldConfig.length; + configSize += (numPceBits + 7) / 8 + 1; // Byte align and add a zero length comment. + byte[] newConfig = new byte[configSize]; + + System.arraycopy(oldConfig, 0, newConfig, 0, oldConfig.length); + pceBuffer.setPosition(3 /* PCE tag */); + pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); + + pendingOutputFormat = pendingOutputFormat.buildUpon() + .setInitializationData(Collections.singletonList(newConfig)).build(); + + // Submit PCE-appended output format. + currentOutput.format(pendingOutputFormat); + hasOutputFormat = true; + } + } + + pendingOutputFormat = null; + + // Pass through all accumulated data as sample data. + ParsableByteArray data = new ParsableByteArray(pceBuffer.data); + pceBuffer = null; + setReadingSampleState(currentOutput, currentSampleDuration, 0, sampleSize); + readSample(data); } /** Reads the rest of the sample */ From 0617ee275011dbf990cf85e903dda84d040ab3d7 Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Wed, 22 Nov 2023 15:03:32 +0000 Subject: [PATCH 2/8] Format with google-java-format --- .../main/java/androidx/media3/extractor/ts/AdtsReader.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java index b35af3b2fd4..74c80f754cf 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java @@ -608,8 +608,11 @@ void readAacProgramConfigElement() { pceBuffer.setPosition(3 /* PCE tag */); pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); - pendingOutputFormat = pendingOutputFormat.buildUpon() - .setInitializationData(Collections.singletonList(newConfig)).build(); + pendingOutputFormat = + pendingOutputFormat + .buildUpon() + .setInitializationData(Collections.singletonList(newConfig)) + .build(); // Submit PCE-appended output format. currentOutput.format(pendingOutputFormat); From 333e4feca23845956bb73b02259d89f4be729756 Mon Sep 17 00:00:00 2001 From: lawadr <3211473+lawadr@users.noreply.github.com> Date: Thu, 23 Nov 2023 02:39:35 +0000 Subject: [PATCH 3/8] Fix based on feedback --- .../media3/extractor/ts/AdtsReader.java | 69 +++++++++++-------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java index 74c80f754cf..553502b4c8d 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java @@ -68,7 +68,7 @@ public final class AdtsReader implements ElementaryStreamReader { private static final int VERSION_UNSET = -1; private static final int AAC_PCE_MIN_SIZE = 6; - private static final int AAC_PCE_MAX_SIZE = 49; + private static final int AAC_PCE_MAX_SIZE = 50; private final boolean exposeId3; private final ParsableBitArray adtsScratch; @@ -568,15 +568,16 @@ private void parseAdtsHeader() throws ParserException { @RequiresNonNull({"pendingOutputFormat", "pceBuffer"}) void readAacProgramConfigElement() { if (pceBuffer.readBits(3) == 5 /* PCE tag */) { + // See `ISO 13818-7 Advanced Audio Coding (2006) Table 25` for format of a PCE pceBuffer.skipBits(10); // Element instance tag, profile, sample frequency index - int extraBits = 0; // Number of bits beyond amount to be parsed. - extraBits += pceBuffer.readBits(4) * 5; // Front channel elements - extraBits += pceBuffer.readBits(4) * 5; // Side channel elements - extraBits += pceBuffer.readBits(4) * 5; // Back channel elements - extraBits += pceBuffer.readBits(2) * 4; // LFE channel elements - extraBits += pceBuffer.readBits(3) * 4; // Data elements - extraBits += pceBuffer.readBits(4) * 5; // Coupling channel elements + int channelBits = 0; + channelBits += pceBuffer.readBits(4) * 5; // Front channel elements + channelBits += pceBuffer.readBits(4) * 5; // Side channel elements + channelBits += pceBuffer.readBits(4) * 5; // Back channel elements + channelBits += pceBuffer.readBits(2) * 4; // LFE channel elements + channelBits += pceBuffer.readBits(3) * 4; // Data elements + channelBits += pceBuffer.readBits(4) * 5; // Coupling channel elements if (pceBuffer.readBit()) { pceBuffer.skipBits(4); // Mono mixdown @@ -590,33 +591,43 @@ void readAacProgramConfigElement() { pceBuffer.skipBits(3); // Matrix mixdown } - // Calculate total PCE size including initial PCE tag and a zero length comment. - int pceSize = (pceBuffer.getPosition() + extraBits + 7) / 8 + 1; + // Beyond this point, pceBuffer may be empty, so check before consuming. - if (sampleSize >= pceSize) { - // Calculate PCE size excluding initial PCE tag, alignment bits and a zero length comment. - int numPceBits = pceBuffer.getPosition() + extraBits - 3 /* PCE tag */; + int numAlignmentBits = 8 - (pceBuffer.getPosition() + channelBits + 7) % 8 - 1; - // Append PCE to format's audio specific config. - byte[] oldConfig = pendingOutputFormat.initializationData.get(0); + if (pceBuffer.bitsLeft() >= channelBits + numAlignmentBits + 8) + { + pceBuffer.skipBits(channelBits); - int configSize = oldConfig.length; - configSize += (numPceBits + 7) / 8 + 1; // Byte align and add a zero length comment. - byte[] newConfig = new byte[configSize]; + // Store PCE size excluding initial PCE tag, alignment bits and comment for later. + int numPceBits = pceBuffer.getPosition() - 3 /* PCE tag */; - System.arraycopy(oldConfig, 0, newConfig, 0, oldConfig.length); - pceBuffer.setPosition(3 /* PCE tag */); - pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); + pceBuffer.skipBits(numAlignmentBits); + int commentSize = pceBuffer.readBits(8); - pendingOutputFormat = - pendingOutputFormat - .buildUpon() - .setInitializationData(Collections.singletonList(newConfig)) - .build(); + if (sampleSize >= pceBuffer.getBytePosition() + commentSize) + { + // Append PCE to format's audio specific config. + byte[] oldConfig = pendingOutputFormat.initializationData.get(0); - // Submit PCE-appended output format. - currentOutput.format(pendingOutputFormat); - hasOutputFormat = true; + int configSize = oldConfig.length; + configSize += (numPceBits + 7) / 8 + 1; // Byte align and add a zero length comment. + byte[] newConfig = new byte[configSize]; + + System.arraycopy(oldConfig, 0, newConfig, 0, oldConfig.length); + pceBuffer.setPosition(3 /* PCE tag */); + pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); + + pendingOutputFormat = + pendingOutputFormat + .buildUpon() + .setInitializationData(Collections.singletonList(newConfig)) + .build(); + + // Submit PCE-appended output format. + currentOutput.format(pendingOutputFormat); + hasOutputFormat = true; + } } } From cba2acdff233dae9aba992c8fdb967435cf9a231 Mon Sep 17 00:00:00 2001 From: lawadr <3211473+lawadr@users.noreply.github.com> Date: Sun, 26 Nov 2023 03:44:15 +0000 Subject: [PATCH 4/8] Add tests --- .../media3/extractor/ts/AdtsReaderTest.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java index cbdfd1872c8..14752fd40ca 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java @@ -61,6 +61,16 @@ public class AdtsReaderTest { private static final byte[] TEST_DATA = Bytes.concat(ID3_DATA_1, ID3_DATA_2, ADTS_HEADER, ADTS_CONTENT); + private static final byte[] AAC_PCE_ADTS_HEADER = + TestUtil.createByteArray(0xff, 0xf1, 0x50, 0x00, 0x02, 0x1f, 0xfc); + + private static final byte[] AAC_PCE_ADTS_CONTENT = + TestUtil.createByteArray( + 0xa0, 0x99, 0x01, 0x20, 0x00, 0x21, 0x19, 0x00, 0x00); + + private static final byte[] AAC_PCE_TEST_DATA = + Bytes.concat(AAC_PCE_ADTS_HEADER, AAC_PCE_ADTS_CONTENT); + private static final long ADTS_SAMPLE_DURATION = 23219L; private FakeTrackOutput adtsOutput; @@ -189,6 +199,28 @@ public void adtsDataOnly() throws ParserException { adtsOutput.assertSample(0, ADTS_CONTENT, 0, C.BUFFER_FLAG_KEY_FRAME, null); } + @Test + public void aacPceData() throws ParserException { + data = new ParsableByteArray(AAC_PCE_TEST_DATA); + feed(); + assertSampleCounts(0, 1); + adtsOutput.assertSample(0, AAC_PCE_ADTS_CONTENT, 0, C.BUFFER_FLAG_KEY_FRAME, null); + } + + @Test(expected = IllegalStateException.class) + public void aacPceDataFail() throws ParserException { + data = new ParsableByteArray(AAC_PCE_TEST_DATA); + byte[] bytes = data.getData(); + + // Remove PCE tag (first 3 bits of content). + bytes[AAC_PCE_ADTS_HEADER.length] &= 0x1f; + // Replace with CPE tag. + bytes[AAC_PCE_ADTS_HEADER.length] |= 0x20; + + // Should throw as FakeTrackOutput expects a format before sampleMetadata. + feed(); + } + private void feedLimited(int limit) throws ParserException { maybeStartPacket(); data.setLimit(limit); From cd3981230420837bedc1014c27857c50c39a97fd Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Mon, 27 Nov 2023 16:39:23 +0000 Subject: [PATCH 5/8] Format and comment changes --- .../media3/extractor/ts/AdtsReader.java | 62 +++++++++++-------- .../media3/extractor/ts/AdtsReaderTest.java | 8 +-- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java index 553502b4c8d..52543a11e8d 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java @@ -285,12 +285,18 @@ private void setCheckingAdtsHeaderState() { bytesRead = 0; } - /** Sets the state to STATE_READING_AAC_PCE. */ + /** + * Sets the state to STATE_READING_AAC_PCE. + * + * @param outputToUse TrackOutput object to write the sample to + * @param currentSampleDuration Duration of the sample to be read + * @param sampleSize Size of the sample + */ private void setReadingAacPceState( TrackOutput outputToUse, long currentSampleDuration, int sampleSize) { state = STATE_READING_AAC_PCE; bytesRead = 0; - currentOutput = outputToUse; + this.currentOutput = outputToUse; this.currentSampleDuration = currentSampleDuration; this.sampleSize = sampleSize; pceBuffer = new ParsableBitArray(new byte[min(sampleSize, AAC_PCE_MAX_SIZE)]); @@ -567,46 +573,52 @@ private void parseAdtsHeader() throws ParserException { @RequiresNonNull({"pendingOutputFormat", "pceBuffer"}) void readAacProgramConfigElement() { + // See ISO 13818-7 Advanced Audio Coding (2006) Table 36 for PCE tag encoding. if (pceBuffer.readBits(3) == 5 /* PCE tag */) { - // See `ISO 13818-7 Advanced Audio Coding (2006) Table 25` for format of a PCE - pceBuffer.skipBits(10); // Element instance tag, profile, sample frequency index + // See ISO 13818-7 Advanced Audio Coding (2006) Table 25 for syntax of a PCE. + pceBuffer.skipBits(10); // element_instance_tag(4), profile(2), element_instance_tag(4) int channelBits = 0; - channelBits += pceBuffer.readBits(4) * 5; // Front channel elements - channelBits += pceBuffer.readBits(4) * 5; // Side channel elements - channelBits += pceBuffer.readBits(4) * 5; // Back channel elements - channelBits += pceBuffer.readBits(2) * 4; // LFE channel elements - channelBits += pceBuffer.readBits(3) * 4; // Data elements - channelBits += pceBuffer.readBits(4) * 5; // Coupling channel elements - - if (pceBuffer.readBit()) { - pceBuffer.skipBits(4); // Mono mixdown + // num_front_channel_elements, front_element_is_cpe(1), front_element_tag_select(4) + channelBits += pceBuffer.readBits(4) * 5; + // num_side_channel_elements, side_element_is_cpe(1), side_element_tag_select(4) + channelBits += pceBuffer.readBits(4) * 5; + // num_back_channel_elements, back_element_is_cpe(1), back_element_tag_select(4) + channelBits += pceBuffer.readBits(4) * 5; + // num_lfe_channel_elements, lfe_element_tag_select(4) + channelBits += pceBuffer.readBits(2) * 4; + // num_assoc_data_elements, assoc_data_element_tag_select(4) + channelBits += pceBuffer.readBits(3) * 4; + // num_valid_cc_elements, cc_element_is_ind_sw(1), valid_cc_element_tag_select(4) + channelBits += pceBuffer.readBits(4) * 5; + + if (pceBuffer.readBit()) { // mono_mixdown_present + pceBuffer.skipBits(4); // mono_mixdown_element_number } - if (pceBuffer.readBit()) { - pceBuffer.skipBits(4); // Stereo mixdown + if (pceBuffer.readBit()) { // stereo_mixdown_present + pceBuffer.skipBits(4); // stereo_mixdown_element_number } - if (pceBuffer.readBit()) { - pceBuffer.skipBits(3); // Matrix mixdown + if (pceBuffer.readBit()) { // matrix_mixdown_idx_present + pceBuffer.skipBits(3); // matrix_mixdown_idx(2), matrix_mixdown_idx(1) } - // Beyond this point, pceBuffer may be empty, so check before consuming. + int numAlignmentBits = + 8 - (pceBuffer.getPosition() + channelBits + 7) % 8 - 1; // byte_alignment + int commentSizeBits = 8; // comment_field_bytes - int numAlignmentBits = 8 - (pceBuffer.getPosition() + channelBits + 7) % 8 - 1; - - if (pceBuffer.bitsLeft() >= channelBits + numAlignmentBits + 8) - { + // Beyond this point, pceBuffer may be empty, so check before consuming. + if (pceBuffer.bitsLeft() >= channelBits + numAlignmentBits + commentSizeBits) { pceBuffer.skipBits(channelBits); // Store PCE size excluding initial PCE tag, alignment bits and comment for later. int numPceBits = pceBuffer.getPosition() - 3 /* PCE tag */; pceBuffer.skipBits(numAlignmentBits); - int commentSize = pceBuffer.readBits(8); + int commentSize = pceBuffer.readBits(commentSizeBits); - if (sampleSize >= pceBuffer.getBytePosition() + commentSize) - { + if (sampleSize >= pceBuffer.getBytePosition() + commentSize) { // Append PCE to format's audio specific config. byte[] oldConfig = pendingOutputFormat.initializationData.get(0); diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java index 14752fd40ca..617236523d4 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java @@ -65,8 +65,7 @@ public class AdtsReaderTest { TestUtil.createByteArray(0xff, 0xf1, 0x50, 0x00, 0x02, 0x1f, 0xfc); private static final byte[] AAC_PCE_ADTS_CONTENT = - TestUtil.createByteArray( - 0xa0, 0x99, 0x01, 0x20, 0x00, 0x21, 0x19, 0x00, 0x00); + TestUtil.createByteArray(0xa0, 0x99, 0x01, 0x20, 0x00, 0x21, 0x19, 0x00, 0x00); private static final byte[] AAC_PCE_TEST_DATA = Bytes.concat(AAC_PCE_ADTS_HEADER, AAC_PCE_ADTS_CONTENT); @@ -202,16 +201,17 @@ public void adtsDataOnly() throws ParserException { @Test public void aacPceData() throws ParserException { data = new ParsableByteArray(AAC_PCE_TEST_DATA); + feed(); + assertSampleCounts(0, 1); adtsOutput.assertSample(0, AAC_PCE_ADTS_CONTENT, 0, C.BUFFER_FLAG_KEY_FRAME, null); } @Test(expected = IllegalStateException.class) public void aacPceDataFail() throws ParserException { - data = new ParsableByteArray(AAC_PCE_TEST_DATA); + data = new ParsableByteArray(Arrays.copyOf(AAC_PCE_TEST_DATA, AAC_PCE_TEST_DATA.length)); byte[] bytes = data.getData(); - // Remove PCE tag (first 3 bits of content). bytes[AAC_PCE_ADTS_HEADER.length] &= 0x1f; // Replace with CPE tag. From ff515d32fe0272401bb65ebfc7e0684386dc579b Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Mon, 27 Nov 2023 16:45:46 +0000 Subject: [PATCH 6/8] Add release note --- RELEASENOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d7df013784c..ca29c265a51 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -44,6 +44,9 @@ `ColorInfo.colorSpace`, `ColorInfo.colorTransfer`, and `ColorInfo.colorRange` values ([#692](https://github.com/androidx/media/pull/692)). + * Fix playback of AAC in TS files when channel config is `0` by reading + Program Config Element (PCE) + ([#722](https://github.com/androidx/media/pull/722)). * Audio: * Video: * Add workaround for a device issue on Galaxy Tab S7 FE, Chromecast with From 2c3eac495ba7500d17b586f9ab032d2c1df01521 Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Tue, 28 Nov 2023 16:58:04 +0000 Subject: [PATCH 7/8] Add nullness check --- .../media3/extractor/ts/AdtsReader.java | 22 +++++++++++-------- .../media3/extractor/ts/AdtsReaderTest.java | 5 +++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java index 52543a11e8d..52db924cb60 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java @@ -15,6 +15,7 @@ */ package androidx.media3.extractor.ts; +import static androidx.media3.common.util.Assertions.checkNotNull; import static java.lang.Math.min; import androidx.annotation.Nullable; @@ -22,7 +23,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.ParserException; -import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Log; import androidx.media3.common.util.ParsableBitArray; import androidx.media3.common.util.ParsableByteArray; @@ -33,6 +33,7 @@ import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.TrackOutput; import androidx.media3.extractor.ts.TsPayloadReader.TrackIdGenerator; +import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.Collections; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; @@ -191,6 +192,7 @@ public void consume(ParsableByteArray data) throws ParserException { } break; case STATE_READING_AAC_PCE: + checkNotNull(pceBuffer); if (continueRead(data, pceBuffer.data, pceBuffer.data.length)) { readAacProgramConfigElement(); } @@ -571,8 +573,11 @@ private void parseAdtsHeader() throws ParserException { } } - @RequiresNonNull({"pendingOutputFormat", "pceBuffer"}) + @RequiresNonNull("currentOutput") void readAacProgramConfigElement() { + Format pendingOutputFormat = checkNotNull(this.pendingOutputFormat); + ParsableBitArray pceBuffer = checkNotNull(this.pceBuffer); + // See ISO 13818-7 Advanced Audio Coding (2006) Table 36 for PCE tag encoding. if (pceBuffer.readBits(3) == 5 /* PCE tag */) { // See ISO 13818-7 Advanced Audio Coding (2006) Table 25 for syntax of a PCE. @@ -624,16 +629,15 @@ void readAacProgramConfigElement() { int configSize = oldConfig.length; configSize += (numPceBits + 7) / 8 + 1; // Byte align and add a zero length comment. - byte[] newConfig = new byte[configSize]; + byte[] newConfig = Arrays.copyOf(oldConfig, configSize); - System.arraycopy(oldConfig, 0, newConfig, 0, oldConfig.length); pceBuffer.setPosition(3 /* PCE tag */); pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); pendingOutputFormat = pendingOutputFormat .buildUpon() - .setInitializationData(Collections.singletonList(newConfig)) + .setInitializationData(ImmutableList.of(newConfig)) .build(); // Submit PCE-appended output format. @@ -643,13 +647,13 @@ void readAacProgramConfigElement() { } } - pendingOutputFormat = null; - // Pass through all accumulated data as sample data. ParsableByteArray data = new ParsableByteArray(pceBuffer.data); - pceBuffer = null; setReadingSampleState(currentOutput, currentSampleDuration, 0, sampleSize); readSample(data); + + this.pendingOutputFormat = null; + this.pceBuffer = null; } /** Reads the rest of the sample */ @@ -669,7 +673,7 @@ private void readSample(ParsableByteArray data) { @EnsuresNonNull({"output", "currentOutput", "id3Output"}) private void assertTracksCreated() { - Assertions.checkNotNull(output); + checkNotNull(output); Util.castNonNull(currentOutput); Util.castNonNull(id3Output); } diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java index 617236523d4..09ec2684a4c 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java @@ -17,6 +17,7 @@ import static androidx.media3.extractor.ts.TsPayloadReader.FLAG_DATA_ALIGNMENT_INDICATOR; import static java.lang.Math.min; +import static org.junit.Assert.assertThrows; import androidx.media3.common.C; import androidx.media3.common.ParserException; @@ -208,7 +209,7 @@ public void aacPceData() throws ParserException { adtsOutput.assertSample(0, AAC_PCE_ADTS_CONTENT, 0, C.BUFFER_FLAG_KEY_FRAME, null); } - @Test(expected = IllegalStateException.class) + @Test public void aacPceDataFail() throws ParserException { data = new ParsableByteArray(Arrays.copyOf(AAC_PCE_TEST_DATA, AAC_PCE_TEST_DATA.length)); byte[] bytes = data.getData(); @@ -218,7 +219,7 @@ public void aacPceDataFail() throws ParserException { bytes[AAC_PCE_ADTS_HEADER.length] |= 0x20; // Should throw as FakeTrackOutput expects a format before sampleMetadata. - feed(); + assertThrows(IllegalStateException.class, this::feed); } private void feedLimited(int limit) throws ParserException { From 1d573e01e22079d8656254b015aaee584b41a668 Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Wed, 29 Nov 2023 15:48:09 +0000 Subject: [PATCH 8/8] Changes based on internal review --- .../media3/extractor/ts/AdtsReader.java | 54 ++++++++++--------- .../media3/extractor/ts/AdtsReaderTest.java | 16 ++++++ 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java index 52db924cb60..d926bdedca0 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/AdtsReader.java @@ -574,8 +574,7 @@ private void parseAdtsHeader() throws ParserException { } @RequiresNonNull("currentOutput") - void readAacProgramConfigElement() { - Format pendingOutputFormat = checkNotNull(this.pendingOutputFormat); + private void readAacProgramConfigElement() throws ParserException { ParsableBitArray pceBuffer = checkNotNull(this.pceBuffer); // See ISO 13818-7 Advanced Audio Coding (2006) Table 36 for PCE tag encoding. @@ -614,37 +613,40 @@ void readAacProgramConfigElement() { int commentSizeBits = 8; // comment_field_bytes // Beyond this point, pceBuffer may be empty, so check before consuming. - if (pceBuffer.bitsLeft() >= channelBits + numAlignmentBits + commentSizeBits) { - pceBuffer.skipBits(channelBits); + if (pceBuffer.bitsLeft() < channelBits + numAlignmentBits + commentSizeBits) { + throw ParserException.createForMalformedContainer(/* message= */ null, /* cause= */ null); + } - // Store PCE size excluding initial PCE tag, alignment bits and comment for later. - int numPceBits = pceBuffer.getPosition() - 3 /* PCE tag */; + pceBuffer.skipBits(channelBits); - pceBuffer.skipBits(numAlignmentBits); - int commentSize = pceBuffer.readBits(commentSizeBits); + // Store PCE size excluding initial PCE tag, alignment bits and comment for later. + int numPceBits = pceBuffer.getPosition() - 3 /* PCE tag */; + pceBuffer.skipBits(numAlignmentBits); + int commentSize = pceBuffer.readBits(commentSizeBits); - if (sampleSize >= pceBuffer.getBytePosition() + commentSize) { - // Append PCE to format's audio specific config. - byte[] oldConfig = pendingOutputFormat.initializationData.get(0); + if (sampleSize < pceBuffer.getBytePosition() + commentSize) { + throw ParserException.createForMalformedContainer(/* message= */ null, /* cause= */ null); + } - int configSize = oldConfig.length; - configSize += (numPceBits + 7) / 8 + 1; // Byte align and add a zero length comment. - byte[] newConfig = Arrays.copyOf(oldConfig, configSize); + Format pendingOutputFormat = checkNotNull(this.pendingOutputFormat); + // Append PCE to format's audio specific config. + byte[] oldConfig = pendingOutputFormat.initializationData.get(0); + int configSize = oldConfig.length; + configSize += (numPceBits + 7) / 8 + 1; // Byte align and add a zero length comment. + byte[] newConfig = Arrays.copyOf(oldConfig, configSize); - pceBuffer.setPosition(3 /* PCE tag */); - pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); + pceBuffer.setPosition(3 /* PCE tag */); + pceBuffer.readBits(newConfig, oldConfig.length, numPceBits); - pendingOutputFormat = - pendingOutputFormat - .buildUpon() - .setInitializationData(ImmutableList.of(newConfig)) - .build(); + pendingOutputFormat = + pendingOutputFormat + .buildUpon() + .setInitializationData(ImmutableList.of(newConfig)) + .build(); - // Submit PCE-appended output format. - currentOutput.format(pendingOutputFormat); - hasOutputFormat = true; - } - } + // Submit PCE-appended output format. + this.currentOutput.format(pendingOutputFormat); + this.hasOutputFormat = true; } // Pass through all accumulated data as sample data. diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java index 09ec2684a4c..3cc444b745f 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/ts/AdtsReaderTest.java @@ -209,6 +209,22 @@ public void aacPceData() throws ParserException { adtsOutput.assertSample(0, AAC_PCE_ADTS_CONTENT, 0, C.BUFFER_FLAG_KEY_FRAME, null); } + @Test + public void aacPceDataSplit() throws ParserException { + byte[] first = Arrays.copyOf(AAC_PCE_TEST_DATA, AAC_PCE_ADTS_HEADER.length + 1); + byte[] second = + Arrays.copyOfRange( + AAC_PCE_TEST_DATA, AAC_PCE_ADTS_HEADER.length + 1, AAC_PCE_TEST_DATA.length); + + data = new ParsableByteArray(first); + feed(); + data = new ParsableByteArray(second); + feed(); + + assertSampleCounts(0, 1); + adtsOutput.assertSample(0, AAC_PCE_ADTS_CONTENT, 0, C.BUFFER_FLAG_KEY_FRAME, null); + } + @Test public void aacPceDataFail() throws ParserException { data = new ParsableByteArray(Arrays.copyOf(AAC_PCE_TEST_DATA, AAC_PCE_TEST_DATA.length));