Skip to content

Commit 2268fb3

Browse files
carlossanlopcarlossanlop
authored
Tar: Fix PAX regression when handling the size of really long unseekable data streams (#88280)
* Fix regression introduced by #84279 preventing PAX entries with really long data streams to get its size correctly stored in the extended attributes when the data stream is unseekable. * Move tests for large files to a new manual tests project.
1 parent bf78b40 commit 2268fb3

9 files changed

+450
-303
lines changed

‎src/libraries/System.Formats.Tar/System.Formats.Tar.sln

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Formats.Tar", "src\S
99
EndProject
1010
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Formats.Tar.Tests", "tests\System.Formats.Tar.Tests.csproj", "{6FD1E284-7B50-4077-B73A-5B31CB0E3577}"
1111
EndProject
12+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Formats.Tar.Manual.Tests", "tests\Manual\System.Formats.Tar.Manual.Tests.csproj", "{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}"
13+
EndProject
1214
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ComInterfaceGenerator", "..\System.Runtime.InteropServices\gen\ComInterfaceGenerator\ComInterfaceGenerator.csproj", "{00477EA4-C3E5-48A9-8CA8-8CCF689E0DB4}"
1315
EndProject
1416
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LibraryImportGenerator", "..\System.Runtime.InteropServices\gen\LibraryImportGenerator\LibraryImportGenerator.csproj", "{E89FEF3E-E0B9-41C4-A51C-9759AD1A3B69}"
@@ -67,6 +69,10 @@ Global
6769
{A00011A0-E609-4A49-B893-EBFC72C98707}.Debug|Any CPU.Build.0 = Debug|Any CPU
6870
{A00011A0-E609-4A49-B893-EBFC72C98707}.Release|Any CPU.ActiveCfg = Release|Any CPU
6971
{A00011A0-E609-4A49-B893-EBFC72C98707}.Release|Any CPU.Build.0 = Release|Any CPU
72+
{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
73+
{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Debug|Any CPU.Build.0 = Debug|Any CPU
74+
{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Release|Any CPU.ActiveCfg = Release|Any CPU
75+
{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D}.Release|Any CPU.Build.0 = Release|Any CPU
7076
EndGlobalSection
7177
GlobalSection(SolutionProperties) = preSolution
7278
HideSolutionNode = FALSE
@@ -78,9 +84,12 @@ Global
7884
{E0B882C6-2082-45F2-806E-568461A61975} = {9BE8AFF4-D37B-49AF-AFD3-A15E514AC8AE}
7985
{A00011A0-E609-4A49-B893-EBFC72C98707} = {9BE8AFF4-D37B-49AF-AFD3-A15E514AC8AE}
8086
{9F751C2B-56DD-4604-A3F3-568627F8C006} = {55A8C7E4-925C-4F21-B68B-CEFC19137A4B}
87+
{6FD1E284-7B50-4077-B73A-5B31CB0E3577} = {6CF0D830-3EE9-44B1-B548-EA8750AD7B3E}
8188
{00477EA4-C3E5-48A9-8CA8-8CCF689E0DB4} = {0345BAA8-92BC-4499-B550-21AC44910FD2}
8289
{E89FEF3E-E0B9-41C4-A51C-9759AD1A3B69} = {0345BAA8-92BC-4499-B550-21AC44910FD2}
8390
{50E6D5FD-0E06-4D07-966E-C28E5448A1D3} = {0345BAA8-92BC-4499-B550-21AC44910FD2}
91+
{A00011A0-E609-4A49-B893-EBFC72C98707} = {9BE8AFF4-D37B-49AF-AFD3-A15E514AC8AE}
92+
{D2788A26-CDAE-4388-AE4B-A36B0E6DFF9D} = {6CF0D830-3EE9-44B1-B548-EA8750AD7B3E}
8493
EndGlobalSection
8594
GlobalSection(ExtensibilityGlobals) = postSolution
8695
SolutionGuid = {F9B8DA67-C83B-466D-907C-9541CDBDCFEF}

‎src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs

Lines changed: 243 additions & 123 deletions
Large diffs are not rendered by default.

‎src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarWriter.cs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -283,8 +283,12 @@ private void WriteEntryInternal(TarEntry entry)
283283

284284
switch (entry.Format)
285285
{
286-
case TarEntryFormat.V7 or TarEntryFormat.Ustar:
287-
entry._header.WriteAs(entry.Format, _archiveStream, buffer);
286+
case TarEntryFormat.V7:
287+
entry._header.WriteAsV7(_archiveStream, buffer);
288+
break;
289+
290+
case TarEntryFormat.Ustar:
291+
entry._header.WriteAsUstar(_archiveStream, buffer);
288292
break;
289293

290294
case TarEntryFormat.Pax:
@@ -321,7 +325,8 @@ private async Task WriteEntryAsyncInternal(TarEntry entry, CancellationToken can
321325

322326
Task task = entry.Format switch
323327
{
324-
TarEntryFormat.V7 or TarEntryFormat.Ustar => entry._header.WriteAsAsync(entry.Format, _archiveStream, buffer, cancellationToken),
328+
TarEntryFormat.V7 => entry._header.WriteAsV7Async(_archiveStream, buffer, cancellationToken),
329+
TarEntryFormat.Ustar => entry._header.WriteAsUstarAsync(_archiveStream, buffer, cancellationToken),
325330
TarEntryFormat.Pax when entry._header._typeFlag is TarEntryType.GlobalExtendedAttributes => entry._header.WriteAsPaxGlobalExtendedAttributesAsync(_archiveStream, buffer, _nextGlobalExtendedAttributesEntryNumber++, cancellationToken),
326331
TarEntryFormat.Pax => entry._header.WriteAsPaxAsync(_archiveStream, buffer, cancellationToken),
327332
TarEntryFormat.Gnu => entry._header.WriteAsGnuAsync(_archiveStream, buffer, cancellationToken),

‎src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using Xunit;
7+
8+
namespace System.Formats.Tar.Tests;
9+
10+
[OuterLoop]
11+
[Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time
12+
public class ManualTests : TarTestsBase
13+
{
14+
public static bool ManualTestsEnabled => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MANUAL_TESTS"));
15+
16+
public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryData()
17+
{
18+
foreach (bool unseekableStream in new[] { false, true })
19+
{
20+
foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax })
21+
{
22+
yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream };
23+
}
24+
25+
// Pax supports unlimited size files.
26+
yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream };
27+
}
28+
}
29+
30+
[ConditionalTheory(nameof(ManualTestsEnabled))]
31+
[MemberData(nameof(WriteEntry_LongFileSize_TheoryData))]
32+
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
33+
public void WriteEntry_LongFileSize(TarEntryFormat entryFormat, long size, bool unseekableStream)
34+
{
35+
// Write archive with a 8 Gb long entry.
36+
using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose });
37+
Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile;
38+
39+
using (TarWriter writer = new(s, leaveOpen: true))
40+
{
41+
TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo");
42+
writeEntry.DataStream = new SimulatedDataStream(size);
43+
writer.WriteEntry(writeEntry);
44+
}
45+
46+
tarFile.Position = 0;
47+
48+
// Read archive back.
49+
using TarReader reader = new TarReader(s);
50+
TarEntry entry = reader.GetNextEntry();
51+
Assert.Equal(size, entry.Length);
52+
53+
Stream dataStream = entry.DataStream;
54+
Assert.Equal(size, dataStream.Length);
55+
Assert.Equal(0, dataStream.Position);
56+
57+
ReadOnlySpan<byte> dummyData = SimulatedDataStream.DummyData.Span;
58+
59+
// Read the first bytes.
60+
Span<byte> buffer = new byte[dummyData.Length];
61+
Assert.Equal(buffer.Length, dataStream.Read(buffer));
62+
AssertExtensions.SequenceEqual(dummyData, buffer);
63+
Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct.
64+
buffer.Clear();
65+
66+
// Read the last bytes.
67+
long dummyDataOffset = size - dummyData.Length - 1;
68+
if (dataStream.CanSeek)
69+
{
70+
Assert.False(unseekableStream);
71+
dataStream.Seek(dummyDataOffset, SeekOrigin.Begin);
72+
}
73+
else
74+
{
75+
Assert.True(unseekableStream);
76+
Span<byte> seekBuffer = new byte[4_096];
77+
78+
while (dataStream.Position < dummyDataOffset)
79+
{
80+
int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position);
81+
int res = dataStream.Read(seekBuffer.Slice(0, bufSize));
82+
Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong");
83+
}
84+
}
85+
86+
Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct.
87+
Assert.Equal(buffer.Length, dataStream.Read(buffer));
88+
AssertExtensions.SequenceEqual(dummyData, buffer);
89+
Assert.Equal(size, dataStream.Position);
90+
91+
Assert.Null(reader.GetNextEntry());
92+
}
93+
}

‎src/libraries/System.Formats.Tar/tests/Manual/ManualTestsAsync.cs

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.IO;
6+
using System.Threading.Tasks;
7+
using Xunit;
8+
9+
namespace System.Formats.Tar.Tests;
10+
11+
[OuterLoop]
12+
[Collection(nameof(DisableParallelization))] // don't create multiple large files at the same time
13+
public class ManualTestsAsync : TarTestsBase
14+
{
15+
public static IEnumerable<object[]> WriteEntry_LongFileSize_TheoryDataAsync()
16+
// Fixes error xUnit1015: MemberData needs to be in the same class
17+
=> ManualTests.WriteEntry_LongFileSize_TheoryData();
18+
19+
[ConditionalTheory(nameof(ManualTests.ManualTestsEnabled))]
20+
[MemberData(nameof(WriteEntry_LongFileSize_TheoryDataAsync))]
21+
[SkipOnPlatform(TestPlatforms.iOS | TestPlatforms.tvOS | TestPlatforms.Android | TestPlatforms.Browser, "Needs too much disk space.")]
22+
public async Task WriteEntry_LongFileSizeAsync(TarEntryFormat entryFormat, long size, bool unseekableStream)
23+
{
24+
// Write archive with a 8 Gb long entry.
25+
await using FileStream tarFile = File.Open(GetTestFilePath(), new FileStreamOptions { Access = FileAccess.ReadWrite, Mode = FileMode.Create, Options = FileOptions.DeleteOnClose });
26+
Stream s = unseekableStream ? new WrappedStream(tarFile, tarFile.CanRead, tarFile.CanWrite, canSeek: false) : tarFile;
27+
28+
await using (TarWriter writer = new(s, leaveOpen: true))
29+
{
30+
TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo");
31+
writeEntry.DataStream = new SimulatedDataStream(size);
32+
await writer.WriteEntryAsync(writeEntry);
33+
}
34+
35+
tarFile.Position = 0;
36+
37+
// Read the archive back.
38+
await using TarReader reader = new TarReader(s);
39+
TarEntry entry = await reader.GetNextEntryAsync();
40+
Assert.Equal(size, entry.Length);
41+
42+
Stream dataStream = entry.DataStream;
43+
Assert.Equal(size, dataStream.Length);
44+
Assert.Equal(0, dataStream.Position);
45+
46+
ReadOnlyMemory<byte> dummyData = SimulatedDataStream.DummyData;
47+
48+
// Read the first bytes.
49+
byte[] buffer = new byte[dummyData.Length];
50+
Assert.Equal(buffer.Length, dataStream.Read(buffer));
51+
AssertExtensions.SequenceEqual(dummyData.Span, buffer);
52+
Assert.Equal(0, dataStream.ReadByte()); // check next byte is correct.
53+
buffer.AsSpan().Clear();
54+
55+
// Read the last bytes.
56+
long dummyDataOffset = size - dummyData.Length - 1;
57+
if (dataStream.CanSeek)
58+
{
59+
Assert.False(unseekableStream);
60+
dataStream.Seek(dummyDataOffset, SeekOrigin.Begin);
61+
}
62+
else
63+
{
64+
Assert.True(unseekableStream);
65+
Memory<byte> seekBuffer = new byte[4_096];
66+
67+
while (dataStream.Position < dummyDataOffset)
68+
{
69+
int bufSize = (int)Math.Min(seekBuffer.Length, dummyDataOffset - dataStream.Position);
70+
int res = await dataStream.ReadAsync(seekBuffer.Slice(0, bufSize));
71+
Assert.True(res > 0, "Unseekable stream finished before expected - Something went very wrong");
72+
}
73+
}
74+
75+
Assert.Equal(0, dataStream.ReadByte()); // check previous byte is correct.
76+
Assert.Equal(buffer.Length, dataStream.Read(buffer));
77+
AssertExtensions.SequenceEqual(dummyData.Span, buffer);
78+
Assert.Equal(size, dataStream.Position);
79+
80+
Assert.Null(await reader.GetNextEntryAsync());
81+
}
82+
}

‎src/libraries/System.Formats.Tar/tests/Manual/System.Formats.Tar.Manual.Tests.csproj

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
4+
<IncludeRemoteExecutor>true</IncludeRemoteExecutor>
5+
</PropertyGroup>
6+
<ItemGroup>
7+
<Compile Include="ManualTests.cs" />
8+
<Compile Include="ManualTestsAsync.cs" />
9+
<Compile Include="..\TarTestsBase.cs" />
10+
<Compile Include="..\SimulatedDataStream.cs" />
11+
<Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" />
12+
<Compile Include="$(CommonTestPath)System\IO\TempDirectory.cs" Link="Common\System\IO\TempDirectory.cs" />
13+
<Compile Include="$(CommonTestPath)System\IO\WrappedStream.cs" Link="Common\System\IO\WrappedStream.cs" />
14+
</ItemGroup>
15+
</Project>

‎src/libraries/System.Formats.Tar/tests/System.Formats.Tar.Tests.csproj

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,8 @@
5353
<Compile Include="TarTestsBase.Ustar.cs" />
5454
<Compile Include="TarTestsBase.V7.cs" />
5555
<Compile Include="TarWriter\TarWriter.WriteEntry.Entry.Roundtrip.Tests.cs" />
56-
<Compile Include="TarWriter\TarWriter.WriteEntry.LongFile.Tests.cs" />
5756
<Compile Include="TarWriter\TarWriter.WriteEntryAsync.File.Tests.cs" />
5857
<Compile Include="TarWriter\TarWriter.WriteEntry.Base.cs" />
59-
<Compile Include="TarWriter\TarWriter.WriteEntryAsync.LongFile.Tests.cs" />
6058
<Compile Include="TarWriter\TarWriter.WriteEntryAsync.Tests.cs" />
6159
<Compile Include="TarWriter\TarWriter.WriteEntryAsync.Entry.Roundtrip.Tests.cs" />
6260
<Compile Include="TarWriter\TarWriter.WriteEntryAsync.Entry.Ustar.Tests.cs" />
@@ -74,7 +72,6 @@
7472
<Compile Include="$(CommonPath)DisableRuntimeMarshalling.cs" Link="Common\DisableRuntimeMarshalling.cs" />
7573
<Compile Include="$(CommonTestPath)System\IO\ReparsePointUtilities.cs" Link="Common\System\IO\ReparsePointUtilities.cs" />
7674
<Compile Include="$(CommonTestPath)System\IO\WrappedStream.cs" Link="Common\System\IO\WrappedStream.cs" />
77-
<Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" />
7875
</ItemGroup>
7976
<!-- Windows specific files -->
8077
<ItemGroup Condition="'$(TargetPlatformIdentifier)' == 'windows'">

‎src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.LongFile.Tests.cs

Lines changed: 0 additions & 92 deletions
This file was deleted.

0 commit comments

Comments
 (0)