Skip to content

Commit

Permalink
Use Steam manifest format
Browse files Browse the repository at this point in the history
  • Loading branch information
@NicknineTheEagle
NicknineTheEagle committed Oct 17, 2024
1 parent a5bdf23 commit 02e323a
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 221 deletions.
108 changes: 51 additions & 57 deletions DepotDownloader/ContentDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -590,20 +590,20 @@ static async Task<DepotDownloadInfo> GetDepotInfo(uint depotId, uint appId, ulon
return new DepotDownloadInfo(depotId, appId, manifestId, branch, installDir, depotKey);
}

private class ChunkMatch(ProtoManifest.ChunkData oldChunk, ProtoManifest.ChunkData newChunk)
private class ChunkMatch(DepotManifest.ChunkData oldChunk, DepotManifest.ChunkData newChunk)
{
public ProtoManifest.ChunkData OldChunk { get; } = oldChunk;
public ProtoManifest.ChunkData NewChunk { get; } = newChunk;
public DepotManifest.ChunkData OldChunk { get; } = oldChunk;
public DepotManifest.ChunkData NewChunk { get; } = newChunk;
}

private class DepotFilesData
{
public DepotDownloadInfo depotDownloadInfo;
public DepotDownloadCounter depotCounter;
public string stagingDir;
public ProtoManifest manifest;
public ProtoManifest previousManifest;
public List<ProtoManifest.FileData> filteredFiles;
public DepotManifest manifest;
public DepotManifest previousManifest;
public List<DepotManifest.FileData> filteredFiles;
public HashSet<string> allFileNames;
}

Expand Down Expand Up @@ -686,8 +686,8 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat

Console.WriteLine("Processing depot {0}", depot.DepotId);

ProtoManifest oldProtoManifest = null;
ProtoManifest newProtoManifest = null;
DepotManifest oldManifest = null;
DepotManifest newManifest = null;
var configDir = Path.Combine(depot.InstallDir, CONFIG_DIR);

var lastManifestId = INVALID_MANIFEST_ID;
Expand All @@ -699,7 +699,7 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat

if (lastManifestId != INVALID_MANIFEST_ID)
{
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, lastManifestId));
var oldManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.manifest", depot.DepotId, lastManifestId));

if (File.Exists(oldManifestFileName))
{
Expand All @@ -714,27 +714,30 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
expectedChecksum = null;
}

oldProtoManifest = ProtoManifest.LoadFromFile(oldManifestFileName, out var currentChecksum);
var currentChecksum = Util.FileSHAHash(oldManifestFileName);

if (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum))
if (expectedChecksum != null && expectedChecksum.SequenceEqual(currentChecksum))
{
oldManifest = DepotManifest.LoadFromFile(oldManifestFileName);
}
else
{
// We only have to show this warning if the old manifest ID was different
if (lastManifestId != depot.ManifestId)
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", lastManifestId);
oldProtoManifest = null;
}
}
}

if (lastManifestId == depot.ManifestId && oldProtoManifest != null)
if (lastManifestId == depot.ManifestId && oldManifest != null)
{
newProtoManifest = oldProtoManifest;
newManifest = oldManifest;
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.bin", depot.DepotId, depot.ManifestId));
if (newManifestFileName != null)
var newManifestFileName = Path.Combine(configDir, string.Format("{0}_{1}.manifest", depot.DepotId, depot.ManifestId));
if (File.Exists(newManifestFileName))
{
byte[] expectedChecksum;

Expand All @@ -747,24 +750,26 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
expectedChecksum = null;
}

newProtoManifest = ProtoManifest.LoadFromFile(newManifestFileName, out var currentChecksum);
var currentChecksum = Util.FileSHAHash(newManifestFileName);

if (newProtoManifest != null && (expectedChecksum == null || !expectedChecksum.SequenceEqual(currentChecksum)))
if (expectedChecksum != null && expectedChecksum.SequenceEqual(currentChecksum))
{
newManifest = DepotManifest.LoadFromFile(newManifestFileName);
}
else
{
Console.WriteLine("Manifest {0} on disk did not match the expected checksum.", depot.ManifestId);
newProtoManifest = null;
}
}

if (newProtoManifest != null)
if (newManifest != null)
{
Console.WriteLine("Already have manifest {0} for depot {1}.", depot.ManifestId, depot.DepotId);
}
else
{
Console.Write("Downloading depot manifest... ");

DepotManifest depotManifest = null;
ulong manifestRequestCode = 0;
var manifestRequestCodeExpiration = DateTime.MinValue;

Expand Down Expand Up @@ -812,7 +817,7 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
depot.ManifestId,
connection,
cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
depotManifest = await cdnPool.CDNClient.DownloadManifestAsync(
newManifest = await cdnPool.CDNClient.DownloadManifestAsync(
depot.DepotId,
depot.ManifestId,
manifestRequestCode,
Expand Down Expand Up @@ -864,9 +869,9 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
cdnPool.ReturnBrokenConnection(connection);
Console.WriteLine("Encountered error downloading manifest for depot {0} {1}: {2}", depot.DepotId, depot.ManifestId, e.Message);
}
} while (depotManifest == null);
} while (newManifest == null);

if (depotManifest == null)
if (newManifest == null)
{
Console.WriteLine("\nUnable to download manifest {0} for depot {1}", depot.ManifestId, depot.DepotId);
cts.Cancel();
Expand All @@ -875,28 +880,26 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
// Throw the cancellation exception if requested so that this task is marked failed
cts.Token.ThrowIfCancellationRequested();


newProtoManifest = new ProtoManifest(depotManifest, depot.ManifestId);
newProtoManifest.SaveToFile(newManifestFileName, out var checksum);
File.WriteAllBytes(newManifestFileName + ".sha", checksum);
if (Util.SaveManifestToFile(newManifest, newManifestFileName))
{
File.WriteAllBytes(newManifestFileName + ".sha", Util.FileSHAHash(newManifestFileName));
}

Console.WriteLine(" Done!");
}
}

newProtoManifest.Files.Sort((x, y) => string.Compare(x.FileName, y.FileName, StringComparison.Ordinal));

Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newProtoManifest.CreationTime);
Console.WriteLine("Manifest {0} ({1})", depot.ManifestId, newManifest.CreationTime);

if (Config.DownloadManifestOnly)
{
DumpManifestToTextFile(depot, newProtoManifest);
DumpManifestToTextFile(depot, newManifest);
return null;
}

var stagingDir = Path.Combine(depot.InstallDir, STAGING_DIR);

var filesAfterExclusions = newProtoManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var filesAfterExclusions = newManifest.Files.AsParallel().Where(f => TestIsFileIncluded(f.FileName)).ToList();
var allFileNames = new HashSet<string>(filesAfterExclusions.Count);

// Pre-process
Expand Down Expand Up @@ -928,8 +931,8 @@ private static async Task<DepotFilesData> ProcessDepotManifestAndFiles(Cancellat
depotDownloadInfo = depot,
depotCounter = depotCounter,
stagingDir = stagingDir,
manifest = newProtoManifest,
previousManifest = oldProtoManifest,
manifest = newManifest,
previousManifest = oldManifest,
filteredFiles = filesAfterExclusions,
allFileNames = allFileNames
};
Expand All @@ -944,7 +947,7 @@ private static async Task DownloadSteam3AsyncDepotFiles(CancellationTokenSource
Console.WriteLine("Downloading depot {0}", depot.DepotId);

var files = depotFilesData.filteredFiles.Where(f => !f.Flags.HasFlag(EDepotFileFlag.Directory)).ToArray();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, ProtoManifest.FileData fileData, ProtoManifest.ChunkData chunk)>();
var networkChunkQueue = new ConcurrentQueue<(FileStreamData fileStreamData, DepotManifest.FileData fileData, DepotManifest.ChunkData chunk)>();

await Util.InvokeAsync(
files.Select(file => new Func<Task>(async () =>
Expand Down Expand Up @@ -998,16 +1001,16 @@ private static void DownloadSteam3AsyncDepotFile(
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
ConcurrentQueue<(FileStreamData, ProtoManifest.FileData, ProtoManifest.ChunkData)> networkChunkQueue)
DepotManifest.FileData file,
ConcurrentQueue<(FileStreamData, DepotManifest.FileData, DepotManifest.ChunkData)> networkChunkQueue)
{
cts.Token.ThrowIfCancellationRequested();

var depot = depotFilesData.depotDownloadInfo;
var stagingDir = depotFilesData.stagingDir;
var depotDownloadCounter = depotFilesData.depotCounter;
var oldProtoManifest = depotFilesData.previousManifest;
ProtoManifest.FileData oldManifestFile = null;
DepotManifest.FileData oldManifestFile = null;
if (oldProtoManifest != null)
{
oldManifestFile = oldProtoManifest.Files.SingleOrDefault(f => f.FileName == file.FileName);
Expand All @@ -1022,7 +1025,7 @@ private static void DownloadSteam3AsyncDepotFile(
File.Delete(fileStagingPath);
}

List<ProtoManifest.ChunkData> neededChunks;
List<DepotManifest.ChunkData> neededChunks;
var fi = new FileInfo(fileFinalPath);
var fileDidExist = fi.Exists;
if (!fileDidExist)
Expand All @@ -1040,7 +1043,7 @@ private static void DownloadSteam3AsyncDepotFile(
throw new ContentDownloaderException(string.Format("Failed to allocate file {0}: {1}", fileFinalPath, ex.Message));
}

neededChunks = new List<ProtoManifest.ChunkData>(file.Chunks);
neededChunks = new List<DepotManifest.ChunkData>(file.Chunks);
}
else
{
Expand Down Expand Up @@ -1084,7 +1087,7 @@ private static void DownloadSteam3AsyncDepotFile(
fsOld.Seek((long)match.OldChunk.Offset, SeekOrigin.Begin);

var adler = Util.AdlerHash(fsOld, (int)match.OldChunk.UncompressedLength);
if (!adler.SequenceEqual(match.OldChunk.Checksum))
if (!adler.SequenceEqual(BitConverter.GetBytes(match.OldChunk.Checksum)))
{
neededChunks.Add(match.NewChunk);
}
Expand Down Expand Up @@ -1203,9 +1206,9 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
CancellationTokenSource cts,
GlobalDownloadCounter downloadCounter,
DepotFilesData depotFilesData,
ProtoManifest.FileData file,
DepotManifest.FileData file,
FileStreamData fileStreamData,
ProtoManifest.ChunkData chunk)
DepotManifest.ChunkData chunk)
{
cts.Token.ThrowIfCancellationRequested();

Expand All @@ -1214,17 +1217,8 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(

var chunkID = Convert.ToHexString(chunk.ChunkID).ToLowerInvariant();

var data = new DepotManifest.ChunkData
{
ChunkID = chunk.ChunkID,
Checksum = BitConverter.ToUInt32(chunk.Checksum),
Offset = chunk.Offset,
CompressedLength = chunk.CompressedLength,
UncompressedLength = chunk.UncompressedLength
};

var written = 0;
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)data.UncompressedLength);
var chunkBuffer = ArrayPool<byte>.Shared.Rent((int)chunk.UncompressedLength);

try
{
Expand All @@ -1248,7 +1242,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
DebugLog.WriteLine("ContentDownloader", "Downloading chunk {0} from {1} with {2}", chunkID, connection, cdnPool.ProxyServer != null ? cdnPool.ProxyServer : "no proxy");
written = await cdnPool.CDNClient.DownloadDepotChunkAsync(
depot.DepotId,
data,
chunk,
connection,
chunkBuffer,
depot.DepotKey,
Expand Down Expand Up @@ -1317,7 +1311,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
fileStreamData.fileStream = File.Open(fileFinalPath, FileMode.Open);
}

fileStreamData.fileStream.Seek((long)data.Offset, SeekOrigin.Begin);
fileStreamData.fileStream.Seek((long)chunk.Offset, SeekOrigin.Begin);
await fileStreamData.fileStream.WriteAsync(chunkBuffer.AsMemory(0, written), cts.Token);
}
finally
Expand Down Expand Up @@ -1361,7 +1355,7 @@ private static async Task DownloadSteam3AsyncDepotFileChunk(
}
}

static void DumpManifestToTextFile(DepotDownloadInfo depot, ProtoManifest manifest)
static void DumpManifestToTextFile(DepotDownloadInfo depot, DepotManifest manifest)
{
var txtManifest = Path.Combine(depot.InstallDir, $"manifest_{depot.DepotId}_{depot.ManifestId}.txt");
using var sw = new StreamWriter(txtManifest);
Expand Down
Loading

0 comments on commit 02e323a

Please sign in to comment.