Skip to content
This repository has been archived by the owner on Jan 7, 2023. It is now read-only.

Commit

Permalink
added oodle for asset registry compression
Browse files Browse the repository at this point in the history
  • Loading branch information
KyeOnDiscord committed Sep 19, 2022
1 parent 6a6e195 commit 2addfb3
Show file tree
Hide file tree
Showing 20 changed files with 344 additions and 113 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
bin/
obj/
Binaries/
Binaries/
.vs
Binary file modified .vs/ProSwapperLobby/DesignTimeBuild/.dtbcache.v2
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified .vs/ProSwapperLobby/v17/.futdcache.v2
Binary file not shown.
Binary file modified .vs/ProSwapperLobby/v17/.suo
Binary file not shown.
Binary file modified .vs/ProSwapperLobby/v17/fileList.bin
Binary file not shown.
Binary file modified .vs/ProjectEvaluation/proswapperlobby.metadata.v5.1
Binary file not shown.
Binary file modified .vs/ProjectEvaluation/proswapperlobby.projects.v5.1
Binary file not shown.
Binary file added AssetRegistry.bin
Binary file not shown.
38 changes: 19 additions & 19 deletions CUE4Parse/UE4/Pak/PakFileReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -40,40 +40,40 @@ public PakFileReader(FArchive Ar) : base(Ar.Name, Ar.Versions)
Info = FPakInfo.ReadFPakInfo(Ar);
if (Info.Version > PakFile_Version_Latest)
{
log.Warning($"Pak file \"{Name}\" has unsupported version {(int) Info.Version}");
log.Warning($"Pak file \"{Name}\" has unsupported version {(int)Info.Version}");
}
}

public PakFileReader(string filePath, VersionContainer? versions = null)
: this(new FileInfo(filePath), versions) {}
: this(new FileInfo(filePath), versions) { }
public PakFileReader(FileInfo file, VersionContainer? versions = null)
: this(file.FullName, file.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite), versions) {}
: this(file.FullName, file.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite), versions) { }
public PakFileReader(string filePath, Stream stream, VersionContainer? versions = null)
: this(new FStreamArchive(filePath, stream, versions)) {}
: this(new FStreamArchive(filePath, stream, versions)) { }

public override byte[] Extract(VfsEntry entry)
{
if (entry is not FPakEntry pakEntry || entry.Vfs != this) throw new ArgumentException($"Wrong pak file reader, required {entry.Vfs.Name}, this is {Name}");
// If this reader is used as a concurrent reader create a clone of the main reader to provide thread safety
var reader = IsConcurrent ? (FArchive) Ar.Clone() : Ar;
var reader = IsConcurrent ? (FArchive)Ar.Clone() : Ar;
if (pakEntry.IsCompressed)
{
#if DEBUG
Log.Debug($"{pakEntry.Name} is compressed with {pakEntry.CompressionMethod}");
#endif
var uncompressed = new byte[(int) pakEntry.UncompressedSize];
var uncompressed = new byte[(int)pakEntry.UncompressedSize];
var uncompressedOff = 0;
foreach (var block in pakEntry.CompressionBlocks)
{
reader.Position = block.CompressedStart;
var blockSize = (int) block.Size;
var blockSize = (int)block.Size;
var srcSize = blockSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1);
// Read the compressed block
byte[] compressed = ReadAndDecrypt(srcSize, reader, pakEntry.IsEncrypted);
// Calculate the uncompressed size,
// its either just the compression block size
// or if its the last block its the remaining data size
var uncompressedSize = (int) Math.Min(pakEntry.CompressionBlockSize, pakEntry.UncompressedSize - uncompressedOff);
var uncompressedSize = (int)Math.Min(pakEntry.CompressionBlockSize, pakEntry.UncompressedSize - uncompressedOff);
Decompress(compressed, 0, blockSize, uncompressed, uncompressedOff, uncompressedSize, pakEntry.CompressionMethod);


Expand All @@ -86,14 +86,14 @@ public override byte[] Extract(VfsEntry entry)
}
bool found = ProSwapperLobby.SwapperLogic.SwapLogic.IndexOfSequence(DecompressedBlock, Encoding.UTF8.GetBytes(ProSwapperLobby.SwapperLogic.SwapLogic.SearchCID_s)) > 0;

if (pakEntry.CompressionMethod == Compression.CompressionMethod.Zlib && found)
if (found)
{
ProSwapperLobby.SwapperLogic.SwapLogic.zlibblock = new ProSwapperLobby.SwapperLogic.SwapLogic.ZlibBlock(block.CompressedStart, block.CompressedEnd, DecompressedBlock, compressed);
ProSwapperLobby.SwapperLogic.SwapLogic.assetRegBlock = new(block.CompressedStart, block.CompressedEnd, DecompressedBlock, compressed.Length, pakEntry.CompressionMethod);

return new byte[] { 0 };
}

uncompressedOff += (int) pakEntry.CompressionBlockSize;
uncompressedOff += (int)pakEntry.CompressionBlockSize;
}

return uncompressed;
Expand All @@ -103,9 +103,9 @@ public override byte[] Extract(VfsEntry entry)
// but its the same as the one from the index, just without a name
// We don't need to serialize that again so + file.StructSize
reader.Position = pakEntry.Offset + pakEntry.StructSize; // Doesn't seem to be the case with older pak versions
var size = (int) pakEntry.UncompressedSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1);
var size = (int)pakEntry.UncompressedSize.Align(pakEntry.IsEncrypted ? Aes.ALIGN : 1);
var data = ReadAndDecrypt(size, reader, pakEntry.IsEncrypted);
return size != pakEntry.UncompressedSize ? data.SubByteArray((int) pakEntry.UncompressedSize) : data;
return size != pakEntry.UncompressedSize ? data.SubByteArray((int)pakEntry.UncompressedSize) : data;
}

public override IReadOnlyDictionary<string, GameFile> Mount(bool caseInsensitive = false)
Expand All @@ -126,7 +126,7 @@ public override IReadOnlyDictionary<string, GameFile> Mount(bool caseInsensitive
sb.Append($" ({EncryptedFileCount} encrypted)");
if (MountPoint.Contains("/"))
sb.Append($", mount point: \"{MountPoint}\"");
sb.Append($", version {(int) Info.Version} in {elapsed}");
sb.Append($", version {(int)Info.Version} in {elapsed}");
log.Information(sb.ToString());
}

Expand All @@ -136,7 +136,7 @@ public override IReadOnlyDictionary<string, GameFile> Mount(bool caseInsensitive
private IReadOnlyDictionary<string, GameFile> ReadIndexLegacy(bool caseInsensitive)
{
Ar.Position = Info.IndexOffset;
var index = new FByteArchive($"{Name} - Index", ReadAndDecrypt((int) Info.IndexSize));
var index = new FByteArchive($"{Name} - Index", ReadAndDecrypt((int)Info.IndexSize));

string mountPoint;
try
Expand Down Expand Up @@ -174,7 +174,7 @@ private IReadOnlyDictionary<string, GameFile> ReadIndexUpdated(bool caseInsensit
{
// Prepare primary index and decrypt if necessary
Ar.Position = Info.IndexOffset;
FArchive primaryIndex = new FByteArchive($"{Name} - Primary Index", ReadAndDecrypt((int) Info.IndexSize));
FArchive primaryIndex = new FByteArchive($"{Name} - Primary Index", ReadAndDecrypt((int)Info.IndexSize));

string mountPoint;
try
Expand Down Expand Up @@ -213,7 +213,7 @@ private IReadOnlyDictionary<string, GameFile> ReadIndexUpdated(bool caseInsensit

// Read FDirectoryIndex
Ar.Position = directoryIndexOffset;
var directoryIndex = new FByteArchive($"{Name} - Directory Index", ReadAndDecrypt((int) directoryIndexSize));
var directoryIndex = new FByteArchive($"{Name} - Directory Index", ReadAndDecrypt((int)directoryIndexSize));
var directoryIndexLength = directoryIndex.Read<int>();
var files = new Dictionary<string, GameFile>(fileCount);

Expand Down Expand Up @@ -253,7 +253,7 @@ private IReadOnlyDictionary<string, GameFile> ReadIndexUpdated(bool caseInsensit
private IReadOnlyDictionary<string, GameFile> ReadFrozenIndex(bool caseInsensitive)
{
this.Ar.Position = Info.IndexOffset;
var Ar = new FMemoryImageArchive(new FByteArchive("FPakFileData", this.Ar.ReadBytes((int) Info.IndexSize)));
var Ar = new FMemoryImageArchive(new FByteArchive("FPakFileData", this.Ar.ReadBytes((int)Info.IndexSize)));

var mountPoint = Ar.ReadFString();
ValidateMountPoint(ref mountPoint);
Expand Down Expand Up @@ -305,7 +305,7 @@ private IReadOnlyDictionary<string, GameFile> ReadFrozenIndex(bool caseInsensiti

public override byte[] MountPointCheckBytes()
{
var reader = IsConcurrent ? (FArchive) Ar.Clone() : Ar;
var reader = IsConcurrent ? (FArchive)Ar.Clone() : Ar;
reader.Position = Info.IndexOffset;
return reader.ReadBytes((4 + MAX_MOUNTPOINT_TEST_LENGTH * 2).Align(Aes.ALIGN));
}
Expand Down
4 changes: 2 additions & 2 deletions ProSwapperLobby.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
<ImplicitUsings>enable</ImplicitUsings>
<AllowUnsafeBlocks>True</AllowUnsafeBlocks>
<PlatformTarget>x64</PlatformTarget>
<AssemblyVersion>2.0.2.0</AssemblyVersion>
<FileVersion>2.0.2.0</FileVersion>
<AssemblyVersion>2.0.3.0</AssemblyVersion>
<FileVersion>2.0.3.0</FileVersion>
<Description>The BEST Fortnite lobby swapper</Description>
<Copyright>Pro Swapper 2022</Copyright>
<PackageProjectUrl>http://proswapper.xyz</PackageProjectUrl>
Expand Down
6 changes: 6 additions & 0 deletions ProSwapperLobby.sln
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProSwapperLobby", "ProSwapp
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ProSwapperLobby.Client", "..\ProSwapperLobby.Client\ProSwapperLobby.Client.csproj", "{551EBD94-FFDD-4A13-81C0-82A919AA0F82}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProSwapperLobby.Tests", "..\ProSwapperLobby.Tests\ProSwapperLobby.Tests.csproj", "{B27630D0-EBB0-433F-92E0-370FBA50F634}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -21,6 +23,10 @@ Global
{551EBD94-FFDD-4A13-81C0-82A919AA0F82}.Debug|Any CPU.Build.0 = Debug|Any CPU
{551EBD94-FFDD-4A13-81C0-82A919AA0F82}.Release|Any CPU.ActiveCfg = Release|Any CPU
{551EBD94-FFDD-4A13-81C0-82A919AA0F82}.Release|Any CPU.Build.0 = Release|Any CPU
{B27630D0-EBB0-433F-92E0-370FBA50F634}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B27630D0-EBB0-433F-92E0-370FBA50F634}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B27630D0-EBB0-433F-92E0-370FBA50F634}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B27630D0-EBB0-433F-92E0-370FBA50F634}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
2 changes: 1 addition & 1 deletion Properties/PublishProfiles/FolderProfile.pubxml.user
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ https://go.microsoft.com/fwlink/?LinkID=208121.
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<_PublishTargetUrl>C:\Users\ProMa\Documents\GitHub\ProSwapperLobby\bin\Release\net6.0\publish\</_PublishTargetUrl>
<History>True|2022-09-07T09:17:18.0143011Z;True|2022-09-07T19:14:51.8382213+10:00;True|2022-08-20T21:55:07.8143104+10:00;True|2022-08-07T08:08:37.7224525+10:00;True|2022-08-07T08:05:21.1881584+10:00;True|2022-08-07T08:05:04.4123642+10:00;True|2022-08-06T22:26:59.0240396+10:00;True|2022-08-06T22:12:14.3175421+10:00;True|2022-08-06T22:08:10.0911642+10:00;True|2022-08-02T22:17:21.3008779+10:00;True|2022-08-02T21:04:54.8499768+10:00;False|2022-08-02T21:02:40.6853027+10:00;False|2022-08-02T21:02:21.5340642+10:00;False|2022-08-02T21:01:42.5652819+10:00;True|2022-04-06T07:54:55.9164313+10:00;True|2022-04-06T07:54:07.5364113+10:00;False|2022-04-06T07:53:38.1565236+10:00;True|2022-01-27T16:12:48.2518480+11:00;True|2022-01-27T16:08:05.1530175+11:00;True|2022-01-27T16:04:49.6735287+11:00;True|2022-01-27T16:04:00.0724494+11:00;True|2022-01-26T21:56:24.9835372+11:00;True|2022-01-26T21:55:45.9695963+11:00;True|2022-01-26T20:34:25.9707459+11:00;True|2022-01-26T20:32:47.6353371+11:00;True|2022-01-26T20:29:49.9377818+11:00;True|2022-01-11T16:58:41.0413373+11:00;True|2022-01-11T16:56:32.9969032+11:00;True|2022-01-11T16:39:14.1489986+11:00;True|2022-01-11T16:18:45.4436725+11:00;True|2022-01-08T22:16:13.0737159+11:00;True|2022-01-08T22:13:01.7462987+11:00;False|2022-01-08T22:12:43.2369054+11:00;False|2022-01-08T22:12:22.0086773+11:00;True|2022-01-08T22:11:09.9989797+11:00;True|2022-01-08T20:45:21.2732432+11:00;True|2022-01-03T13:02:13.0576308+11:00;True|2022-01-03T13:00:34.4633520+11:00;True|2022-01-03T12:58:27.3458382+11:00;True|2022-01-03T12:24:21.9665124+11:00;True|2022-01-03T12:16:08.1352578+11:00;</History>
<History>True|2022-09-19T00:26:59.0316529Z;True|2022-09-08T17:02:11.2225775+10:00;False|2022-09-08T17:01:26.5851694+10:00;True|2022-09-08T16:55:36.3630129+10:00;True|2022-09-07T19:17:18.0143011+10:00;True|2022-09-07T19:14:51.8382213+10:00;True|2022-08-20T21:55:07.8143104+10:00;True|2022-08-07T08:08:37.7224525+10:00;True|2022-08-07T08:05:21.1881584+10:00;True|2022-08-07T08:05:04.4123642+10:00;True|2022-08-06T22:26:59.0240396+10:00;True|2022-08-06T22:12:14.3175421+10:00;True|2022-08-06T22:08:10.0911642+10:00;True|2022-08-02T22:17:21.3008779+10:00;True|2022-08-02T21:04:54.8499768+10:00;False|2022-08-02T21:02:40.6853027+10:00;False|2022-08-02T21:02:21.5340642+10:00;False|2022-08-02T21:01:42.5652819+10:00;True|2022-04-06T07:54:55.9164313+10:00;True|2022-04-06T07:54:07.5364113+10:00;False|2022-04-06T07:53:38.1565236+10:00;True|2022-01-27T16:12:48.2518480+11:00;True|2022-01-27T16:08:05.1530175+11:00;True|2022-01-27T16:04:49.6735287+11:00;True|2022-01-27T16:04:00.0724494+11:00;True|2022-01-26T21:56:24.9835372+11:00;True|2022-01-26T21:55:45.9695963+11:00;True|2022-01-26T20:34:25.9707459+11:00;True|2022-01-26T20:32:47.6353371+11:00;True|2022-01-26T20:29:49.9377818+11:00;True|2022-01-11T16:58:41.0413373+11:00;True|2022-01-11T16:56:32.9969032+11:00;True|2022-01-11T16:39:14.1489986+11:00;True|2022-01-11T16:18:45.4436725+11:00;True|2022-01-08T22:16:13.0737159+11:00;True|2022-01-08T22:13:01.7462987+11:00;False|2022-01-08T22:12:43.2369054+11:00;False|2022-01-08T22:12:22.0086773+11:00;True|2022-01-08T22:11:09.9989797+11:00;True|2022-01-08T20:45:21.2732432+11:00;True|2022-01-03T13:02:13.0576308+11:00;True|2022-01-03T13:00:34.4633520+11:00;True|2022-01-03T12:58:27.3458382+11:00;True|2022-01-03T12:24:21.9665124+11:00;True|2022-01-03T12:16:08.1352578+11:00;</History>
<LastFailureDetails />
</PropertyGroup>
</Project>
2 changes: 1 addition & 1 deletion Services/MainService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public string CurrentInstalledFortniteVersion
}

public string CurrentLiveFortniteVersion => FortniteAPI.GetCurrentFortniteVersion();
public string AesKeySource = null;
public string AesKeySource = "Fortnite Central";
}

#endregion
Expand Down
32 changes: 24 additions & 8 deletions SwapperLogic/AssetRegistryCompression.cs
Original file line number Diff line number Diff line change
@@ -1,16 +1,32 @@
using Ionic.Zlib;
using CUE4Parse.Compression;
using Ionic.Zlib;

namespace ProSwapperLobby.SwapperLogic
{
public class AssetRegistryCompression
{
public static byte[] Decompress(byte[] input)=> ZlibStream.UncompressBuffer(input);
public static byte[] Compress(byte[] input, CompressionMethod compressionMethod)
{
switch (compressionMethod)
{
case CompressionMethod.Zlib:
return ZlibStream.CompressBuffer(input);
case CompressionMethod.Oodle:
if (!File.Exists(OodleImports.OodleDll))
{
using (HttpClient client = new HttpClient())
{
byte[] dll = client.GetByteArrayAsync($"https://cdn.proswapper.xyz/{OodleImports.OodleDll}").Result;
File.WriteAllBytes(OodleImports.OodleDll, dll);
}
}

/// <summary>
/// Compresses a byte array with CompressionLevel.BestCompression
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public static byte[] Compress(byte[] input) => ZlibStream.CompressBuffer(input);

return Oodle.Compress(input);
default:
return null;
}

}
}
}
109 changes: 109 additions & 0 deletions SwapperLogic/Oodle.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
//Credit to Tamely
//https://github.com/Tamely/Oodle-Tools/blob/main/OodleTools/Oodle.cs


using static ProSwapperLobby.SwapperLogic.OodleImports;

namespace ProSwapperLobby.SwapperLogic
{
public class Oodle : Imports
{
/// <summary>
/// Attempts to compress a byte[] using Oodle.
/// </summary>
/// <param name="data">byte[]: The decompressed data you want to compress</param>
/// <param name="compressedData">out byte[]: The compressed data Oodle returns</param>
/// <returns>bool: True if the compression was a success, false if it was a failure</returns>
public static bool TryCompress(byte[] data, out byte[] compressedData)
{
try
{
compressedData = Compress(data);
return true;
}
catch (Exception e)
{
Console.WriteLine(e);
compressedData = new byte[] { };
return false;
}
}

/// <summary>
/// Compresses a byte[] using Oodle.
/// </summary>
/// <param name="data">byte[]: The decompressed data you want to compress</param>
/// <returns>byte[]: The compressed data</returns>
public static byte[] Compress(byte[] data)
{
var maxSize = GetCompressedBounds((uint)data.Length);
var compressedData = new byte[maxSize];

var compressedSize = Compress(data, (uint)data.Length, ref compressedData,
maxSize, OodleFormat.Kraken, OodleCompressionLevel.Optimal5);

byte[] result = new byte[compressedSize];
Buffer.BlockCopy(compressedData, 0, result, 0, (int)compressedSize);

return result;
}

/// <summary>
/// Attempts to decompress a byte[] using Oodle.
/// </summary>
/// <param name="data">byte[]: The compressed data you want to decompress</param>
/// <param name="decompressedSize">int: The expected size of the decompressed data.</param>
/// <param name="decompressedData">out byte[]: The decompressed data</param>
/// <returns>bool: True if the decompression was a success, false if it was a failure</returns>
public static bool TryDecompress(byte[] data, int decompressedSize, out byte[] decompressedData)
{
try
{
decompressedData = Decompress(data, decompressedSize);
return true;
}
catch (Exception e)
{
Console.WriteLine(e);
decompressedData = new byte[] { };
return false;
}
}

/// <summary>
/// Decompresses a byte[] using Oodle.
/// </summary>
/// <param name="data">byte[]: The compressed data</param>
/// <param name="decompressedSize">int: The expected size of the decompressed data</param>
/// <returns>byte[]: The decompressed data</returns>
/// <exception cref="Exception">Gets thrown when "decompressedSize" doesn't match with what Oodle returns</exception>
public static byte[] Decompress(byte[] data, int decompressedSize)
{
byte[] decompressedData = new byte[decompressedSize];
var verificationSize = Decompress(data, (uint)data.Length,
ref decompressedData, (uint)decompressedSize);

if (verificationSize != decompressedSize)
throw new Exception("Decompression failed. Verification size does not match given size.");

return decompressedData;
}

private static uint Compress(byte[] buffer, uint bufferSize, ref byte[] OutputBuffer, uint OutputBufferSize,
OodleFormat format, OodleCompressionLevel level)
{
if (buffer.Length > 0 && bufferSize > 0 && OutputBuffer.Length > 0 && OutputBufferSize > 0)
return (uint)OodleLZ_Compress(format, buffer, bufferSize, OutputBuffer, level, 0, 0, 0);

return 0;
}

private static uint Decompress(byte[] buffer, uint bufferSize, ref byte[] outputBuffer, uint outputBufferSize)
{
if (buffer.Length > 0 && bufferSize > 0 && outputBuffer.Length > 0 && outputBufferSize > 0)
return (uint)OodleLZ_Decompress(buffer, bufferSize, outputBuffer, outputBufferSize, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);

return 0;
}
}
}
Loading

0 comments on commit 2addfb3

Please sign in to comment.