diff --git a/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByIntentionSystem.cs b/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByIntentionSystem.cs index e78d44e007..683c500d11 100644 --- a/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByIntentionSystem.cs +++ b/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByIntentionSystem.cs @@ -40,7 +40,7 @@ IRealmData realmData } protected sealed override async UniTask> FlowInternalAsync(TIntention intention, - IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { await realmData.WaitConfiguredAsync(); diff --git a/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByPointersSystem.cs b/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByPointersSystem.cs index b5518c3b28..6c768bc512 100644 --- a/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByPointersSystem.cs +++ b/Explorer/Assets/DCL/AvatarRendering/Loading/Systems/Abstract/LoadElementsByPointersSystem.cs @@ -33,8 +33,7 @@ protected LoadElementsByPointersSystem(World world, IStreamableCache> FlowInternalAsync( - TIntention intention, IAcquiredBudget acquiredBudget, + protected sealed override async UniTask> FlowInternalAsync(TIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { var finalTargetList = RepoolableList.NewList(); diff --git a/Explorer/Assets/DCL/AvatarRendering/Wearables/Systems/Load/LoadWearableAssetBundleManifestSystem.cs b/Explorer/Assets/DCL/AvatarRendering/Wearables/Systems/Load/LoadWearableAssetBundleManifestSystem.cs index 95ce74c72e..5cfb419482 100644 --- a/Explorer/Assets/DCL/AvatarRendering/Wearables/Systems/Load/LoadWearableAssetBundleManifestSystem.cs +++ b/Explorer/Assets/DCL/AvatarRendering/Wearables/Systems/Load/LoadWearableAssetBundleManifestSystem.cs @@ -31,7 +31,7 @@ internal LoadWearableAssetBundleManifestSystem(World world, this.webRequestController = webRequestController; } - protected override async UniTask> FlowInternalAsync(GetWearableAssetBundleManifestIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) => + protected override async UniTask> FlowInternalAsync(GetWearableAssetBundleManifestIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) => new ( await LoadThumbnailsUtils.LoadAssetBundleManifestAsync( webRequestController, diff --git a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportCategory.cs b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportCategory.cs index 2cce3e7758..73c7c21dfc 100644 --- a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportCategory.cs +++ b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportCategory.cs @@ -79,6 +79,11 @@ public static class ReportCategory /// public const string GENERIC_WEB_REQUEST = nameof(GENERIC_WEB_REQUEST); + /// + /// Partial Loading requests + /// + public const string PARTIAL_LOADING = nameof(PARTIAL_LOADING); + /// /// Texture related web request /// diff --git a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportsHandlingSettingsDevelopment.asset b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportsHandlingSettingsDevelopment.asset index f70e816924..3d05753a9e 100644 --- a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportsHandlingSettingsDevelopment.asset +++ b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Diagnostics/ReportsHandling/ReportsHandlingSettingsDevelopment.asset @@ -486,6 +486,14 @@ MonoBehaviour: Severity: 2 - Category: COMMS_SCENE_HANDLER Severity: 2 + - Category: ASSET_BUNDLES + Severity: 3 + - Category: ASSET_BUNDLES + Severity: 2 + - Category: ASSET_BUNDLES + Severity: 0 + - Category: ASSET_BUNDLES + Severity: 4 sentryMatrix: entries: [] debounceEnabled: 1 diff --git a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/PerformanceBudgeting/AcquiredBudget/IAcquiredBudget.cs b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/PerformanceBudgeting/AcquiredBudget/IAcquiredBudget.cs index 0087249854..2a081725ff 100644 --- a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/PerformanceBudgeting/AcquiredBudget/IAcquiredBudget.cs +++ b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/PerformanceBudgeting/AcquiredBudget/IAcquiredBudget.cs @@ -5,6 +5,7 @@ namespace DCL.Optimization.PerformanceBudgeting public interface IAcquiredBudget : IDisposable { /// + /// Releases budget preemptively without waiting for the full resolution of the flow
/// Implementation should be safe for repetitive invocations /// void Release(); diff --git a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/Pools/PoolConstants.cs b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/Pools/PoolConstants.cs index e9f71cb609..36498dcc46 100644 --- a/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/Pools/PoolConstants.cs +++ b/Explorer/Assets/DCL/PerformanceAndDiagnostics/Optimization/Pools/PoolConstants.cs @@ -19,11 +19,6 @@ public static class PoolConstants /// public const int GLOBAL_WORLD_COUNT = 50; - /// - /// Initial capacity of pools that should exist per empty scene context - /// - public const int EMPTY_SCENES_COUNT = 400; - /// /// The maximum number of scenes before everything explodes according to our expectations /// @@ -34,6 +29,11 @@ public static class PoolConstants /// public const int ENTITIES_COUNT_PER_SCENE = 2000; + /// + /// Initial capacity of pools connected to the initial capacity of the asset promises per scene + /// + public const int ASSET_PROMISES_PER_SCENE_COUNT = ENTITIES_COUNT_PER_SCENE / 2; + /// /// Initial capacity of pools connected to pointer events processing per scene /// diff --git a/Explorer/Assets/DCL/PluginSystem/DCL.Plugins.asmdef b/Explorer/Assets/DCL/PluginSystem/DCL.Plugins.asmdef index 7e9497b148..901f2c88d7 100644 --- a/Explorer/Assets/DCL/PluginSystem/DCL.Plugins.asmdef +++ b/Explorer/Assets/DCL/PluginSystem/DCL.Plugins.asmdef @@ -151,7 +151,8 @@ "GUID:0f5ab34acaa8744d8a22d7dc4c45a487", "GUID:134508e23dba8457db8766637d1d33ec", "GUID:fdc035e0abb695e408d8ccf2c3bd63a5", - "GUID:ead265f036164eb7899edc341131f4d5" + "GUID:ead265f036164eb7899edc341131f4d5", + "GUID:62152fb9d26054b378ea1b93cb8a7f16" ], "includePlatforms": [], "excludePlatforms": [], diff --git a/Explorer/Assets/DCL/PluginSystem/World/AssetBundlesPlugin.cs b/Explorer/Assets/DCL/PluginSystem/World/AssetBundlesPlugin.cs index 9f697c24cb..310912738a 100644 --- a/Explorer/Assets/DCL/PluginSystem/World/AssetBundlesPlugin.cs +++ b/Explorer/Assets/DCL/PluginSystem/World/AssetBundlesPlugin.cs @@ -9,6 +9,7 @@ using ECS.LifeCycle; using ECS.StreamableLoading.AssetBundles; using System; +using System.Buffers; using System.Collections.Generic; using System.Threading; using UnityEngine; @@ -17,6 +18,8 @@ namespace DCL.PluginSystem.World { public class AssetBundlesPlugin : IDCLWorldPluginWithoutSettings, IDCLGlobalPluginWithoutSettings { + private readonly ArrayPool buffersPool = ArrayPool.Create(1024 * 1024, 100); + public static readonly URLDomain STREAMING_ASSETS_URL = URLDomain.FromString( #if UNITY_EDITOR || UNITY_STANDALONE @@ -48,7 +51,7 @@ public void InjectToWorld(ref ArchSystemsWorldBuilder builder, PrepareAssetBundleLoadingParametersSystem.InjectToWorld(ref builder, sharedDependencies.SceneData, STREAMING_ASSETS_URL); // TODO create a runtime ref-counting cache - LoadAssetBundleSystem.InjectToWorld(ref builder, assetBundleCache, webRequestController, assetBundleLoadingMutex); + LoadAssetBundleSystem.InjectToWorld(ref builder, assetBundleCache, webRequestController, buffersPool, assetBundleLoadingMutex); } public void InjectToWorld(ref ArchSystemsWorldBuilder builder, in GlobalPluginArguments arguments) @@ -57,7 +60,7 @@ public void InjectToWorld(ref ArchSystemsWorldBuilder builder, PrepareGlobalAssetBundleLoadingParametersSystem.InjectToWorld(ref builder, STREAMING_ASSETS_URL); // TODO create a runtime ref-counting cache - LoadGlobalAssetBundleSystem.InjectToWorld(ref builder, assetBundleCache, webRequestController, assetBundleLoadingMutex); + LoadGlobalAssetBundleSystem.InjectToWorld(ref builder, assetBundleCache, webRequestController, assetBundleLoadingMutex, buffersPool); } UniTask IDCLPlugin.InitializeAsync(NoExposedPluginSettings settings, CancellationToken ct) => diff --git a/Explorer/Assets/DCL/PluginSystem/World/NFTShapePlugin.cs b/Explorer/Assets/DCL/PluginSystem/World/NFTShapePlugin.cs index 19e6f6d865..8cd36be39c 100644 --- a/Explorer/Assets/DCL/PluginSystem/World/NFTShapePlugin.cs +++ b/Explorer/Assets/DCL/PluginSystem/World/NFTShapePlugin.cs @@ -24,6 +24,8 @@ using ECS.StreamableLoading.NFTShapes; using ECS.StreamableLoading.NFTShapes.URNs; using ECS.StreamableLoading.Textures; +using Plugins.TexturesFuse.TexturesServerWrap.Unzips; +using System.Buffers; using System.Collections.Generic; using System.Threading; @@ -31,6 +33,7 @@ namespace DCL.PluginSystem.World { public class NFTShapePlugin : IDCLWorldPlugin { + private readonly ArrayPool buffersPool = ArrayPool.Create(1024 * 1024, 100); private readonly IDecentralandUrlsSource decentralandUrlsSource; private readonly INFTShapeRendererFactory nftShapeRendererFactory; private readonly IPerformanceBudget instantiationFrameTimeBudgetProvider; @@ -38,6 +41,7 @@ public class NFTShapePlugin : IDCLWorldPlugin private readonly IWebRequestController webRequestController; private readonly IFramePrefabs framePrefabs; private readonly ILazyMaxSize lazyMaxSize; + private readonly ITexturesFuse texturesFuse; private readonly ISizedStreamableCache cache = new NftShapeCache(); static NFTShapePlugin() @@ -51,7 +55,8 @@ public NFTShapePlugin( IPerformanceBudget instantiationFrameTimeBudgetProvider, IComponentPoolsRegistry componentPoolsRegistry, IWebRequestController webRequestController, - CacheCleaner cacheCleaner + CacheCleaner cacheCleaner, + ITexturesFuse texturesFuse ) : this( decentralandUrlsSource, instantiationFrameTimeBudgetProvider, @@ -61,7 +66,8 @@ CacheCleaner cacheCleaner webRequestController, cacheCleaner, new IWebContentSizes.Default(LazyMaxSize(out var lazyMaxSize)), - lazyMaxSize + lazyMaxSize, + texturesFuse ) { } public NFTShapePlugin( @@ -73,7 +79,8 @@ public NFTShapePlugin( IWebRequestController webRequestController, CacheCleaner cacheCleaner, IWebContentSizes webContentSizes, - ILazyMaxSize lazyMaxSize + ILazyMaxSize lazyMaxSize, + ITexturesFuse texturesFuse ) : this( decentralandUrlsSource, new PoolNFTShapeRendererFactory(componentPoolsRegistry, framesPool), @@ -82,7 +89,8 @@ ILazyMaxSize lazyMaxSize webRequestController, cacheCleaner, framePrefabs, - lazyMaxSize + lazyMaxSize, + texturesFuse ) { } public NFTShapePlugin( @@ -93,7 +101,8 @@ public NFTShapePlugin( IWebRequestController webRequestController, CacheCleaner cacheCleaner, IFramePrefabs framePrefabs, - ILazyMaxSize lazyMaxSize + ILazyMaxSize lazyMaxSize, + ITexturesFuse texturesFuse ) { this.decentralandUrlsSource = decentralandUrlsSource; @@ -103,6 +112,7 @@ ILazyMaxSize lazyMaxSize this.webRequestController = webRequestController; this.framePrefabs = framePrefabs; this.lazyMaxSize = lazyMaxSize; + this.texturesFuse = texturesFuse; cacheCleaner.Register(cache); } @@ -126,7 +136,7 @@ public void InjectToWorld(ref ArchSystemsWorldBuilder builder, { var buffer = sharedDependencies.EntityEventsBuilder.Rent(); - LoadNFTShapeSystem.InjectToWorld(ref builder, cache, webRequestController); + LoadNFTShapeSystem.InjectToWorld(ref builder, cache, webRequestController, buffersPool, texturesFuse); LoadCycleNftShapeSystem.InjectToWorld(ref builder, new BasedURNSource(decentralandUrlsSource)); InstantiateNftShapeSystem.InjectToWorld(ref builder, nftShapeRendererFactory, instantiationFrameTimeBudgetProvider, framePrefabs, buffer); VisibilityNftShapeSystem.InjectToWorld(ref builder, buffer); diff --git a/Explorer/Assets/DCL/PluginSystem/World/TexturesLoadingPlugin.cs b/Explorer/Assets/DCL/PluginSystem/World/TexturesLoadingPlugin.cs index d13be0ede7..6d566403ac 100644 --- a/Explorer/Assets/DCL/PluginSystem/World/TexturesLoadingPlugin.cs +++ b/Explorer/Assets/DCL/PluginSystem/World/TexturesLoadingPlugin.cs @@ -6,7 +6,9 @@ using DCL.WebRequests; using ECS.LifeCycle; using ECS.StreamableLoading.Textures; +using Plugins.TexturesFuse.TexturesServerWrap.Unzips; using System; +using System.Buffers; using System.Collections.Generic; using System.Threading; @@ -15,23 +17,26 @@ namespace DCL.PluginSystem.World public class TexturesLoadingPlugin : IDCLWorldPluginWithoutSettings, IDCLGlobalPluginWithoutSettings { private readonly IWebRequestController webRequestController; + private readonly ITexturesFuse texturesFuse; private readonly TexturesCache texturesCache = new (); + private readonly ArrayPool buffersPool = ArrayPool.Create(1024 * 1024, 100); - public TexturesLoadingPlugin(IWebRequestController webRequestController, CacheCleaner cacheCleaner) + public TexturesLoadingPlugin(IWebRequestController webRequestController, CacheCleaner cacheCleaner, ITexturesFuse texturesFuse) { this.webRequestController = webRequestController; + this.texturesFuse = texturesFuse; cacheCleaner.Register(texturesCache); } public void InjectToWorld(ref ArchSystemsWorldBuilder builder, in ECSWorldInstanceSharedDependencies sharedDependencies, in PersistentEntities persistentEntities, List finalizeWorldSystems, List sceneIsCurrentListeners) { - LoadTextureSystem.InjectToWorld(ref builder, texturesCache, webRequestController); + LoadTextureSystem.InjectToWorld(ref builder, texturesCache, webRequestController, buffersPool, texturesFuse); } public void InjectToWorld(ref ArchSystemsWorldBuilder builder, in GlobalPluginArguments arguments) { - LoadGlobalTextureSystem.InjectToWorld(ref builder, texturesCache, webRequestController); + LoadGlobalTextureSystem.InjectToWorld(ref builder, texturesCache, webRequestController, buffersPool, texturesFuse); } UniTask IDCLPlugin.InitializeAsync(NoExposedPluginSettings settings, CancellationToken ct) => diff --git a/Explorer/Assets/DCL/Profiles/Systems/LoadProfileSystem.cs b/Explorer/Assets/DCL/Profiles/Systems/LoadProfileSystem.cs index a4c7ca921f..8a4e636895 100644 --- a/Explorer/Assets/DCL/Profiles/Systems/LoadProfileSystem.cs +++ b/Explorer/Assets/DCL/Profiles/Systems/LoadProfileSystem.cs @@ -38,7 +38,7 @@ protected override void Update(float t) } protected override async UniTask> FlowInternalAsync(GetProfileIntention intention, - IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { Profile? profile = await profileRepository.GetAsync(intention.ProfileId, intention.Version, ct); diff --git a/Explorer/Assets/DCL/SDKComponents/NFTShape/Demo/NFTShapeDemoWorld.cs b/Explorer/Assets/DCL/SDKComponents/NFTShape/Demo/NFTShapeDemoWorld.cs index 48b35265e4..cc41df4a57 100644 --- a/Explorer/Assets/DCL/SDKComponents/NFTShape/Demo/NFTShapeDemoWorld.cs +++ b/Explorer/Assets/DCL/SDKComponents/NFTShape/Demo/NFTShapeDemoWorld.cs @@ -11,7 +11,6 @@ using DCL.SDKComponents.NFTShape.Renderer.Factory; using DCL.SDKComponents.NFTShape.System; using DCL.Utilities.Extensions; -using DCL.Web3.Identities; using DCL.WebRequests; using ECS.Abstract; using ECS.Prioritization.Components; @@ -20,6 +19,7 @@ using ECS.StreamableLoading.NFTShapes.URNs; using ECS.Unity.Transforms.Components; using Plugins.TexturesFuse.TexturesServerWrap.Unzips; +using System.Buffers; using System.Collections.Generic; using UnityEngine; @@ -57,7 +57,9 @@ public NFTShapeDemoWorld(World world, IFramesPool framesPool, w => new LoadNFTShapeSystem( w, new NftShapeCache(), - IWebRequestController.DEFAULT + IWebRequestController.DEFAULT, + ArrayPool.Create(1024 * 1024, 100), + ITexturesFuse.NewTestInstance() ).InitializeAndReturnSelf(), w => new LoadCycleNftShapeSystem(w, new BasedURNSource(new DecentralandUrlsSource(DecentralandEnvironment.Org))), w => new InstantiateNftShapeSystem(w, new PoolNFTShapeRendererFactory(new ComponentPoolsRegistry(), framesPool), new FrameTimeCapBudget.Default(), framePrefabs, buffer), diff --git a/Explorer/Assets/DCL/WebRequests/CommonArguments.cs b/Explorer/Assets/DCL/WebRequests/CommonArguments.cs index 5776aa99e1..509d27dc6b 100644 --- a/Explorer/Assets/DCL/WebRequests/CommonArguments.cs +++ b/Explorer/Assets/DCL/WebRequests/CommonArguments.cs @@ -24,12 +24,9 @@ public readonly struct CommonArguments public readonly int Timeout; - public readonly DownloadHandler? CustomDownloadHandler; - - public CommonArguments(URLAddress url, DownloadHandler? customDownloadHandler = null, int attemptsCount = DEFAULT_ATTEMPTS_COUNT, int timeout = DEFAULT_TIMEOUT, float attemptsDelay = DEFAULT_ATTEMPTS_DELAY) + public CommonArguments(URLAddress url, int attemptsCount = DEFAULT_ATTEMPTS_COUNT, int timeout = DEFAULT_TIMEOUT, float attemptsDelay = DEFAULT_ATTEMPTS_DELAY) { URL = url; - CustomDownloadHandler = customDownloadHandler; AttemptsCount = attemptsCount; Timeout = timeout; AttemptsDelay = attemptsDelay; @@ -53,6 +50,6 @@ public float AttemptsDelayInMilliseconds() => Mathf.Max(0, AttemptsDelay); public override string ToString() => - $"CommonArguments: {URL} with attempts {AttemptsCount} with timeout {Timeout} with downloadHandler {CustomDownloadHandler}"; + $"CommonArguments: {URL} with attempts {AttemptsCount} with timeout {Timeout}"; } } diff --git a/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers.meta b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers.meta new file mode 100644 index 0000000000..cae54020be --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 65aa3c1f1d024250afa5770005da87af +timeCreated: 1736518356 \ No newline at end of file diff --git a/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/DownloadHandlersUtils.cs b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/DownloadHandlersUtils.cs new file mode 100644 index 0000000000..7b0e07bc26 --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/DownloadHandlersUtils.cs @@ -0,0 +1,32 @@ +using System.Text; + +namespace DCL.WebRequests.CustomDownloadHandlers +{ + public class DownloadHandlersUtils + { + private static readonly StringBuilder STRING_BUILDER = new (); + + public static string GetContentRangeHeaderValue(long start, long end) + { + STRING_BUILDER.Clear(); + STRING_BUILDER.Append("bytes="); + STRING_BUILDER.Append(start); + STRING_BUILDER.Append("-"); + STRING_BUILDER.Append(end); + return STRING_BUILDER.ToString(); + } + + public static bool TryGetFullSize(string input, out int result) + { + result = 0; + + if (string.IsNullOrEmpty(input)) return false; + + int separatorIndex = input.IndexOf('/'); + if (separatorIndex == -1 || separatorIndex == input.Length - 1) return false; + + string secondPart = input.Substring(separatorIndex + 1); + return int.TryParse(secondPart, out result); + } + } +} diff --git a/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/DownloadHandlersUtils.cs.meta b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/DownloadHandlersUtils.cs.meta new file mode 100644 index 0000000000..ffc708b9fc --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/DownloadHandlersUtils.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 6d5b92644b1742c7a69bd6895ecdce3b +timeCreated: 1736518369 \ No newline at end of file diff --git a/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/PartialDownloadHandler.cs b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/PartialDownloadHandler.cs new file mode 100644 index 0000000000..42c67e5cc3 --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/PartialDownloadHandler.cs @@ -0,0 +1,45 @@ +using DCL.WebRequests.PartialDownload; +using System; +using System.IO; +using UnityEngine; +using UnityEngine.Networking; + +namespace DCL.WebRequests.CustomDownloadHandlers +{ + public class PartialDownloadHandler : DownloadHandlerScript + { + private readonly PartialDownloadingData partialData; + private int bufferPointer = 0; + + public PartialDownloadHandler(ref PartialDownloadingData partialData, byte[] preallocatedBuffer) : base(preallocatedBuffer) + { + this.partialData = partialData; + } + + protected override void CompleteContent() + { + + } + + protected override bool ReceiveData(byte[] receivedData, int dataLength) + { + if (dataLength == 0) + return false; // No data received + + try + { + for(var i = 0; i < dataLength; i++) + { + partialData.DataBuffer[bufferPointer] = receivedData[i]; + bufferPointer++; + } + return true; + } + catch (Exception ex) + { + Debug.LogError($"Error writing data: {ex.Message}"); + return false; + } + } + } +} diff --git a/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/PartialDownloadHandler.cs.meta b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/PartialDownloadHandler.cs.meta new file mode 100644 index 0000000000..a7fa36399c --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/CustomDownloadHandlers/PartialDownloadHandler.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 497985c08e704c98a5cb6f56dad0ab19 +timeCreated: 1736518391 \ No newline at end of file diff --git a/Explorer/Assets/DCL/WebRequests/PartialDownload.meta b/Explorer/Assets/DCL/WebRequests/PartialDownload.meta new file mode 100644 index 0000000000..4d6a44099b --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/PartialDownload.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: b496a6826bca4a189a9251e443a155b8 +timeCreated: 1736518414 \ No newline at end of file diff --git a/Explorer/Assets/DCL/WebRequests/PartialDownload/FullDownloadedData.cs b/Explorer/Assets/DCL/WebRequests/PartialDownload/FullDownloadedData.cs new file mode 100644 index 0000000000..c01252b348 --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/PartialDownload/FullDownloadedData.cs @@ -0,0 +1,14 @@ +namespace DCL.WebRequests.PartialDownload +{ + //Consider making disposable + public struct FullDownloadedData + { + //Use memory or Stream + public readonly byte[] DataBuffer; + + public FullDownloadedData(byte[] dataBuffer) + { + DataBuffer = dataBuffer; + } + } +} diff --git a/Explorer/Assets/DCL/WebRequests/PartialDownload/FullDownloadedData.cs.meta b/Explorer/Assets/DCL/WebRequests/PartialDownload/FullDownloadedData.cs.meta new file mode 100644 index 0000000000..674b32c1f9 --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/PartialDownload/FullDownloadedData.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: b721072fdd2c4b0a8a02221bba8f60a8 +timeCreated: 1736518430 \ No newline at end of file diff --git a/Explorer/Assets/DCL/WebRequests/PartialDownload/PartialDownloadingData.cs b/Explorer/Assets/DCL/WebRequests/PartialDownload/PartialDownloadingData.cs new file mode 100644 index 0000000000..77623574f1 --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/PartialDownload/PartialDownloadingData.cs @@ -0,0 +1,28 @@ +using System; + +namespace DCL.WebRequests.PartialDownload +{ + public struct PartialDownloadingData + { + public const int CHUNK_SIZE = 1024 * 1024; // 1MB + + public readonly byte[] DataBuffer; + + public int RangeStart; + public int RangeEnd; + public int FullFileSize; + + public PartialDownloadingData(byte[] dataBuffer, int rangeStart, int rangeEnd, int fullFileSize = 0) + { + this.DataBuffer = dataBuffer; + this.RangeStart = rangeStart; + this.RangeEnd = rangeEnd; + FullFileSize = fullFileSize; + } + + public void ClearBuffer() + { + Array.Clear(DataBuffer, 0, DataBuffer.Length); + } + } +} diff --git a/Explorer/Assets/DCL/WebRequests/PartialDownload/PartialDownloadingData.cs.meta b/Explorer/Assets/DCL/WebRequests/PartialDownload/PartialDownloadingData.cs.meta new file mode 100644 index 0000000000..92247de80f --- /dev/null +++ b/Explorer/Assets/DCL/WebRequests/PartialDownload/PartialDownloadingData.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 202344407be04f078df7ab2c59fdc270 +timeCreated: 1736518449 \ No newline at end of file diff --git a/Explorer/Assets/DCL/WebRequests/RequestEnvelope.cs b/Explorer/Assets/DCL/WebRequests/RequestEnvelope.cs index 54bae560ea..151e8ac0ce 100644 --- a/Explorer/Assets/DCL/WebRequests/RequestEnvelope.cs +++ b/Explorer/Assets/DCL/WebRequests/RequestEnvelope.cs @@ -19,6 +19,7 @@ namespace DCL.WebRequests public readonly bool SuppressErrors; private readonly InitializeRequest initializeRequest; private readonly TWebRequestArgs args; + private readonly DownloadHandler? customDownloadHandler; private readonly WebRequestHeadersInfo headersInfo; private readonly WebRequestSignInfo? signInfo; private readonly ISet? responseCodeIgnores; @@ -33,7 +34,8 @@ public RequestEnvelope( WebRequestHeadersInfo headersInfo, WebRequestSignInfo? signInfo, ISet? responseCodeIgnores = null, - bool suppressErrors = false + bool suppressErrors = false, + DownloadHandler? customDownloadHandler = null ) { this.initializeRequest = initializeRequest; @@ -44,6 +46,7 @@ public RequestEnvelope( this.headersInfo = headersInfo; this.signInfo = signInfo; SuppressErrors = suppressErrors; + this.customDownloadHandler = customDownloadHandler; this.responseCodeIgnores = responseCodeIgnores; } @@ -92,8 +95,8 @@ private void AssignHeaders(UnityWebRequest unityWebRequest, IWeb3IdentityCache w private void AssignDownloadHandler(UnityWebRequest unityWebRequest) { - if (CommonArguments.CustomDownloadHandler != null) - unityWebRequest.downloadHandler = CommonArguments.CustomDownloadHandler; + if (customDownloadHandler != null) + unityWebRequest.downloadHandler = customDownloadHandler; } private void AssignTimeout(UnityWebRequest unityWebRequest) diff --git a/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs b/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs index d4eac20aea..73806d5612 100644 --- a/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs +++ b/Explorer/Assets/DCL/WebRequests/WebRequestControllerExtensions.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Threading; using DCL.DebugUtilities.UIBindings; +using DCL.WebRequests.CustomDownloadHandlers; +using DCL.WebRequests.PartialDownload; using Plugins.TexturesFuse.TexturesServerWrap.Unzips; using UnityEngine; using UnityEngine.Networking; @@ -14,6 +16,10 @@ namespace DCL.WebRequests { public static class WebRequestControllerExtensions { + private const string CONTENT_RANGE_HEADER = "Content-Range"; + private const string CONTENT_LENGTH_HEADER = "Content-Length"; + private static readonly byte[] PARTIAL_DOWNLOAD_BUFFER = new byte[1024 * 1024]; + public static UniTask SendAsync( this IWebRequestController controller, CommonArguments commonArguments, @@ -24,7 +30,8 @@ public static UniTask SendAsync? ignoreErrorCodes = null, - bool suppressErrors = false + bool suppressErrors = false, + DownloadHandler? downloadHandler = null ) where TWebRequestArgs: struct where TWebRequest: struct, ITypedWebRequest @@ -39,7 +46,8 @@ public static UniTask SendAsync GetAsync( ISet? ignoreErrorCodes = null ) where TOp: struct, IWebRequestOp => controller.SendAsync(commonArguments, default(GenericGetArguments), webRequestOp, ct, reportData, headersInfo, signInfo, ignoreErrorCodes); + public static UniTask GetPartialAsync( + this IWebRequestController controller, + CommonArguments commonArguments, + CancellationToken ct, + ReportData reportData, + PartialDownloadingData partialData, + WebRequestHeadersInfo? headersInfo = null, + WebRequestSignInfo? signInfo = null, + ISet? ignoreErrorCodes = null + ) + { + PartialDownloadHandler handler = new PartialDownloadHandler(ref partialData, PARTIAL_DOWNLOAD_BUFFER); + return controller.SendAsync(commonArguments, default(GenericGetArguments), new PartialDownloadOp(partialData), ct, reportData, headersInfo, signInfo, ignoreErrorCodes, downloadHandler: handler, suppressErrors: true); + } + public struct PartialDownloadOp : IWebRequestOp + { + private PartialDownloadingData data; + + public PartialDownloadOp(PartialDownloadingData data) + { + this.data = data; + } + + public async UniTask ExecuteAsync(GenericGetRequest webRequest, CancellationToken ct) + { + if (data.FullFileSize == 0) + { + if (DownloadHandlersUtils.TryGetFullSize(webRequest.UnityWebRequest.GetResponseHeader(CONTENT_RANGE_HEADER), out int fullSize)) + { + data.FullFileSize = fullSize; + } + else if (int.TryParse(webRequest.UnityWebRequest.GetResponseHeader(CONTENT_LENGTH_HEADER), out int contentSize)) + { + data.FullFileSize = contentSize; + } + else + { + data.FullFileSize = Convert.ToInt32(webRequest.UnityWebRequest.downloadedBytes); + } + } + + return this.data; + } + } public static UniTask PostAsync( this IWebRequestController controller, CommonArguments commonArguments, diff --git a/Explorer/Assets/DCL/WebRequests/WebRequestHeadersInfo.cs b/Explorer/Assets/DCL/WebRequests/WebRequestHeadersInfo.cs index fc74bf5af5..501297da85 100644 --- a/Explorer/Assets/DCL/WebRequests/WebRequestHeadersInfo.cs +++ b/Explorer/Assets/DCL/WebRequests/WebRequestHeadersInfo.cs @@ -1,4 +1,5 @@ using DCL.Optimization.Pools; +using DCL.WebRequests.CustomDownloadHandlers; using System; using System.Collections.Generic; using System.Linq; @@ -8,6 +9,7 @@ namespace DCL.WebRequests { public struct WebRequestHeadersInfo : IDisposable { + private const string RANGE_HEADER = "Range"; private const string EMPTY_HEADER = ""; private static readonly IReadOnlyDictionary EMPTY_HEADERS = new Dictionary(); @@ -26,6 +28,12 @@ public WebRequestHeadersInfo(IReadOnlyDictionary? headers) Add(key, s); } + public WebRequestHeadersInfo WithRange(long start, long end) + { + Add(RANGE_HEADER, DownloadHandlersUtils.GetContentRangeHeaderValue(start, end)); + return this; + } + internal static WebRequestHeadersInfo NewEmpty() => new (); diff --git a/Explorer/Assets/DCL/WorldTime/WorldTimeProvider.cs b/Explorer/Assets/DCL/WorldTime/WorldTimeProvider.cs index cb7a452384..5d54414069 100644 --- a/Explorer/Assets/DCL/WorldTime/WorldTimeProvider.cs +++ b/Explorer/Assets/DCL/WorldTime/WorldTimeProvider.cs @@ -1,4 +1,5 @@ -using Cysharp.Threading.Tasks; +using Arch.Core; +using Cysharp.Threading.Tasks; using DCL.Diagnostics; using DCL.Multiplayer.Connections.DecentralandUrls; using DCL.Optimization.PerformanceBudgeting; diff --git a/Explorer/Assets/Scripts/CrdtEcsBridge/JsModulesImplementation/RuntimeImplementation.cs b/Explorer/Assets/Scripts/CrdtEcsBridge/JsModulesImplementation/RuntimeImplementation.cs index 71db337b5f..ce99021dcc 100644 --- a/Explorer/Assets/Scripts/CrdtEcsBridge/JsModulesImplementation/RuntimeImplementation.cs +++ b/Explorer/Assets/Scripts/CrdtEcsBridge/JsModulesImplementation/RuntimeImplementation.cs @@ -1,3 +1,4 @@ +using Arch.Core; using CommunicationData.URLHelpers; using Cysharp.Threading.Tasks; using DCL.Diagnostics; diff --git a/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionListSystem.cs b/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionListSystem.cs index 0bcdd2de86..ed09bb6a9e 100644 --- a/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionListSystem.cs +++ b/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionListSystem.cs @@ -38,7 +38,7 @@ internal LoadSceneDefinitionListSystem(World world, IWebRequestController webReq this.webRequestController = webRequestController; } - protected override async UniTask> FlowInternalAsync(GetSceneDefinitionList intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + protected override async UniTask> FlowInternalAsync(GetSceneDefinitionList intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { bodyBuilder.Clear(); bodyBuilder.Append("{\"pointers\":["); diff --git a/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionSystem.cs b/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionSystem.cs index 6aedbc3565..1b761c4bf5 100644 --- a/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionSystem.cs +++ b/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneDefinition/Systems/LoadSceneDefinitionSystem.cs @@ -29,7 +29,7 @@ internal LoadSceneDefinitionSystem(World world, IWebRequestController webRequest this.webRequestController = webRequestController; } - protected override async UniTask> FlowInternalAsync(GetSceneDefinition intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + protected override async UniTask> FlowInternalAsync(GetSceneDefinition intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { SceneEntityDefinition sceneEntityDefinition = await webRequestController.GetAsync(intention.CommonArguments, ct, GetReportData()) diff --git a/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneFacade/Systems/LoadSceneSystem.cs b/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneFacade/Systems/LoadSceneSystem.cs index a5642114b5..44d73a6134 100644 --- a/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneFacade/Systems/LoadSceneSystem.cs +++ b/Explorer/Assets/Scripts/ECS/SceneLifeCycle/SceneFacade/Systems/LoadSceneSystem.cs @@ -33,7 +33,7 @@ internal LoadSceneSystem(World world, this.loadSceneSystemLogic = loadSceneSystemLogic; } - protected override async UniTask> FlowInternalAsync(GetSceneFacadeIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) => + protected override async UniTask> FlowInternalAsync(GetSceneFacadeIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) => new (await loadSceneSystemLogic.FlowAsync(sceneFactory, intention, GetReportData(), partition, ct)); protected override void DisposeAbandonedResult(ISceneFacade asset) diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/GetAssetBundleIntention.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/GetAssetBundleIntention.cs index d9291837a6..be5069f168 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/GetAssetBundleIntention.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/GetAssetBundleIntention.cs @@ -40,12 +40,11 @@ public struct GetAssetBundleIntention : ILoadingIntention, IEquatable /// /// - /// - // /// Used to check if the asset bundle has shader assets in it - // /// + /// Used to check if the asset bundle has shader assets in it + /// public bool LookForShaderAssets; - + private GetAssetBundleIntention(Type? expectedObjectType, string? name = null, string? hash = null, AssetSource permittedSources = AssetSource.ALL, SceneAssetBundleManifest? assetBundleManifest = null, @@ -77,15 +76,18 @@ public bool Equals(GetAssetBundleIntention other) => public CancellationTokenSource CancellationTokenSource => CommonArguments.CancellationTokenSource; - public static GetAssetBundleIntention FromHash(Type? expectedAssetType, string hash, AssetSource permittedSources = AssetSource.ALL, SceneAssetBundleManifest? manifest = null, URLSubdirectory customEmbeddedSubDirectory = default, bool lookForShaderAsset = false) => - new (expectedAssetType, hash: hash, permittedSources: permittedSources, assetBundleManifest: manifest, customEmbeddedSubDirectory: customEmbeddedSubDirectory, lookForShaderAssets:lookForShaderAsset); + public static GetAssetBundleIntention FromHash(Type? expectedAssetType, string hash, AssetSource permittedSources = AssetSource.ALL, SceneAssetBundleManifest? manifest = null, URLSubdirectory customEmbeddedSubDirectory = default, + bool lookForShaderAsset = false) => + new (expectedAssetType, hash: hash, permittedSources: permittedSources, assetBundleManifest: manifest, customEmbeddedSubDirectory: customEmbeddedSubDirectory, lookForShaderAssets: lookForShaderAsset); public static GetAssetBundleIntention FromHash(Type expectedAssetType, string hash, CancellationTokenSource cancellationTokenSource, AssetSource permittedSources = AssetSource.ALL, SceneAssetBundleManifest? manifest = null, URLSubdirectory customEmbeddedSubDirectory = default) => - new (expectedAssetType, hash: hash, permittedSources: permittedSources, assetBundleManifest: manifest, customEmbeddedSubDirectory: customEmbeddedSubDirectory,cancellationTokenSource: cancellationTokenSource); + new (expectedAssetType, hash: hash, permittedSources: permittedSources, assetBundleManifest: manifest, customEmbeddedSubDirectory: customEmbeddedSubDirectory, cancellationTokenSource: cancellationTokenSource); + + public static GetAssetBundleIntention Create(Type? expectedAssetType, string hash, string name, AssetSource permittedSources = AssetSource.ALL, SceneAssetBundleManifest? manifest = null, + URLSubdirectory customEmbeddedSubDirectory = default) => + new (expectedAssetType, hash: hash, name: name, permittedSources: permittedSources, assetBundleManifest: manifest, customEmbeddedSubDirectory: customEmbeddedSubDirectory); - public static GetAssetBundleIntention Create(Type? expectedAssetType, string hash, string name,AssetSource permittedSources = AssetSource.ALL, SceneAssetBundleManifest? manifest = null, URLSubdirectory customEmbeddedSubDirectory = default) => - new (expectedAssetType, hash: hash, name: name,permittedSources: permittedSources, assetBundleManifest: manifest, customEmbeddedSubDirectory: customEmbeddedSubDirectory); public override bool Equals(object obj) => obj is GetAssetBundleIntention other && Equals(other); diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadAssetBundleSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadAssetBundleSystem.cs index 6d743d0eee..14c2152ab0 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadAssetBundleSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadAssetBundleSystem.cs @@ -16,15 +16,15 @@ using System.Threading; using AssetManagement; using DCL.WebRequests; +using System.Buffers; using UnityEngine; -using UnityEngine.Networking; using Object = UnityEngine.Object; namespace ECS.StreamableLoading.AssetBundles { [UpdateInGroup(typeof(StreamableLoadingGroup))] [LogCategory(ReportCategory.ASSET_BUNDLES)] - public partial class LoadAssetBundleSystem : LoadSystemBase + public partial class LoadAssetBundleSystem : PartialDownloadSystemBase { private const string METADATA_FILENAME = "metadata.json"; private const string METRICS_FILENAME = "metrics.json"; @@ -34,47 +34,29 @@ private static readonly ThreadSafeObjectPool METADATA_POOL , maxSize: 100); private readonly AssetBundleLoadingMutex loadingMutex; - private readonly IWebRequestController webRequestController; internal LoadAssetBundleSystem(World world, IStreamableCache cache, - IWebRequestController webRequestController, AssetBundleLoadingMutex loadingMutex) : base(world, cache) + IWebRequestController webRequestController, + ArrayPool buffersPool, + AssetBundleLoadingMutex loadingMutex) : base(world, cache, webRequestController, buffersPool) { this.loadingMutex = loadingMutex; - this.webRequestController = webRequestController; } - private async UniTask LoadDependenciesAsync(GetAssetBundleIntention parentIntent, IPartitionComponent partition, AssetBundleMetadata assetBundleMetadata, CancellationToken ct) + protected override async UniTask> ProcessCompletedData(byte[] completeData, GetAssetBundleIntention intention, IPartitionComponent partition, CancellationToken ct, StreamableLoadingState state) { - // Construct dependency promises and wait for them - // Switch to main thread to create dependency promises - await UniTask.SwitchToMainThread(); - - SceneAssetBundleManifest? manifest = parentIntent.Manifest; - URLSubdirectory customEmbeddedSubdirectory = parentIntent.CommonArguments.CustomEmbeddedSubDirectory; - - return await UniTask.WhenAll(assetBundleMetadata.dependencies.Select(hash => WaitForDependencyAsync(manifest, hash, customEmbeddedSubdirectory, partition, ct))); - } - - protected override async UniTask> FlowInternalAsync(GetAssetBundleIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) - { - AssetBundleLoadingResult assetBundleResult = await webRequestController - .GetAssetBundleAsync(intention.CommonArguments, new GetAssetBundleArguments(loadingMutex, intention.cacheHash), ct, GetReportCategory(), - suppressErrors: true); // Suppress errors because here we have our own error handling - - AssetBundle? assetBundle = assetBundleResult.AssetBundle; + AssetBundle? assetBundle = await AssetBundle.LoadFromMemoryAsync(completeData); // Release budget now to not hold it until dependencies are resolved to prevent a deadlock - acquiredBudget.Release(); + state.AcquiredBudget!.Release(); - // if GetContent prints an error, null will be thrown if (assetBundle == null) - throw new NullReferenceException($"{intention.Hash} Asset Bundle is null: {assetBundleResult.DataProcessingError}"); + throw new NullReferenceException($"{intention.Hash} Asset Bundle is null"); try { // get metrics - string? metricsJSON; string? metadataJSON; @@ -85,7 +67,6 @@ protected override async UniTask> FlowI } // Switch to thread pool to parse JSONs - await UniTask.SwitchToThreadPool(); ct.ThrowIfCancellationRequested(); @@ -105,19 +86,18 @@ protected override async UniTask> FlowI else dependencies = Array.Empty(); + Debug.Log($"Dependencies count {dependencies.Length}"); + ct.ThrowIfCancellationRequested(); string version = intention.Manifest != null ? intention.Manifest.GetVersion() : string.Empty; string source = intention.CommonArguments.CurrentSource.ToStringNonAlloc(); - // if the type was not specified don't load any assets return await CreateAssetBundleDataAsync(assetBundle, metrics, intention.ExpectedObjectType, mainAsset, loadingMutex, dependencies, GetReportData(), version, source, intention.LookForShaderAssets, ct); } - catch (Exception e) + catch (Exception) { // If the loading process didn't finish successfully unload the bundle - // Otherwise, it gets stuck in Unity's memory but not cached in our cache - // Can only be done in main thread await UniTask.SwitchToMainThread(); if (assetBundle) @@ -127,6 +107,18 @@ protected override async UniTask> FlowI } } + private async UniTask LoadDependenciesAsync(GetAssetBundleIntention parentIntent, IPartitionComponent partition, AssetBundleMetadata assetBundleMetadata, CancellationToken ct) + { + // Construct dependency promises and wait for them + // Switch to main thread to create dependency promises + await UniTask.SwitchToMainThread(); + + SceneAssetBundleManifest? manifest = parentIntent.Manifest; + URLSubdirectory customEmbeddedSubdirectory = parentIntent.CommonArguments.CustomEmbeddedSubDirectory; + + return await UniTask.WhenAll(assetBundleMetadata.dependencies.Select(hash => WaitForDependencyAsync(manifest, hash, customEmbeddedSubdirectory, partition, ct))); + } + public static async UniTask> CreateAssetBundleDataAsync( AssetBundle assetBundle, AssetBundleMetrics? metrics, Type? expectedObjType, string? mainAsset, AssetBundleLoadingMutex loadingMutex, @@ -140,7 +132,7 @@ public static async UniTask> CreateAsse // if the type was not specified don't load any assets if (expectedObjType == null) return new StreamableLoadingResult(new AssetBundleData(assetBundle, metrics, dependencies)); - + if (lookForShaderAssets && expectedObjType == typeof(GameObject)) { //If there are no dependencies, it means that this gameobject asset bundle has the shader in it. diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadGlobalAssetBundleSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadGlobalAssetBundleSystem.cs index 0b7eff0c9c..5d3dae8118 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadGlobalAssetBundleSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/LoadGlobalAssetBundleSystem.cs @@ -4,6 +4,7 @@ using DCL.Diagnostics; using DCL.WebRequests; using ECS.StreamableLoading.Cache; +using System.Buffers; using Utility.Multithreading; namespace ECS.StreamableLoading.AssetBundles @@ -15,6 +16,6 @@ namespace ECS.StreamableLoading.AssetBundles [LogCategory(ReportCategory.ASSET_BUNDLES)] public partial class LoadGlobalAssetBundleSystem : LoadAssetBundleSystem { - internal LoadGlobalAssetBundleSystem(World world, IStreamableCache cache, IWebRequestController webRequestController, AssetBundleLoadingMutex loadingMutex) : base(world, cache, webRequestController, loadingMutex) { } + internal LoadGlobalAssetBundleSystem(World world, IStreamableCache cache, IWebRequestController webRequestController, AssetBundleLoadingMutex loadingMutex, ArrayPool buffersPool) : base(world, cache, webRequestController, buffersPool, loadingMutex) { } } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/PrepareAssetBundleLoadingParametersSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/PrepareAssetBundleLoadingParametersSystem.cs index 12af2f72da..00619d41d4 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/PrepareAssetBundleLoadingParametersSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/PrepareAssetBundleLoadingParametersSystem.cs @@ -37,6 +37,6 @@ protected override void Update(float t) assetBundleIntention.Manifest = sceneData.AssetBundleManifest; base.PrepareCommonArguments(in entity, ref assetBundleIntention, ref state); } - + } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/Tests/LoadAssetBundleSystemShould.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/Tests/LoadAssetBundleSystemShould.cs index 20b8043996..b3b56c7601 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/Tests/LoadAssetBundleSystemShould.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/AssetBundles/Tests/LoadAssetBundleSystemShould.cs @@ -2,6 +2,7 @@ using ECS.StreamableLoading.Common.Components; using ECS.StreamableLoading.Tests; using NUnit.Framework; +using System.Buffers; using UnityEngine; using Utility.Multithreading; @@ -34,6 +35,6 @@ protected override GetAssetBundleIntention CreateWrongTypeIntention() => new (new CommonLoadingArguments(wrongTypePath)); protected override LoadAssetBundleSystem CreateSystem() => - new (world, cache, IWebRequestController.DEFAULT, new AssetBundleLoadingMutex()); + new (world, cache, IWebRequestController.DEFAULT, ArrayPool.Shared, new AssetBundleLoadingMutex()); } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/AudioClips/LoadAudioClipSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/AudioClips/LoadAudioClipSystem.cs index c0cdf401df..d4825c88d5 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/AudioClips/LoadAudioClipSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/AudioClips/LoadAudioClipSystem.cs @@ -24,7 +24,7 @@ internal LoadAudioClipSystem(World world, IStreamableCache> FlowInternalAsync(GetAudioClipIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + protected override async UniTask> FlowInternalAsync(GetAudioClipIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { // Attempts should be always 1 as there is a repeat loop in `LoadSystemBase` AudioClip? result = await webRequestController.GetAudioClipAsync( diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/AssetPromise.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/AssetPromise.cs index c79b5b8223..19da5f37c5 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/AssetPromise.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/AssetPromise.cs @@ -39,7 +39,7 @@ public static AssetPromise Create(World world, TLoadi new () { LoadingIntention = loadingIntention, - Entity = world.Reference(world.Create(loadingIntention, partition, new StreamableLoadingState())), + Entity = world.Reference(world.Create(loadingIntention, partition, StreamableLoadingState.Create())), }; public static AssetPromise CreateFinalized(TLoadingIntention loadingIntention, StreamableLoadingResult? result) => @@ -132,6 +132,8 @@ public void ForgetLoading(World world) private void DestroyEntity(World world) { + world.Get(Entity).Dispose(); + world.Destroy(Entity); Entity = EntityReference.Null; } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/CommonLoadingArguments.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/CommonLoadingArguments.cs index ecd0d67814..df9de23760 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/CommonLoadingArguments.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/CommonLoadingArguments.cs @@ -30,8 +30,7 @@ public struct CommonLoadingArguments public readonly CancellationTokenSource CancellationTokenSource; - public CommonLoadingArguments(URLAddress url, - URLSubdirectory customEmbeddedSubDirectory = default, + public CommonLoadingArguments(URLAddress url, URLSubdirectory customEmbeddedSubDirectory = default, int timeout = StreamableLoadingDefaults.TIMEOUT, int attempts = StreamableLoadingDefaults.ATTEMPTS_COUNT, AssetSource permittedSources = AssetSource.WEB, @@ -51,8 +50,7 @@ public CommonLoadingArguments(URLAddress url, /// /// Use URLAddress instead of string /// - public CommonLoadingArguments(string url, - URLSubdirectory customEmbeddedSubDirectory = default, + public CommonLoadingArguments(string url, URLSubdirectory customEmbeddedSubDirectory = default, int timeout = StreamableLoadingDefaults.TIMEOUT, int attempts = StreamableLoadingDefaults.ATTEMPTS_COUNT, AssetSource permittedSources = AssetSource.WEB, diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/PartialLoadingState.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/PartialLoadingState.cs new file mode 100644 index 0000000000..5ab77fe727 --- /dev/null +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/PartialLoadingState.cs @@ -0,0 +1,29 @@ +using System; + +namespace ECS.StreamableLoading.Common.Components +{ + public struct PartialLoadingState + { + public Memory FullData { get; } + public readonly int FullFileSize; + + // TODO assign properly + public int NextRangeStart; + + // Add expiration time/TTL/additional data here as required + + public bool FullyDownloaded => NextRangeStart >= FullFileSize; + + public PartialLoadingState(Memory fullData, int fullFileSize) + { + FullData = fullData; + FullFileSize = fullFileSize; + NextRangeStart = 0; + } + + public void Dispose() + { + // TODO release data to the pool + } + } +} diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/PartialLoadingState.cs.meta b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/PartialLoadingState.cs.meta new file mode 100644 index 0000000000..52506365ea --- /dev/null +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/PartialLoadingState.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 7142b9f1b35b41fda6fc5badd8a2a84f +timeCreated: 1736765701 \ No newline at end of file diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/StreamableLoadingState.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/StreamableLoadingState.cs index 5a6ea888c5..4552bcd183 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/StreamableLoadingState.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Components/StreamableLoadingState.cs @@ -1,15 +1,16 @@ using DCL.Optimization.PerformanceBudgeting; +using DCL.Optimization.Pools; +using DCL.WebRequests.PartialDownload; using System; -using System.Collections.Generic; using System.Runtime.CompilerServices; -using Utility; +using UnityEngine.Pool; namespace ECS.StreamableLoading.Common.Components { /// /// Common state for all streamable types /// - public struct StreamableLoadingState + public class StreamableLoadingState { public enum Status : byte { @@ -39,6 +40,29 @@ public enum Status : byte Finished, } + private static readonly ObjectPool POOL = new (() => new StreamableLoadingState(), + actionOnGet: state => + { + state.disposed = false; + state.Value = Status.NotStarted; + }, + actionOnRelease: state => + { + state.DisposeBudgetIfExists(); + + state.PartialDownloadingData?.Dispose(); + state.PartialDownloadingData = null; + }, + collectionCheck: PoolConstants.CHECK_COLLECTIONS, + defaultCapacity: PoolConstants.ASSET_PROMISES_PER_SCENE_COUNT / 2, maxSize: PoolConstants.ASSET_PROMISES_PER_SCENE_COUNT); + + public static StreamableLoadingState Create() => + POOL.Get(); + + private bool disposed; + + internal StreamableLoadingState() { } + public Status Value { get; private set; } /// @@ -46,6 +70,11 @@ public enum Status : byte /// public IAcquiredBudget? AcquiredBudget { get; private set; } + /// + /// Is set when the partial downloading is supported for the given type of asset promise and has started + /// + public PartialLoadingState? PartialDownloadingData { get; private set; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SetAllowed(IAcquiredBudget budget) { @@ -81,7 +110,7 @@ public void StartProgress() public void Finish() { #if UNITY_EDITOR - if (Value is not Status.InProgress) + if (Value is not Status.InProgress && Value is not Status.NotStarted) throw new InvalidOperationException($"Unexpected transition from \"{Value}\" to \"Finished\""); #endif Value = Status.Finished; @@ -100,11 +129,27 @@ public void RequestReevaluate() Value = Status.NotStarted; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetChunkData(PartialLoadingState partialDownloadingData) + { + PartialDownloadingData = partialDownloadingData; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void DisposeBudgetIfExists() { AcquiredBudget?.Dispose(); AcquiredBudget = null; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Dispose() + { + if (disposed) return; + + disposed = true; + + POOL.Release(this); + } } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/AssetsLoadingUtility.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/AssetsLoadingUtility.cs index 9ef87e613a..6b4cb40117 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/AssetsLoadingUtility.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/AssetsLoadingUtility.cs @@ -1,3 +1,4 @@ +using Arch.Core; using AssetManagement; using Cysharp.Threading.Tasks; using DCL.WebRequests; @@ -12,8 +13,9 @@ namespace ECS.StreamableLoading.Common.Systems { public static class AssetsLoadingUtility { - public delegate UniTask> InternalFlowDelegate(TIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) - where TIntention: struct, ILoadingIntention; + public delegate UniTask> InternalFlowDelegate(TIntention intention, TState state, IPartitionComponent partition, CancellationToken ct) + where TIntention: struct, ILoadingIntention + where TState: class; /// /// Repeat the internal flow until attempts do not exceed or an irrecoverable error occurs @@ -21,11 +23,11 @@ public delegate UniTask> InternalFlowDelegate /// Null - if PermittedSources have value /// - public static async UniTask?> RepeatLoopAsync(this TIntention intention, - IAcquiredBudget acquiredBudget, + public static async UniTask?> RepeatLoopAsync(this TIntention intention, + TState state, IPartitionComponent partition, - InternalFlowDelegate flow, ReportData reportData, CancellationToken ct) - where TIntention: struct, ILoadingIntention + InternalFlowDelegate flow, ReportData reportData, CancellationToken ct) + where TIntention: struct, ILoadingIntention where TState: class { int attemptCount = intention.CommonArguments.Attempts; @@ -33,7 +35,7 @@ public delegate UniTask> InternalFlowDelegate : BaseUnityLoopSystem w protected readonly IStreamableCache cache; - private readonly AssetsLoadingUtility.InternalFlowDelegate cachedInternalFlowDelegate; + private readonly AssetsLoadingUtility.InternalFlowDelegate cachedInternalFlowDelegate; private readonly Query query; private readonly CancellationTokenSource cancellationTokenSource; @@ -77,6 +77,7 @@ private void Execute(Entity entity, ref StreamableLoadingState state, ref TInten EntityReference entityReference = World.Reference(entity); + //If a chunk is already loading, don't start another one, if it is a partial request it will resume from the point it was stopped if (state.Value != StreamableLoadingState.Status.Allowed) { // If state is in progress the flow was already launched and it will call FinalizeLoading on its own @@ -85,26 +86,22 @@ private void Execute(Entity entity, ref StreamableLoadingState state, ref TInten // If we don't finalize promises preemptively they are being stacked in DeferredLoadingSystem // if it's unable to keep up with their number - FinalizeLoading(entityReference, intention, null, currentSource, state.AcquiredBudget); + FinalizeLoading(entityReference, intention, null, currentSource, state); return; } - // Remove current source flag from the permitted sources - // it indicates that the current source was used - intention.RemoveCurrentSource(); - // Indicate that loading has started state.StartProgress(); - FlowAsync(entityReference, currentSource, intention, state.AcquiredBudget!, partitionComponent, cancellationTokenSource.Token).Forget(); + FlowAsync(entityReference, currentSource, intention, state, partitionComponent, cancellationTokenSource.Token).Forget(); } private async UniTask FlowAsync( EntityReference entity, AssetSource source, TIntention intention, - IAcquiredBudget acquiredBudget, + StreamableLoadingState state, IPartitionComponent partition, CancellationToken disposalCt ) @@ -120,14 +117,14 @@ CancellationToken disposalCt if (cache.OngoingRequests.SyncTryGetValue(intention.CommonArguments.GetCacheableURL(), out var cachedSource)) { // Release budget immediately, if we don't do it and load a lot of bundles with dependencies sequentially, it will be a deadlock - acquiredBudget.Release(); + state.AcquiredBudget!.Release(); // if the cached request is cancelled it does not mean failure for the new intent (requestIsNotFulfilled, result) = await cachedSource.Task.SuppressCancellationThrow(); if (requestIsNotFulfilled) { - await FlowAsync(entity, source, intention, acquiredBudget, partition, disposalCt); + await FlowAsync(entity, source, intention, state, partition, disposalCt); return; } } @@ -149,7 +146,7 @@ CancellationToken disposalCt // if this request must be cancelled by `intention.CommonArguments.CancellationToken` it will be cancelled after `if (!requestIsNotFulfilled)` if (requestIsNotFulfilled) - result = await CacheableFlowAsync(intention, acquiredBudget, partition, CancellationTokenSource.CreateLinkedTokenSource(intention.CommonArguments.CancellationToken, disposalCt).Token); + result = await CacheableFlowAsync(intention, state, partition, CancellationTokenSource.CreateLinkedTokenSource(intention.CommonArguments.CancellationToken, disposalCt).Token); if (!result.HasValue) @@ -163,33 +160,37 @@ CancellationToken disposalCt // If we don't set an exception it will spin forever result = new StreamableLoadingResult(GetReportCategory(), e); } - finally { FinalizeLoading(entity, intention, result, source, acquiredBudget); } + finally { FinalizeLoading(entity, intention, result, source, state); } } protected virtual void DisposeAbandonedResult(TAsset asset) { } private void FinalizeLoading(EntityReference entity, TIntention intention, StreamableLoadingResult? result, AssetSource source, - IAcquiredBudget? acquiredBudget) + StreamableLoadingState state) { - if (IsWorldInvalid(entity, acquiredBudget)) + if (IsWorldInvalid(entity, state.AcquiredBudget)) return; - ref StreamableLoadingState state = ref World!.TryGetRef(entity, out bool exists); + state.DisposeBudgetIfExists(); - if (!exists) + // Special path for partial downloading + if (state.PartialDownloadingData is { FullyDownloaded: false }) { - ReportHub.LogError(GetReportData(), $"Leak detected on loading {intention.ToString()} from {source}"); - - // it could be already disposed of, but it's safe to call it again - acquiredBudget?.Dispose(); + // Return the promise for re-evaluation + state.RequestReevaluate(); return; } - state.DisposeBudgetIfExists(); + // Remove current source flag from the permitted sources + // it indicates that the current source was used + // don't do it for partial requests + intention.RemoveCurrentSource(); + + World.Set(entity, intention); if (result.HasValue) - ApplyLoadedResult(entity, ref state, intention, result, source); + ApplyLoadedResult(entity, state, intention, result, source); else if (intention.IsCancelled()) { if (World.IsAlive(entity)) @@ -199,7 +200,7 @@ private void FinalizeLoading(EntityReference entity, TIntention intention, state.RequestReevaluate(); } - private void ApplyLoadedResult(Entity entity, ref StreamableLoadingState state, TIntention intention, StreamableLoadingResult? result, AssetSource source) + private void ApplyLoadedResult(Entity entity, StreamableLoadingState state, TIntention intention, StreamableLoadingResult? result, AssetSource source) { state.Finish(); @@ -235,12 +236,12 @@ private void IncreaseRefCount(in TIntention intention, TAsset asset) /// /// All exceptions are handled by the upper functions, just do pure work /// - protected abstract UniTask> FlowInternalAsync(TIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct); + protected abstract UniTask> FlowInternalAsync(TIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct); /// /// Part of the flow that can be reused by multiple intentions /// - private async UniTask?> CacheableFlowAsync(TIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + private async UniTask?> CacheableFlowAsync(TIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { var source = new UniTaskCompletionSource?>(); //AutoResetUniTaskCompletionSource?>.Create(); @@ -253,7 +254,7 @@ private void IncreaseRefCount(in TIntention intention, TAsset asset) try { - result = await RepeatLoopAsync(intention, acquiredBudget, partition, ct); + result = await RepeatLoopAsync(intention, state, partition, ct); // Ensure that we returned to the main thread await UniTask.SwitchToMainThread(ct); @@ -305,9 +306,9 @@ void TryRemoveOngoingRequest() } } - private async UniTask?> RepeatLoopAsync(TIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + private async UniTask?> RepeatLoopAsync(TIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { - StreamableLoadingResult? result = await intention.RepeatLoopAsync(acquiredBudget, partition, cachedInternalFlowDelegate, GetReportData(), ct); + StreamableLoadingResult? result = await intention.RepeatLoopAsync(state, partition, cachedInternalFlowDelegate, GetReportData(), ct); return result is { Succeeded: false } ? SetIrrecoverableFailure(intention, result.Value) : result; } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/PartialDownloadSystemBase.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/PartialDownloadSystemBase.cs new file mode 100644 index 0000000000..c07bef85df --- /dev/null +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/PartialDownloadSystemBase.cs @@ -0,0 +1,91 @@ +using Arch.Core; +using Cysharp.Threading.Tasks; +using DCL.Diagnostics; +using DCL.WebRequests; +using DCL.WebRequests.PartialDownload; +using ECS.Prioritization.Components; +using ECS.StreamableLoading.Cache; +using ECS.StreamableLoading.Common.Components; +using System; +using System.Buffers; +using System.Threading; +using UnityEngine; + +namespace ECS.StreamableLoading.Common.Systems +{ + public abstract class PartialDownloadSystemBase : LoadSystemBase + where TIntention: struct, ILoadingIntention + { + private readonly IWebRequestController webRequestController; + private readonly ArrayPool buffersPool; + + protected PartialDownloadSystemBase( + World world, + IStreamableCache cache, + IWebRequestController webRequestController, + ArrayPool buffersPool) + : base(world, cache) + { + this.webRequestController = webRequestController; + this.buffersPool = buffersPool; + } + + protected override async UniTask> FlowInternalAsync(TIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) + { + PartialLoadingState partialState = default; + PartialDownloadingData chunkData; + + byte[] partialDownloadBuffer = buffersPool.Rent(PartialDownloadingData.CHUNK_SIZE)!; + + try + { + // If the downloading has not started yet + if (state.PartialDownloadingData == null) + { + chunkData = new PartialDownloadingData(partialDownloadBuffer, 0, PartialDownloadingData.CHUNK_SIZE); + } + else + { + partialState = state.PartialDownloadingData.Value; + + chunkData = new PartialDownloadingData(partialDownloadBuffer, partialState.NextRangeStart, + Mathf.Min(partialState.FullFileSize - 1, partialState.NextRangeStart + PartialDownloadingData.CHUNK_SIZE)); + } + + chunkData = await webRequestController.GetPartialAsync( + intention.CommonArguments, + ct, + reportData: ReportCategory.PARTIAL_LOADING, + chunkData, + headersInfo: new WebRequestHeadersInfo().WithRange(chunkData.RangeStart, chunkData.RangeEnd)); + + if (state.PartialDownloadingData == null) + { + var fullDataMemory = new Memory(new byte[chunkData.FullFileSize]); + partialState = new PartialLoadingState(fullDataMemory, chunkData.FullFileSize); + } + + int finalBytesCount = chunkData.DataBuffer.Length; + + if (chunkData.RangeEnd > chunkData.FullFileSize) + finalBytesCount = chunkData.FullFileSize - chunkData.RangeStart; + + chunkData.DataBuffer.AsMemory(0, finalBytesCount).CopyTo(partialState.FullData.Slice(chunkData.RangeStart, finalBytesCount)); + partialState.NextRangeStart = chunkData.RangeEnd + 1; + + if (partialState.FullyDownloaded) + { + StreamableLoadingResult loadedResult = await ProcessCompletedData(partialState.FullData.ToArray(), intention, partition, ct, state); + state.SetChunkData(partialState); + return loadedResult; + } + + state.SetChunkData(partialState); + return default; + } + finally { buffersPool.Return(partialDownloadBuffer); } + } + + protected abstract UniTask> ProcessCompletedData(byte[] completeData, TIntention intention, IPartitionComponent partition, CancellationToken ct, StreamableLoadingState state); + } +} diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/PartialDownloadSystemBase.cs.meta b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/PartialDownloadSystemBase.cs.meta new file mode 100644 index 0000000000..36a1e46f31 --- /dev/null +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Common/Systems/PartialDownloadSystemBase.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: a6757d65ec6654d5993c68f92d3bb26f +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/DeferredLoading/DeferredLoadingSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/DeferredLoading/DeferredLoadingSystem.cs index 54aadaca0f..ac8b1fa53b 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/DeferredLoading/DeferredLoadingSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/DeferredLoading/DeferredLoadingSystem.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using UnityEngine.Pool; -using Utility; using static ECS.StreamableLoading.Common.Components.StreamableLoadingState; namespace ECS.StreamableLoading.DeferredLoading @@ -57,7 +56,7 @@ protected override void Update(float t) var intentionData = new IntentionData { - StatePointer = new ManagedTypePointer(ref state), + State = state, PartitionComponent = partition }; @@ -81,16 +80,12 @@ private void AnalyzeBudget() if (!memoryBudget.TrySpendBudget()) break; if (!releasablePerformanceLoadingBudget.TrySpendBudget(out IAcquiredBudget acquiredBudget)) break; - ref StreamableLoadingState state = ref loadingIntentions[i].StatePointer.Value; - state.SetAllowed(acquiredBudget!); + loadingIntentions[i].State.SetAllowed(acquiredBudget); } // Set the rest to forbidden for (; i < loadingIntentions.Count; i++) - { - ref StreamableLoadingState state = ref loadingIntentions[i].StatePointer.Value; - state.Forbid(); - } + loadingIntentions[i].State.Forbid(); } protected override void OnDispose() @@ -101,7 +96,7 @@ protected override void OnDispose() internal struct IntentionData { public IPartitionComponent PartitionComponent; - public ManagedTypePointer StatePointer; + public StreamableLoadingState State; } } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/GLTF/LoadGLTFSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/GLTF/LoadGLTFSystem.cs index 97f3cff7ae..01349b5d66 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/GLTF/LoadGLTFSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/GLTF/LoadGLTFSystem.cs @@ -33,7 +33,7 @@ internal LoadGLTFSystem(World world, IStreamableCache> FlowInternalAsync(GetGLTFIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + protected override async UniTask> FlowInternalAsync(GetGLTFIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { var reportData = new ReportData(GetReportCategory()); @@ -43,7 +43,7 @@ protected override async UniTask> FlowInternal new Exception("The content to download couldn't be found")); // Acquired budget is released inside GLTFastDownloadedProvider once the GLTF has been fetched - GltFastDownloadProvider gltfDownloadProvider = new GltFastDownloadProvider(World, sceneData, partition, intention.Name!, reportData, webRequestController, acquiredBudget); + var gltfDownloadProvider = new GltFastDownloadProvider(World, sceneData, partition, intention.Name!, reportData, webRequestController, state.AcquiredBudget!); var gltfImport = new GltfImport( downloadProvider: gltfDownloadProvider, logger: gltfConsoleLogger, diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/NFTShapes/LoadNFTShapeSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/NFTShapes/LoadNFTShapeSystem.cs index b55e39e0f9..3b90db51ab 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/NFTShapes/LoadNFTShapeSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/NFTShapes/LoadNFTShapeSystem.cs @@ -11,44 +11,46 @@ using ECS.StreamableLoading.Common.Systems; using ECS.StreamableLoading.NFTShapes.DTOs; using ECS.StreamableLoading.Textures; +using Plugins.TexturesFuse.TexturesServerWrap; using Plugins.TexturesFuse.TexturesServerWrap.Unzips; using System; +using System.Buffers; using System.Threading; +using Utility.Types; namespace ECS.StreamableLoading.NFTShapes { [UpdateInGroup(typeof(StreamableLoadingGroup))] [LogCategory(ReportCategory.NFT_SHAPE_WEB_REQUEST)] - public partial class LoadNFTShapeSystem : LoadSystemBase + public partial class LoadNFTShapeSystem : PartialDownloadSystemBase { private readonly IWebRequestController webRequestController; + private readonly ITexturesFuse texturesFuse; - public LoadNFTShapeSystem(World world, IStreamableCache cache, IWebRequestController webRequestController) : base(world, cache) + public LoadNFTShapeSystem( + World world, + IStreamableCache cache, + IWebRequestController webRequestController, + ArrayPool buffersPool, + ITexturesFuse texturesFuse) : base(world, cache, webRequestController, buffersPool) { this.webRequestController = webRequestController; + this.texturesFuse = texturesFuse; } - protected override async UniTask> FlowInternalAsync(GetNFTShapeIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + protected override async UniTask> FlowInternalAsync(GetNFTShapeIntention intention, StreamableLoadingState state, IPartitionComponent partition, CancellationToken ct) { string imageUrl = await ImageUrlAsync(intention.CommonArguments, ct); + var arguments = intention.CommonArguments; + arguments.URL = URLAddress.FromString(imageUrl); + intention.CommonArguments = arguments; + return await base.FlowInternalAsync(intention, state, partition, ct); + } - // texture request - // Attempts should be always 1 as there is a repeat loop in `LoadSystemBase` - var result = await webRequestController.GetTextureAsync( - new CommonLoadingArguments(URLAddress.FromString(imageUrl), attempts: 1), - new GetTextureArguments(TextureType.Albedo), - new GetTextureWebRequest.CreateTextureOp(GetNFTShapeIntention.WRAP_MODE, GetNFTShapeIntention.FILTER_MODE), - ct, - GetReportData() - ); - - if (result == null) - return new StreamableLoadingResult( - GetReportData(), - new Exception($"Error loading texture from url {intention.CommonArguments.URL}") - ); - - return new StreamableLoadingResult(new Texture2DData(result)); + protected override async UniTask> ProcessCompletedData(byte[] completeData, GetNFTShapeIntention intention, IPartitionComponent partition, CancellationToken ct, StreamableLoadingState state) + { + EnumResult textureFromBytesAsync = await texturesFuse.TextureFromBytesAsync(completeData, TextureType.Albedo, ct); + return new StreamableLoadingResult(new Texture2DData(textureFromBytesAsync.Value)); } private async UniTask ImageUrlAsync(CommonArguments commonArguments, CancellationToken ct) diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Tests/MultipleLoadSystemShould.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Tests/MultipleLoadSystemShould.cs index ec451f243f..93ada41cd9 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Tests/MultipleLoadSystemShould.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Tests/MultipleLoadSystemShould.cs @@ -2,8 +2,10 @@ using DCL.WebRequests; using ECS.Prioritization.Components; using ECS.StreamableLoading.Textures; +using NSubstitute; using NUnit.Framework; using Plugins.TexturesFuse.TexturesServerWrap.Unzips; +using System.Buffers; using System.Collections.Generic; using UnityEngine; using Promise = ECS.StreamableLoading.Common.AssetPromise; @@ -13,6 +15,7 @@ namespace ECS.StreamableLoading.Tests [TestFixture] public class MultipleLoadSystemShould { + private readonly ArrayPool buffersPool = ArrayPool.Create(1024 * 1024, 100); private string successPath => $"file://{Application.dataPath + "/../TestResources/Images/alphaTexture.png"}"; private const int REQUESTS_COUNT = 1000; @@ -22,7 +25,7 @@ public void MultipleLoadsShould() { // set-up var world = World.Create(); - var loadSystem = new LoadTextureSystem(world, new TexturesCache(), IWebRequestController.DEFAULT); + var loadSystem = new LoadTextureSystem(world, new TexturesCache(), IWebRequestController.DEFAULT, buffersPool, Substitute.For()); var promises = new List(REQUESTS_COUNT); for (var i = 0; i < REQUESTS_COUNT; i++) promises.Add(NewPromise(world)); diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadGlobalTextureSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadGlobalTextureSystem.cs index 329851208e..0defb9c4d0 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadGlobalTextureSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadGlobalTextureSystem.cs @@ -4,6 +4,8 @@ using DCL.Diagnostics; using DCL.WebRequests; using ECS.StreamableLoading.Cache; +using Plugins.TexturesFuse.TexturesServerWrap.Unzips; +using System.Buffers; namespace ECS.StreamableLoading.Textures { @@ -14,8 +16,8 @@ namespace ECS.StreamableLoading.Textures [LogCategory(ReportCategory.TEXTURES)] public partial class LoadGlobalTextureSystem : LoadTextureSystem { - internal LoadGlobalTextureSystem(World world, IStreamableCache cache, IWebRequestController webRequestController) : base( - world, cache, webRequestController + internal LoadGlobalTextureSystem(World world, IStreamableCache cache, IWebRequestController webRequestController, ArrayPool buffersPools, ITexturesFuse texturesFuse) : base( + world, cache, webRequestController, buffersPools, texturesFuse ) { } } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadTextureSystem.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadTextureSystem.cs index b15d4501b9..b5d526564e 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadTextureSystem.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/LoadTextureSystem.cs @@ -3,43 +3,40 @@ using Cysharp.Threading.Tasks; using DCL.WebRequests; using DCL.Diagnostics; -using DCL.Optimization.PerformanceBudgeting; -using DCL.Utilities.Extensions; using ECS.Prioritization.Components; using ECS.StreamableLoading.Cache; using ECS.StreamableLoading.Common.Components; using ECS.StreamableLoading.Common.Systems; -using ECS.Unity.Textures.Utils; +using Plugins.TexturesFuse.TexturesServerWrap; +using Plugins.TexturesFuse.TexturesServerWrap.Unzips; using System; +using System.Buffers; using System.Threading; +using Utility.Types; namespace ECS.StreamableLoading.Textures { [UpdateInGroup(typeof(StreamableLoadingGroup))] [LogCategory(ReportCategory.TEXTURES)] - public partial class LoadTextureSystem : LoadSystemBase + public partial class LoadTextureSystem : PartialDownloadSystemBase { - private readonly IWebRequestController webRequestController; + private readonly ITexturesFuse texturesFuse; - internal LoadTextureSystem(World world, IStreamableCache cache, IWebRequestController webRequestController) : base(world, cache) + public LoadTextureSystem( + World world, + IStreamableCache cache, + IWebRequestController webRequestController, + ArrayPool buffersPool, + ITexturesFuse texturesFuse + ) : base(world, cache, webRequestController, buffersPool) { - this.webRequestController = webRequestController; + this.texturesFuse = texturesFuse; } - protected override async UniTask> FlowInternalAsync(GetTextureIntention intention, IAcquiredBudget acquiredBudget, IPartitionComponent partition, CancellationToken ct) + protected override async UniTask> ProcessCompletedData(byte[] completeData, GetTextureIntention intention, IPartitionComponent partition, CancellationToken ct, StreamableLoadingState state) { - if (intention.IsVideoTexture) throw new NotSupportedException($"{nameof(LoadTextureSystem)} does not support video textures. They should be handled by {nameof(VideoTextureUtils)}"); - - // Attempts should be always 1 as there is a repeat loop in `LoadSystemBase` - var result = await webRequestController.GetTextureAsync( - intention.CommonArguments, - new GetTextureArguments(intention.TextureType), - GetTextureWebRequest.CreateTexture(intention.WrapMode, intention.FilterMode), - ct, - GetReportData() - ); - - return new StreamableLoadingResult(new Texture2DData(result.EnsureNotNull())); + EnumResult textureFromBytesAsync = await texturesFuse.TextureFromBytesAsync(completeData, TextureType.Albedo, ct); + return new StreamableLoadingResult(new Texture2DData(textureFromBytesAsync.Value)); } } } diff --git a/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/Tests/LoadTextureSystemShould.cs b/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/Tests/LoadTextureSystemShould.cs index 4155e0e069..4124fcdb7d 100644 --- a/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/Tests/LoadTextureSystemShould.cs +++ b/Explorer/Assets/Scripts/ECS/StreamableLoading/Textures/Tests/LoadTextureSystemShould.cs @@ -1,8 +1,10 @@ using ECS.StreamableLoading.Common.Components; using ECS.StreamableLoading.Tests; using ECS.TestSuite; +using NSubstitute; using NUnit.Framework; using Plugins.TexturesFuse.TexturesServerWrap.Unzips; +using System.Buffers; using UnityEngine; namespace ECS.StreamableLoading.Textures.Tests @@ -10,6 +12,7 @@ namespace ECS.StreamableLoading.Textures.Tests [TestFixture] public class LoadTextureSystemShould : LoadSystemBaseShould { + private readonly ArrayPool buffersPool = ArrayPool.Create(1024 * 1024, 100); private string successPath => $"file://{Application.dataPath + "/../TestResources/Images/alphaTexture.png"}"; private string failPath => $"file://{Application.dataPath + "/../TestResources/Images/non_existing.png"}"; private string wrongTypePath => $"file://{Application.dataPath + "/../TestResources/CRDT/arraybuffer.test"}"; @@ -24,7 +27,7 @@ protected override GetTextureIntention CreateWrongTypeIntention() => new () { CommonArguments = new CommonLoadingArguments(wrongTypePath) }; protected override LoadTextureSystem CreateSystem() => - new (world, cache, TestWebRequestController.INSTANCE); + new (world, cache, TestWebRequestController.INSTANCE, buffersPool, Substitute.For()); protected override void AssertSuccess(Texture2DData data) { diff --git a/Explorer/Assets/Scripts/Global/StaticContainer.cs b/Explorer/Assets/Scripts/Global/StaticContainer.cs index e827f4c140..c7d9a98f94 100644 --- a/Explorer/Assets/Scripts/Global/StaticContainer.cs +++ b/Explorer/Assets/Scripts/Global/StaticContainer.cs @@ -207,7 +207,7 @@ await UniTask.WhenAll( container.FeatureFlagsCache); var assetBundlePlugin = new AssetBundlesPlugin(reportHandlingSettings, container.CacheCleaner, container.WebRequestsContainer.WebRequestController); - var textureResolvePlugin = new TexturesLoadingPlugin(container.WebRequestsContainer.WebRequestController, container.CacheCleaner); + var textureResolvePlugin = new TexturesLoadingPlugin(container.WebRequestsContainer.WebRequestController, container.CacheCleaner, texturesFuse); ExtendedObjectPool videoTexturePool = VideoTextureFactory.CreateVideoTexturesPool(); @@ -223,7 +223,7 @@ await UniTask.WhenAll( { new TransformsPlugin(sharedDependencies, exposedPlayerTransform, exposedGlobalDataContainer.ExposedCameraData), new BillboardPlugin(exposedGlobalDataContainer.ExposedCameraData), - new NFTShapePlugin(decentralandUrlsSource, container.assetsProvisioner, sharedDependencies.FrameTimeBudget, componentsContainer.ComponentPoolsRegistry, container.WebRequestsContainer.WebRequestController, container.CacheCleaner), + new NFTShapePlugin(decentralandUrlsSource, container.assetsProvisioner, sharedDependencies.FrameTimeBudget, componentsContainer.ComponentPoolsRegistry, container.WebRequestsContainer.WebRequestController, container.CacheCleaner, texturesFuse), new TextShapePlugin(sharedDependencies.FrameTimeBudget, container.CacheCleaner, componentsContainer.ComponentPoolsRegistry), new MaterialsPlugin(sharedDependencies, videoTexturePool), textureResolvePlugin,