Resource aggregate rework

CHANGELOG.md

@@ -3,19 +3,54 @@ Release built: _not released yet_
 
 > [!CAUTION]
 > **Breaking Changes:**
-> - Changed ordering of entity metadata returned from `/state/entity/metadata`, `/state/entity/page/metadata` endpoints. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending.
-
-
-### Bug fixes
+> - Changed ordering of entity metadata. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending. Affected endpoints:
+>   - `/state/entity/metadata`
+>   - `/state/entity/page/metadata`
+> - Changed ordering of fungible and non fungible resources. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending. Affected endpoints:
+>   - `/state/entity/details`
+>   - `/state/entity/page/fungibles/`
+>   - `/state/entity/page/non-fungibles/`
+> - Changed ordering of vaults when using vault aggregation level. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending.  Affected endpoints:
+>   - `/state/entity/details`
+>   - `/state/entity/page/fungibles/`
+>   - `/state/entity/page/fungible-vaults/`
+>   - `/state/entity/page/non-fungibles/`
+>   - `/state/entity/page/non-fungible-vaults/`
+> - Changed ordering of non fungible ids. Entries are no longer ordered by their last modification state version but rather by their first appearance on the network, descending. Affected endpoints:
+>   - `/state/entity/page/non-fungible-vault/ids`
+>   - `/state/entity/details` (when using `non_fungible_include_nfids` opt-in)
+>   - `/state/entity/page/non-fungibles/` (when using `non_fungible_include_nfids` opt-in)
+>   - `/state/entity/page/non-fungible-vaults/` (when using `non_fungible_include_nfids` opt-in)
+> -  Existing non fungible vaults with no items will no longer return `items: null` and will return an empty array `items: []` instead, as we do in all other collections. Affected endpoints:
+>   - `/state/entity/page/non-fungible-vault/ids`
+>   - `/state/entity/details` (when using `non_fungible_include_nfids` opt-in)
+>   - `/state/entity/page/non-fungibles/` (when using `non_fungible_include_nfids` opt-in)
+>   - `/state/entity/page/non-fungible-vaults/` (when using `non_fungible_include_nfids` opt-in)
 
 ### API Changes
 
+
 ### Database changes
-- Refactored entity metadata aggregate. Queries for metadata follow a similar strategy as key value stores and utilize `_entry_definition`, `_entry_history`, and `_totals_history` tables to return data
-  - Removed `entity_metadata_aggregate_history` table.
-  - New `entity_metadata_totals_history` table which holds total counts of metadata per entity.
-  - New `entity_metadata_entry_definition` table which holds information about all the metadata keys ever created for a given entity.
-  - Renamed `entity_metadata_history` to `entity_metadata_entry_history`, replaced `entity_id` and `key` columns with `entity_metadata_entry_definition_id`.
+- Refactored multiple aggregates. Queries follow a similar strategy as key value stores and utilize `_entry_definition`, `_entry_history`, and `_totals_history` tables to return data
+    - Metadata
+        - Removed `entity_metadata_aggregate_history` table.
+        - New `entity_metadata_entry_definition` table, which holds information about all the metadata keys ever created for a given entity.
+        - Renamed `entity_metadata_history` to `entity_metadata_entry_history`, replaced `entity_id` and `key` columns with `entity_metadata_entry_definition_id`. Holds history of given metadata key at a given state version.
+        - New `entity_metadata_totals_history` table, which holds total counts of metadata per entity.
+    - Resource globally aggregated
+        - Removed `entity_resource_aggregate_history` table.
+        - New `entity_resource_entry_definition` table, which holds information about all resources which have ever been held by a given global entity.
+        - New `entity_resource_balance_history` table, which holds the sum of globally aggregated resource held by a global entity at a given state version.
+        - New `entity_resource_totals_history` table, which holds total count of different resources under a given global entity at a given state version.
+    - Resource vault aggregated
+        - Removed `entity_resource_aggregated_vaults_history` and `entity_resource_vault_aggregate_history` tables.
+        - New `entity_resource_vault_entry_definition` table, which holds information about vaults of a given resource held under a given global entity.
+        - New `entity_resource_vault_totals_history` table, which holds total count of all vaults of a given resource held under a given global entity at a given state version.
+    - Vault content
+        - New `non_fungible_vault_entry_definition` table, which holds information about non fungible held by a given vault.
+        - New `non_fungible_vault_entry_history` table which holds history of given non fungible inside vault.
+        - Renamed `entity_vault_history` to `vault_balance_history`. Holds information about vault content (amount of fungibles or count of non fungible ids inside vault) at a given state version.
+
 
 ## 1.7.3
 Release built: 26.09.2024

apps/DataAggregator/DataAggregatorStartup.cs

@@ -69,7 +69,7 @@
 using Newtonsoft.Json;
 using Prometheus;
 using RadixDlt.NetworkGateway.DataAggregator;
-using RadixDlt.NetworkGateway.PostgresIntegration;
+using RadixDlt.NetworkGateway.PostgresIntegration.IoC;
 using RadixDlt.NetworkGateway.PrometheusIntegration;
 
 namespace DataAggregator;

apps/DatabaseMigrations/Program.cs

@@ -65,7 +65,7 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
-using RadixDlt.NetworkGateway.PostgresIntegration;
+using RadixDlt.NetworkGateway.PostgresIntegration.IoC;
 using System.Threading.Tasks;
 
 namespace DatabaseMigrations;

apps/GatewayApi/GatewayApiStartup.cs

@@ -72,7 +72,7 @@
 using Prometheus;
 using RadixDlt.NetworkGateway.GatewayApi;
 using RadixDlt.NetworkGateway.GatewayApi.Services;
-using RadixDlt.NetworkGateway.PostgresIntegration;
+using RadixDlt.NetworkGateway.PostgresIntegration.IoC;
 using RadixDlt.NetworkGateway.PrometheusIntegration;
 using System;
 

src/RadixDlt.NetworkGateway.Abstractions/Model/EntityRelationship.cs

@@ -74,8 +74,6 @@ public enum EntityRelationship
 
     VaultToResource,
 
-    RoyaltyVaultOfComponent,
-
     // Validators
 
     ValidatorToStakeVault,
@@ -86,6 +84,12 @@ public enum EntityRelationship
     StakeUnitOfValidator,
     ClaimTokenOfValidator,
 
+    // Royalties
+
+    EntityToRoyaltyVault,
+
+    RoyaltyVaultOfEntity,
+
     // Account Lockers (used on related Vaults and KeyValueStores)
 
     AccountLockerOfLocker,

src/RadixDlt.NetworkGateway.Abstractions/Numerics/TokenAmount.cs

@@ -73,6 +73,9 @@ namespace RadixDlt.NetworkGateway.Abstractions.Numerics;
     public const string StringForNaN = "NaN";
 
     public static readonly TokenAmount Zero;
+    public static readonly TokenAmount One;
+    public static readonly TokenAmount MinusOne;
+    public static readonly TokenAmount _100;
     public static readonly TokenAmount NaN;
     public static readonly TokenAmount MaxValue;
 
@@ -83,9 +86,11 @@ namespace RadixDlt.NetworkGateway.Abstractions.Numerics;
     static TokenAmount()
     {
         _divisor = BigInteger.Pow(10, DecimalPrecision);
-
         MaxValue = new TokenAmount(BigInteger.Pow(2, 192), 0);
         Zero = new TokenAmount(0);
+        One = FromDecimalString("1");
+        MinusOne = FromDecimalString("-1");
+        _100 = FromDecimalString("100");
         NaN = new TokenAmount(true);
     }
 

src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultAccountLockerHandler.cs

@@ -76,16 +76,16 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Handlers;
 internal class DefaultAccountLockerHandler : IAccountLockerHandler
 {
     private readonly ILedgerStateQuerier _ledgerStateQuerier;
-    private readonly IEntityStateQuerier _entityStateQuerier;
+    private readonly IAccountLockerQuerier _accountLockerQuerier;
     private readonly IOptionsSnapshot<EndpointOptions> _endpointConfiguration;
 
     public DefaultAccountLockerHandler(
         ILedgerStateQuerier ledgerStateQuerier,
-        IEntityStateQuerier entityStateQuerier,
+        IAccountLockerQuerier accountLockerQuerier,
         IOptionsSnapshot<EndpointOptions> endpointConfiguration)
     {
         _ledgerStateQuerier = ledgerStateQuerier;
-        _entityStateQuerier = entityStateQuerier;
+        _accountLockerQuerier = accountLockerQuerier;
         _endpointConfiguration = endpointConfiguration;
     }
 
@@ -96,7 +96,7 @@ public DefaultAccountLockerHandler(
         var limit = _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage);
         var pageRequest = new IEntityStateQuerier.AccountLockerPageRequest(new AccountLockerAddress((EntityAddress)request.LockerAddress, (EntityAddress)request.AccountAddress), cursor, limit);
 
-        return await _entityStateQuerier.AccountLockerVaultsPage(pageRequest, ledgerState, token);
+        return await _accountLockerQuerier.AccountLockerVaultsPage(pageRequest, ledgerState, token);
     }
 
     public async Task<GatewayModel.StateAccountLockersTouchedAtResponse?> TouchedAt(GatewayModel.StateAccountLockersTouchedAtRequest request, CancellationToken token = default)
@@ -106,6 +106,6 @@ public DefaultAccountLockerHandler(
             .Select(l => new AccountLockerAddress((EntityAddress)l.LockerAddress, (EntityAddress)l.AccountAddress))
             .ToList();
 
-        return await _entityStateQuerier.AccountLockersTouchedAt(accountLockers, atLedgerState, token);
+        return await _accountLockerQuerier.AccountLockersTouchedAt(accountLockers, atLedgerState, token);
     }
 }

src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultEntityHandler.cs

@@ -103,9 +103,9 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
     {
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
 
-        var pageRequest = new IEntityStateQuerier.MetadataPageRequest(
+        var pageRequest = new IEntityStateQuerier.PageRequestByCursor(
             Address: (EntityAddress)request.Address,
-            Cursor: !string.IsNullOrEmpty(request.Cursor) ? GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor) : null,
+            Cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 
@@ -130,9 +130,9 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
         var aggregatePerVault = request.AggregationLevel == GatewayModel.ResourceAggregationLevel.Vault;
 
-        var pageRequest = new IEntityStateQuerier.PageRequest(
+        var pageRequest = new IEntityStateQuerier.PageRequestByCursor(
             Address: (EntityAddress)request.Address,
-            Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0,
+            Cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 
@@ -146,7 +146,7 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
         var pageRequest = new IEntityStateQuerier.ResourceVaultsPageRequest(
             Address: (EntityAddress)request.Address,
             ResourceAddress: (EntityAddress)request.ResourceAddress,
-            Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0,
+            Cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 
@@ -158,9 +158,9 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
         var aggregatePerVault = request.AggregationLevel == GatewayModel.ResourceAggregationLevel.Vault;
 
-        var pageRequest = new IEntityStateQuerier.PageRequest(
+        var pageRequest = new IEntityStateQuerier.PageRequestByCursor(
             Address: (EntityAddress)request.Address,
-            Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0,
+            Cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 
@@ -177,7 +177,7 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
         var pageRequest = new IEntityStateQuerier.ResourceVaultsPageRequest(
             Address: (EntityAddress)request.Address,
             ResourceAddress: (EntityAddress)request.ResourceAddress,
-            Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0,
+            Cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 
@@ -190,9 +190,9 @@ public DefaultEntityHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityState
     {
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
 
-        var pageRequest = new IEntityStateQuerier.PageRequest(
+        var pageRequest = new IEntityStateQuerier.PageRequestByCursor(
             Address: (EntityAddress)request.Address,
-            Offset: GatewayModel.OffsetCursor.FromCursorString(request.Cursor)?.Offset ?? 0,
+            Cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             Limit: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage)
         );
 

src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultExtensionsHandler.cs

@@ -72,13 +72,13 @@
 
 namespace RadixDlt.NetworkGateway.GatewayApi.Handlers;
 
-internal class DefaultExtensionsHandler(IExtensionsQuerier extensionsQuerier, IOptionsSnapshot<EndpointOptions> endpointConfiguration) : IExtensionsHandler
+internal class DefaultExtensionsHandler(IResourceHoldersQuerier resourceHoldersQuerier, IOptionsSnapshot<EndpointOptions> endpointConfiguration) : IExtensionsHandler
 {
     public async Task<GatewayModel.ResourceHoldersResponse> ResourceHolders(GatewayModel.ResourceHoldersRequest request, CancellationToken token)
     {
         var cursor = GatewayModel.ResourceHoldersCursor.FromCursorString(request.Cursor);
 
-        return await extensionsQuerier.ResourceHolders(
+        return await resourceHoldersQuerier.ResourceHolders(
             (EntityAddress)request.ResourceAddress,
             endpointConfiguration.Value.ResolveResourceHoldersPageSize(request.LimitPerPage),
             cursor,

src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultNonFungibleHandler.cs

@@ -75,22 +75,22 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Handlers;
 internal class DefaultNonFungibleHandler : INonFungibleHandler
 {
     private readonly ILedgerStateQuerier _ledgerStateQuerier;
-    private readonly IEntityStateQuerier _entityStateQuerier;
+    private readonly INonFungibleQuerier _nonFungibleQuerier;
     private readonly IOptionsSnapshot<EndpointOptions> _endpointConfiguration;
 
-    public DefaultNonFungibleHandler(ILedgerStateQuerier ledgerStateQuerier, IEntityStateQuerier entityStateQuerier, IOptionsSnapshot<EndpointOptions> endpointConfiguration)
+    public DefaultNonFungibleHandler(ILedgerStateQuerier ledgerStateQuerier, INonFungibleQuerier nonFungibleQuerier, IOptionsSnapshot<EndpointOptions> endpointConfiguration)
     {
         _ledgerStateQuerier = ledgerStateQuerier;
-        _entityStateQuerier = entityStateQuerier;
+        _nonFungibleQuerier = nonFungibleQuerier;
         _endpointConfiguration = endpointConfiguration;
     }
 
     public async Task<GatewayModel.StateNonFungibleIdsResponse> Ids(GatewayModel.StateNonFungibleIdsRequest request, CancellationToken token = default)
     {
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
 
-        return await _entityStateQuerier.NonFungibleIds(
-            nonFungibleResourceAddress: (EntityAddress)request.ResourceAddress,
+        return await _nonFungibleQuerier.NonFungibleIds(
+            resourceAddress: (EntityAddress)request.ResourceAddress,
             ledgerState: ledgerState,
             cursor: GatewayModel.IdBoundaryCoursor.FromCursorString(request.Cursor),
             pageSize: _endpointConfiguration.Value.ResolvePageSize(request.LimitPerPage),
@@ -101,13 +101,13 @@ public DefaultNonFungibleHandler(ILedgerStateQuerier ledgerStateQuerier, IEntity
     {
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
 
-        return await _entityStateQuerier.NonFungibleIdData((EntityAddress)request.ResourceAddress, request.NonFungibleIds, ledgerState, token);
+        return await _nonFungibleQuerier.NonFungibleIdData((EntityAddress)request.ResourceAddress, request.NonFungibleIds, ledgerState, token);
     }
 
     public async Task<GatewayModel.StateNonFungibleLocationResponse> Location(GatewayModel.StateNonFungibleLocationRequest request, CancellationToken token = default)
     {
         var ledgerState = await _ledgerStateQuerier.GetValidLedgerStateForReadRequest(request.AtLedgerState, token);
 
-        return await _entityStateQuerier.NonFungibleIdLocation((EntityAddress)request.ResourceAddress, request.NonFungibleIds, ledgerState, token);
+        return await _nonFungibleQuerier.NonFungibleIdLocation((EntityAddress)request.ResourceAddress, request.NonFungibleIds, ledgerState, token);
     }
 }

src/RadixDlt.NetworkGateway.GatewayApi/Handlers/DefaultPackageHandler.cs

@@ -75,16 +75,16 @@ namespace RadixDlt.NetworkGateway.GatewayApi.Handlers;
 internal class DefaultPackageHandler : IPackageHandler
 {
     private readonly ILedgerStateQuerier _ledgerStateQuerier;
-    private readonly IEntityStateQuerier _entityStateQuerier;
+    private readonly IPackageQuerier _packageQuerier;
     private readonly IOptionsSnapshot<EndpointOptions> _endpointConfiguration;
 
     public DefaultPackageHandler(
         ILedgerStateQuerier ledgerStateQuerier,
-        IEntityStateQuerier entityStateQuerier,
+        IPackageQuerier packageQuerier,
         IOptionsSnapshot<EndpointOptions> endpointConfiguration)
     {
         _ledgerStateQuerier = ledgerStateQuerier;
-        _entityStateQuerier = entityStateQuerier;
+        _packageQuerier = packageQuerier;
         _endpointConfiguration = endpointConfiguration;
     }
 
@@ -97,7 +97,7 @@ public DefaultPackageHandler(
             Limit: _endpointConfiguration.Value.ResolveHeavyPageSize(request.LimitPerPage)
         );
 
-        return await _entityStateQuerier.PackageBlueprints(pageRequest, ledgerState, token);
+        return await _packageQuerier.PackageBlueprints(pageRequest, ledgerState, token);
     }
 
     public async Task<GatewayModel.StatePackageCodePageResponse?> Codes(GatewayModel.StatePackageCodePageRequest request, CancellationToken token = default)
@@ -109,6 +109,6 @@ public DefaultPackageHandler(
             Limit: _endpointConfiguration.Value.ResolveHeavyPageSize(request.LimitPerPage)
         );
 
-        return await _entityStateQuerier.PackageCodes(pageRequest, ledgerState, token);
+        return await _packageQuerier.PackageCodes(pageRequest, ledgerState, token);
     }
 }