From 82d657d5bf3d16a16616934b35e6a5b57e75bc70 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Wed, 31 Mar 2021 12:55:38 +0300 Subject: [PATCH 01/35] Add bigmaps indexing --- Tzkt.Data/Models/AppState.cs | 3 + Tzkt.Data/Models/Blocks/BlockEvents.cs | 3 +- Tzkt.Data/Models/Scripts/BigMap.cs | 75 +++ Tzkt.Data/Models/Scripts/BigMapKey.cs | 75 +++ Tzkt.Data/Models/Scripts/BigMapUpdate.cs | 69 +++ Tzkt.Data/Tzkt.Data.csproj | 2 +- Tzkt.Data/TzktContext.cs | 12 + Tzkt.Sync/Extensions/NetezosExtension.cs | 29 + .../Handlers/Proto1/Commits/BigMapCommit.cs | 513 ++++++++++++++++++ .../Commits/Operations/OriginationsCommit.cs | 85 ++- .../Commits/Operations/TransactionsCommit.cs | 77 +++ .../Handlers/Proto1/Proto1Handler.cs | 17 +- .../Handlers/Proto2/Commits/BigMapCommit.cs | 7 + .../Commits/Operations/TransactionsCommit.cs | 24 +- .../Handlers/Proto2/Proto2Handler.cs | 17 +- .../Handlers/Proto3/Commits/BigMapCommit.cs | 7 + .../Commits/Operations/TransactionsCommit.cs | 2 +- .../Handlers/Proto3/Proto3Handler.cs | 17 +- .../Handlers/Proto4/Commits/BigMapCommit.cs | 7 + .../Handlers/Proto4/Proto4Handler.cs | 17 +- .../Proto5/Activation/ProtoActivator.cs | 69 +++ .../Handlers/Proto5/Commits/BigMapCommit.cs | 7 + .../Commits/Operations/OriginationsCommit.cs | 12 +- .../Commits/Operations/TransactionsCommit.cs | 14 + .../Handlers/Proto5/Proto5Handler.cs | 24 +- .../Handlers/Proto6/Commits/BigMapCommit.cs | 7 + .../Handlers/Proto6/Proto6Handler.cs | 22 +- .../Handlers/Proto7/Commits/BigMapCommit.cs | 7 + .../Handlers/Proto7/Proto7Handler.cs | 22 +- .../Handlers/Proto8/Commits/BigMapCommit.cs | 7 + .../Handlers/Proto8/Proto8Handler.cs | 22 +- Tzkt.Sync/Protocols/Helpers/BigMapDiff.cs | 76 +++ Tzkt.Sync/Services/Cache/AppStateCache.cs | 18 + Tzkt.Sync/Services/Cache/BigMapKeysCache.cs | 84 +++ Tzkt.Sync/Services/Cache/BigMapsCache.cs | 77 +++ Tzkt.Sync/Services/Cache/CacheService.cs | 6 + Tzkt.Sync/Services/Cache/SchemasCache.cs | 7 +- Tzkt.Sync/Services/Cache/StoragesCache.cs | 11 +- Tzkt.Sync/Tzkt.Sync.csproj | 4 +- 39 files changed, 1494 insertions(+), 60 deletions(-) create mode 100644 Tzkt.Data/Models/Scripts/BigMap.cs create mode 100644 Tzkt.Data/Models/Scripts/BigMapKey.cs create mode 100644 Tzkt.Data/Models/Scripts/BigMapUpdate.cs create mode 100644 Tzkt.Sync/Extensions/NetezosExtension.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto2/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto3/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto4/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto5/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto6/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto7/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto8/Commits/BigMapCommit.cs create mode 100644 Tzkt.Sync/Protocols/Helpers/BigMapDiff.cs create mode 100644 Tzkt.Sync/Services/Cache/BigMapKeysCache.cs create mode 100644 Tzkt.Sync/Services/Cache/BigMapsCache.cs diff --git a/Tzkt.Data/Models/AppState.cs b/Tzkt.Data/Models/AppState.cs index 667f2aa9b..e338b0a4e 100644 --- a/Tzkt.Data/Models/AppState.cs +++ b/Tzkt.Data/Models/AppState.cs @@ -22,6 +22,9 @@ public class AppState public int AccountCounter { get; set; } public int OperationCounter { get; set; } public int ManagerCounter { get; set; } + public int BigMapCounter { get; set; } + public int BigMapKeyCounter { get; set; } + public int BigMapUpdateCounter { get; set; } #region entities count public int CommitmentsCount { get; set; } diff --git a/Tzkt.Data/Models/Blocks/BlockEvents.cs b/Tzkt.Data/Models/Blocks/BlockEvents.cs index 3aebc6991..8ed1ddf53 100644 --- a/Tzkt.Data/Models/Blocks/BlockEvents.cs +++ b/Tzkt.Data/Models/Blocks/BlockEvents.cs @@ -14,6 +14,7 @@ public enum BlockEvents NewAccounts = 0b_0000_0010_0000, BalanceSnapshot = 0b_0000_0100_0000, SmartContracts = 0b_0000_1000_0000, - DelegatorContracts = 0b_0001_0000_0000 + DelegatorContracts = 0b_0001_0000_0000, + Bigmaps = 0b_0010_0000_0000 } } diff --git a/Tzkt.Data/Models/Scripts/BigMap.cs b/Tzkt.Data/Models/Scripts/BigMap.cs new file mode 100644 index 000000000..e2d649d30 --- /dev/null +++ b/Tzkt.Data/Models/Scripts/BigMap.cs @@ -0,0 +1,75 @@ +using Microsoft.EntityFrameworkCore; +using Netezos.Contracts; +using Netezos.Encoding; +using System.Collections.Generic; + +namespace Tzkt.Data.Models +{ + public class BigMap + { + public int Id { get; set; } + public int Ptr { get; set; } + public int ContractId { get; set; } + public string StoragePath { get; set; } + public bool Active { get; set; } + + public byte[] KeyType { get; set; } + public byte[] ValueType { get; set; } + + public int FirstLevel { get; set; } + public int LastLevel { get; set; } + public int TotalKeys { get; set; } + public int ActiveKeys { get; set; } + public int Updates { get; set; } + + #region schema + BigMapSchema _Schema = null; + public BigMapSchema Schema + { + get + { + _Schema ??= new BigMapSchema(new MichelinePrim + { + Prim = PrimType.big_map, + Args = new List + { + Micheline.FromBytes(KeyType), + Micheline.FromBytes(ValueType) + } + }); + return _Schema; + } + } + #endregion + } + + public static class BigMapModel + { + public static void BuildBigMapModel(this ModelBuilder modelBuilder) + { + #region indexes + modelBuilder.Entity() + .HasIndex(x => x.Id) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(x => x.Ptr) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(x => x.ContractId); + + modelBuilder.Entity() + .HasIndex(x => x.LastLevel); + #endregion + + #region keys + modelBuilder.Entity() + .HasKey(x => x.Id); + + modelBuilder.Entity() + .HasAlternateKey(x => x.Ptr); + #endregion + } + } +} diff --git a/Tzkt.Data/Models/Scripts/BigMapKey.cs b/Tzkt.Data/Models/Scripts/BigMapKey.cs new file mode 100644 index 000000000..ef7428b66 --- /dev/null +++ b/Tzkt.Data/Models/Scripts/BigMapKey.cs @@ -0,0 +1,75 @@ +using Microsoft.EntityFrameworkCore; + +namespace Tzkt.Data.Models +{ + public class BigMapKey + { + public int Id { get; set; } + public int BigMapPtr { get; set; } + public int FirstLevel { get; set; } + public int LastLevel { get; set; } + public int Updates { get; set; } + public bool Active { get; set; } + + public string KeyHash { get; set; } + public byte[] RawKey { get; set; } + public string JsonKey { get; set; } + + public byte[] RawValue { get; set; } + public string JsonValue { get; set; } + } + + public static class BigMapKeyModel + { + public static void BuildBigMapKeyModel(this ModelBuilder modelBuilder) + { + #region indexes + modelBuilder.Entity() + .HasIndex(x => x.Id) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(x => x.BigMapPtr); + + modelBuilder.Entity() + .HasIndex(x => x.LastLevel); + + modelBuilder.Entity() + .HasIndex(x => new { x.BigMapPtr, x.Active }) + .HasFilter($@"""{nameof(BigMapKey.Active)}"" = true"); + + modelBuilder.Entity() + .HasIndex(x => new { x.BigMapPtr, x.KeyHash }); + + //modelBuilder.Entity() + // .HasIndex(x => new { x.BigMapPtr, x.JsonKey }) + // .HasMethod("GIN") + // .HasOperators("jsonb_path_ops"); + + //modelBuilder.Entity() + // .HasIndex(x => new { x.BigMapPtr, x.JsonValue }) + // .HasMethod("GIN") + // .HasOperators("jsonb_path_ops"); + #endregion + + #region keys + modelBuilder.Entity() + .HasKey(x => x.Id); + #endregion + + #region props + modelBuilder.Entity() + .Property(x => x.KeyHash) + .HasMaxLength(54); + + modelBuilder.Entity() + .Property(x => x.JsonKey) + .HasColumnType("jsonb"); + + modelBuilder.Entity() + .Property(x => x.JsonValue) + .HasColumnType("jsonb"); + #endregion + } + } +} diff --git a/Tzkt.Data/Models/Scripts/BigMapUpdate.cs b/Tzkt.Data/Models/Scripts/BigMapUpdate.cs new file mode 100644 index 000000000..f0468785a --- /dev/null +++ b/Tzkt.Data/Models/Scripts/BigMapUpdate.cs @@ -0,0 +1,69 @@ +using Microsoft.EntityFrameworkCore; + +namespace Tzkt.Data.Models +{ + public class BigMapUpdate + { + public int Id { get; set; } + public int BigMapPtr { get; set; } + public BigMapAction Action { get; set; } + + public int Level { get; set; } + public int? OriginationId { get; set; } + public int? TransactionId { get; set; } + + public int? BigMapKeyId { get; set; } + public byte[] RawValue { get; set; } + public string JsonValue { get; set; } + } + + public enum BigMapAction + { + Allocate, + AddKey, + UpdateKey, + RemoveKey, + Remove + } + + public static class BigMapUpdateModel + { + public static void BuildBigMapUpdateModel(this ModelBuilder modelBuilder) + { + #region indexes + modelBuilder.Entity() + .HasIndex(x => x.Id) + .IsUnique(); + + modelBuilder.Entity() + .HasIndex(x => x.BigMapPtr); + + modelBuilder.Entity() + .HasIndex(x => x.BigMapKeyId) + .HasFilter($@"""{nameof(BigMapUpdate.BigMapKeyId)}"" is not null"); + + modelBuilder.Entity() + .HasIndex(x => x.Level); + + modelBuilder.Entity() + .HasIndex(x => x.OriginationId) + .HasFilter($@"""{nameof(BigMapUpdate.OriginationId)}"" is not null"); + + modelBuilder.Entity() + .HasIndex(x => x.TransactionId) + .HasFilter($@"""{nameof(BigMapUpdate.TransactionId)}"" is not null"); + #endregion + + #region keys + modelBuilder.Entity() + .HasKey(x => x.Id); + #endregion + + #region props + modelBuilder.Entity() + .Property(x => x.JsonValue) + .HasColumnType("jsonb"); + #endregion + } + } +} diff --git a/Tzkt.Data/Tzkt.Data.csproj b/Tzkt.Data/Tzkt.Data.csproj index 2f52fdf13..c0ee22487 100644 --- a/Tzkt.Data/Tzkt.Data.csproj +++ b/Tzkt.Data/Tzkt.Data.csproj @@ -6,7 +6,7 @@ - + diff --git a/Tzkt.Data/TzktContext.cs b/Tzkt.Data/TzktContext.cs index bf5990cf3..8c87f0485 100644 --- a/Tzkt.Data/TzktContext.cs +++ b/Tzkt.Data/TzktContext.cs @@ -68,6 +68,12 @@ public class TzktContext : DbContext public DbSet Storages { get; set; } #endregion + #region bigmaps + public DbSet BigMaps { get; set; } + public DbSet BigMapKeys { get; set; } + public DbSet BigMapUpdates { get; set; } + #endregion + public TzktContext(DbContextOptions options) : base(options) { } protected override void OnModelCreating(ModelBuilder modelBuilder) @@ -133,6 +139,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.BuildScriptModel(); modelBuilder.BuildStorageModel(); #endregion + + #region bigmaps + modelBuilder.BuildBigMapModel(); + modelBuilder.BuildBigMapKeyModel(); + modelBuilder.BuildBigMapUpdateModel(); + #endregion } } } diff --git a/Tzkt.Sync/Extensions/NetezosExtension.cs b/Tzkt.Sync/Extensions/NetezosExtension.cs new file mode 100644 index 000000000..71a1c53f5 --- /dev/null +++ b/Tzkt.Sync/Extensions/NetezosExtension.cs @@ -0,0 +1,29 @@ +using Netezos.Encoding; + +namespace Tzkt.Sync +{ + static class NetezosExtension + { + public static IMicheline Replace(this IMicheline micheline, IMicheline oldNode, IMicheline newNode) + { + if (micheline == oldNode) + return newNode; + + if (micheline is MichelineArray arr && arr.Count > 0) + { + for (int i = 0; i < arr.Count; i++) + arr[i] = arr[i].Replace(oldNode, newNode); + return arr; + } + + if (micheline is MichelinePrim prim && prim.Args != null) + { + for (int i = 0; i < prim.Args.Count; i++) + prim.Args[i] = prim.Args[i].Replace(oldNode, newNode); + return prim; + } + + return micheline; + } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs new file mode 100644 index 000000000..db89eda9c --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs @@ -0,0 +1,513 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Netezos.Contracts; +using Netezos.Encoding; + +using Tzkt.Data.Models; +using Tzkt.Data.Models.Base; + +namespace Tzkt.Sync.Protocols.Proto1 +{ + class BigMapCommit : ProtocolCommit + { + readonly List<(BaseOperation op, Contract contract, BigMapDiff diff)> Diffs = new(); + readonly Dictionary TempPtrs = new(7); + int TempPtr = 0; + + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + + public virtual void Append(BaseOperation op, Contract contract, IEnumerable diffs) + { + foreach (var diff in diffs) + { + #region transform temp ptrs + if (diff.Ptr < 0) + { + if (diff.Action <= BigMapDiffAction.Copy) + { + TempPtrs[diff.Ptr] = --TempPtr; + diff.Ptr = TempPtr; + } + else + { + diff.Ptr = TempPtrs[diff.Ptr]; + } + } + else if (diff is CopyDiff copy && copy.SourcePtr < 0) + { + copy.SourcePtr = TempPtrs[copy.SourcePtr]; + } + #endregion + Diffs.Add((op, contract, diff)); + } + } + + public virtual async Task Apply() + { + if (Diffs.Count == 0) return; + Diffs[0].op.Block.Events |= BlockEvents.Bigmaps; + + #region prefetch + var allocated = new HashSet(7); + var copiedFrom = new HashSet(7); + + foreach (var diff in Diffs.Where(x => x.diff.Ptr >= 0)) + { + if (diff.diff.Action == BigMapDiffAction.Alloc) + { + allocated.Add(diff.diff.Ptr); + } + else if (diff.diff is CopyDiff copy) + { + var origin = GetOrigin(copy); + if (origin < 0) + allocated.Add(diff.diff.Ptr); + else + copiedFrom.Add(origin); + } + } + + await Cache.BigMaps.Prefetch(Diffs + .Where(x => x.diff.Ptr >= 0 && !allocated.Contains(x.diff.Ptr)) + .Select(x => x.diff.Ptr)); + + await Cache.BigMapKeys.Prefetch(Diffs + .Where(x => x.diff.Ptr >= 0 && !allocated.Contains(x.diff.Ptr) && x.diff.Action == BigMapDiffAction.Update) + .Select(x => (x.diff.Ptr, (x.diff as UpdateDiff).KeyHash))); + + var copiedKeys = copiedFrom.Count == 0 ? null : + await Db.BigMapKeys.AsNoTracking().Where(x => copiedFrom.Contains(x.BigMapPtr)).ToListAsync(); + #endregion + + var images = new Dictionary>(); + foreach (var diff in Diffs) + { + switch (diff.diff) + { + case AllocDiff alloc: + if (alloc.Ptr >= 0) + { + #region allocate new + var script = await Cache.Schemas.GetAsync(diff.contract); + var storage = await Cache.Storages.GetAsync(diff.contract); + var storageView = script.Storage.Schema.ToTreeView(Micheline.FromBytes(storage.RawValue)); + var bigMapNode = storageView.Nodes() + .FirstOrDefault(x => x.Schema.Prim == PrimType.big_map && x.Value is MichelineInt v && v.Value == alloc.Ptr); + + if (bigMapNode == null) + { + storage = Db.ChangeTracker.Entries() + .FirstOrDefault(x => x.Entity is Storage s && (s.OriginationId == diff.op.Id || s.TransactionId == diff.op.Id)) + .Entity as Storage; + storageView = script.Storage.Schema.ToTreeView(Micheline.FromBytes(storage.RawValue)); + bigMapNode = storageView.Nodes() + .FirstOrDefault(x => x.Schema.Prim == PrimType.big_map && x.Value is MichelineInt v && v.Value == alloc.Ptr) + ?? throw new Exception($"Allocated big_map {alloc.Ptr} missed in the storage"); + } + + var bigMapSchema = bigMapNode.Schema as BigMapSchema; + + Db.BigMapUpdates.Add(new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.Allocate, + BigMapPtr = alloc.Ptr, + Level = diff.op.Level, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + }); + var allocatedBigMap = new BigMap + { + Id = Cache.AppState.NextBigMapId(), + Ptr = alloc.Ptr, + ContractId = diff.contract.Id, + StoragePath = bigMapNode.Path, + KeyType = bigMapSchema.Key.ToMicheline().ToBytes(), + ValueType = bigMapSchema.Value.ToMicheline().ToBytes(), + Active = true, + FirstLevel = diff.op.Level, + LastLevel = diff.op.Level, + ActiveKeys = 0, + TotalKeys = 0, + Updates = 1 + }; + Db.BigMaps.Add(allocatedBigMap); + Cache.BigMaps.Cache(allocatedBigMap); + + images.Add(alloc.Ptr, new()); + #endregion + } + else + { + #region alloc temp + images.Add(alloc.Ptr, new()); + #endregion + } + break; + case CopyDiff copy: + if (copy.SourcePtr >= 0 && !copiedFrom.Contains(copy.SourcePtr)) + break; + if (!images.TryGetValue(copy.SourcePtr, out var src)) + { + src = copiedKeys + .Where(x => x.BigMapPtr == copy.SourcePtr) + .ToDictionary(x => x.KeyHash); + } + if (copy.Ptr >= 0) + { + #region copy to new + var script = await Cache.Schemas.GetAsync(diff.contract); + var storage = await Cache.Storages.GetAsync(diff.contract); + var storageView = script.Storage.Schema.ToTreeView(Micheline.FromBytes(storage.RawValue)); + var bigMapNode = storageView.Nodes() + .FirstOrDefault(x => x.Schema.Prim == PrimType.big_map && x.Value is MichelineInt v && v.Value == copy.Ptr); + + if (bigMapNode == null) + { + storage = Db.ChangeTracker.Entries() + .FirstOrDefault(x => x.Entity is Storage s && (s.OriginationId == diff.op.Id || s.TransactionId == diff.op.Id)) + .Entity as Storage; + storageView = script.Storage.Schema.ToTreeView(Micheline.FromBytes(storage.RawValue)); + bigMapNode = storageView.Nodes() + .FirstOrDefault(x => x.Schema.Prim == PrimType.big_map && x.Value is MichelineInt v && v.Value == copy.Ptr) + ?? throw new Exception($"Copied big_map {copy.Ptr} missed in the storage"); + } + + var bigMapSchema = bigMapNode.Schema as BigMapSchema; + + var keys = src.Values.Select(x => + { + var rawKey = Micheline.FromBytes(x.RawKey); + var rawValue = Micheline.FromBytes(x.RawValue); + return new BigMapKey + { + Id = Cache.AppState.NextBigMapKeyId(), + BigMapPtr = copy.Ptr, + Active = true, + KeyHash = x.KeyHash, + JsonKey = bigMapSchema.Key.Humanize(rawKey), + JsonValue = bigMapSchema.Value.Humanize(rawValue), + RawKey = bigMapSchema.Key.Optimize(rawKey).ToBytes(), + RawValue = bigMapSchema.Value.Optimize(rawValue).ToBytes(), + FirstLevel = diff.op.Level, + LastLevel = diff.op.Level, + Updates = 1 + }; + }).ToList(); + + Db.BigMapKeys.AddRange(keys); + Cache.BigMapKeys.Cache(keys); + + Db.BigMapUpdates.Add(new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.Allocate, + BigMapPtr = copy.Ptr, + Level = diff.op.Level, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + }); + Db.BigMapUpdates.AddRange(keys.Select(x => new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.AddKey, + BigMapKeyId = x.Id, + BigMapPtr = x.BigMapPtr, + JsonValue = x.JsonValue, + RawValue = x.RawValue, + Level = x.FirstLevel, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + })); + + var copiedBigMap = new BigMap + { + Id = Cache.AppState.NextBigMapId(), + Ptr = copy.Ptr, + ContractId = diff.contract.Id, + StoragePath = bigMapNode.Path, + KeyType = bigMapSchema.Key.ToMicheline().ToBytes(), + ValueType = bigMapSchema.Value.ToMicheline().ToBytes(), + Active = true, + FirstLevel = diff.op.Level, + LastLevel = diff.op.Level, + ActiveKeys = keys.Count(), + TotalKeys = keys.Count(), + Updates = keys.Count() + 1 + }; + + Db.BigMaps.Add(copiedBigMap); + Cache.BigMaps.Cache(copiedBigMap); + + images.Add(copy.Ptr, keys.ToDictionary(x => x.KeyHash)); + #endregion + } + else + { + #region copy to temp + images.Add(copy.Ptr, src.Values + .Select(x => new BigMapKey + { + KeyHash = x.KeyHash, + RawKey = x.RawKey, + RawValue = x.RawValue + }) + .ToDictionary(x => x.KeyHash)); + #endregion + } + break; + case UpdateDiff update: + if (update.Ptr >= 0) + { + var bigMap = Cache.BigMaps.Get(update.Ptr); + + if (Cache.BigMapKeys.TryGet(update.Ptr, update.KeyHash, out var key)) + { + if (update.Value != null) + { + #region update key + Db.TryAttach(bigMap); + bigMap.LastLevel = diff.op.Level; + if (!key.Active) bigMap.ActiveKeys++; + bigMap.Updates++; + + Db.TryAttach(key); + key.Active = true; + key.JsonValue = bigMap.Schema.Value.Humanize(update.Value); + key.RawValue = bigMap.Schema.Value.Optimize(update.Value).ToBytes(); + key.LastLevel = diff.op.Level; + key.Updates++; + + Db.BigMapUpdates.Add(new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.UpdateKey, + BigMapKeyId = key.Id, + BigMapPtr = key.BigMapPtr, + JsonValue = key.JsonValue, + RawValue = key.RawValue, + Level = key.LastLevel, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + }); + #endregion + } + else if (key.Active) // WTF: edo2net:76611 - key was removed twice + { + #region remove key + Db.TryAttach(bigMap); + bigMap.LastLevel = diff.op.Level; + bigMap.ActiveKeys--; + bigMap.Updates++; + + Db.TryAttach(key); + key.Active = false; + key.LastLevel = diff.op.Level; + key.Updates++; + + Db.BigMapUpdates.Add(new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.RemoveKey, + BigMapKeyId = key.Id, + BigMapPtr = key.BigMapPtr, + JsonValue = key.JsonValue, + RawValue = key.RawValue, + Level = key.LastLevel, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + }); + #endregion + } + } + else if (update.Value != null) // WTF: edo2net:34839 - non-existent key was removed + { + #region add key + Db.TryAttach(bigMap); + bigMap.LastLevel = diff.op.Level; + bigMap.ActiveKeys++; + bigMap.TotalKeys++; + bigMap.Updates++; + + key = new BigMapKey + { + Id = Cache.AppState.NextBigMapKeyId(), + Active = true, + BigMapPtr = update.Ptr, + FirstLevel = diff.op.Level, + LastLevel = diff.op.Level, + JsonKey = bigMap.Schema.Key.Humanize(update.Key), + JsonValue = bigMap.Schema.Value.Humanize(update.Value), + RawKey = bigMap.Schema.Key.Optimize(update.Key).ToBytes(), + RawValue = bigMap.Schema.Value.Optimize(update.Value).ToBytes(), + KeyHash = update.KeyHash, + Updates = 1 + }; + + Db.BigMapKeys.Add(key); + Cache.BigMapKeys.Cache(key); + + Db.BigMapUpdates.Add(new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.AddKey, + BigMapKeyId = key.Id, + BigMapPtr = key.BigMapPtr, + JsonValue = key.JsonValue, + RawValue = key.RawValue, + Level = key.LastLevel, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + }); + #endregion + } + } + else + { + #region update temp + if (!images.TryGetValue(update.Ptr, out var image)) + throw new Exception("Can't update non-existent temporary big_map"); + + if (image.TryGetValue(update.KeyHash, out var key)) + { + if (update.Value != null) + { + key.RawValue = update.Value.ToBytes(); + } + else + { + image.Remove(update.KeyHash); + } + } + else if (update.Value != null) // WTF: edo2net:34839 - non-existent key was removed + { + image.Add(update.KeyHash, new BigMapKey + { + KeyHash = update.KeyHash, + RawKey = update.Key.ToBytes(), + RawValue = update.Value.ToBytes() + }); + } + #endregion + } + break; + case RemoveDiff remove: + if (remove.Ptr >= 0) + { + Db.BigMapUpdates.Add(new BigMapUpdate + { + Id = Cache.AppState.NextBigMapUpdateId(), + Action = BigMapAction.Remove, + BigMapPtr = remove.Ptr, + Level = diff.op.Level, + TransactionId = (diff.op as TransactionOperation)?.Id, + OriginationId = (diff.op as OriginationOperation)?.Id + }); + + var removed = Cache.BigMaps.Get(remove.Ptr); + Db.TryAttach(removed); + removed.Active = false; + removed.LastLevel = diff.op.Level; + removed.Updates++; + } + else + { + // is it possible? + } + break; + default: + break; + } + } + } + + int GetOrigin(CopyDiff copy) + { + return Diffs + .FirstOrDefault(x => x.diff.Action == BigMapDiffAction.Copy && x.diff.Ptr == copy.SourcePtr).diff is CopyDiff prevCopy + ? GetOrigin(prevCopy) + : copy.SourcePtr; + } + + public virtual async Task Revert(Block block) + { + if (block.Events.HasFlag(BlockEvents.Bigmaps)) + { + var bigmaps = await Db.BigMaps.Where(x => x.LastLevel == block.Level).ToListAsync(); + var keys = await Db.BigMapKeys.Where(x => x.LastLevel == block.Level).ToListAsync(); + var updates = await Db.BigMapUpdates + .AsNoTracking() + .Where(x => x.Level == block.Level) + .Select(x => new + { + Ptr = x.BigMapPtr, + KeyId = x.BigMapKeyId + }) + .ToListAsync(); + + await Db.Database.ExecuteSqlRawAsync(@$" + DELETE FROM ""BigMapUpdates"" WHERE ""Level"" = {block.Level}; + "); + + foreach (var key in keys) + { + var bigmap = bigmaps.First(x => x.Ptr == key.BigMapPtr); + Cache.BigMaps.Cache(bigmap); + Cache.BigMapKeys.Cache(key); + + if (key.FirstLevel == block.Level) + { + if (key.Active) bigmap.ActiveKeys--; + bigmap.TotalKeys--; + Db.BigMapKeys.Remove(key); + Cache.BigMapKeys.Remove(key); + } + else + { + var prevUpdate = await Db.BigMapUpdates + .Where(x => x.BigMapKeyId == key.Id) + .OrderByDescending(x => x.Id) + .FirstAsync(); + + var prevActive = prevUpdate.Action != BigMapAction.RemoveKey; + if (key.Active && !prevActive) + bigmap.ActiveKeys--; + else if (!key.Active && prevActive) + bigmap.ActiveKeys++; + + key.Active = prevActive; + key.JsonValue = prevUpdate.JsonValue; + key.RawValue = prevUpdate.RawValue; + key.LastLevel = prevUpdate.Level; + key.Updates -= updates.Count(x => x.KeyId == key.Id); + } + } + + foreach (var bigmap in bigmaps) + { + Cache.BigMaps.Cache(bigmap); + if (bigmap.FirstLevel == block.Level) + { + Db.BigMaps.Remove(bigmap); + Cache.BigMaps.Remove(bigmap); + } + else + { + bigmap.Active = true; + bigmap.Updates -= updates.Count(x => x.Ptr == bigmap.Ptr); + bigmap.LastLevel = bigmap.Updates > 1 + ? (await Db.BigMapUpdates + .Where(x => x.BigMapPtr == bigmap.Ptr) + .OrderByDescending(x => x.Id) + .FirstAsync()) + .Level + : bigmap.FirstLevel; + } + } + } + } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs index 42477ee30..c4733eb33 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Text.Json; using System.Threading.Tasks; @@ -12,6 +13,9 @@ namespace Tzkt.Sync.Protocols.Proto1 { class OriginationsCommit : ProtocolCommit { + public OriginationOperation Origination { get; private set; } + public IEnumerable BigMapDiffs { get; private set; } + public OriginationsCommit(ProtocolHandler protocol) : base(protocol) { } public virtual async Task Apply(Block block, JsonElement op, JsonElement content) @@ -77,7 +81,7 @@ public virtual async Task Apply(Block block, JsonElement op, JsonElement content StorageUsed = result.OptionalInt32("paid_storage_size_diff") ?? 0, StorageFee = (result.OptionalInt32("paid_storage_size_diff") ?? 0) * block.Protocol.ByteCost, AllocationFee = block.Protocol.OriginationSize * block.Protocol.ByteCost - }; + }; #endregion #region entities @@ -145,11 +149,18 @@ await Spend(sender, Db.Contracts.Add(contract); if (contract.Kind > ContractKind.DelegatorContract) - await ProcessScript(origination, content); + { + var code = Micheline.FromJson(content.Required("script").Required("code")) as MichelineArray; + var storage = Micheline.FromJson(content.Required("script").Required("storage")); + + BigMapDiffs = ParseBigMapDiffs(origination, result, code, storage); + ProcessScript(origination, content, code, storage); + } } #endregion Db.OriginationOps.Add(origination); + Origination = origination; } public virtual async Task ApplyInternal(Block block, TransactionOperation parent, JsonElement content) @@ -288,11 +299,18 @@ await Spend(parentSender, Db.Contracts.Add(contract); if (contract.Kind > ContractKind.DelegatorContract) - await ProcessScript(origination, content); + { + var code = Micheline.FromJson(content.Required("script").Required("code")) as MichelineArray; + var storage = Micheline.FromJson(content.Required("script").Required("storage")); + + BigMapDiffs = ParseBigMapDiffs(origination, result, code, storage); + ProcessScript(origination, content, code, storage); + } } #endregion Db.OriginationOps.Add(origination); + Origination = origination; } public virtual async Task Revert(Block block, OriginationOperation origination) @@ -495,11 +513,9 @@ protected virtual BlockEvents GetBlockEvents(Contract contract) : BlockEvents.None; } - protected async Task ProcessScript(OriginationOperation origination, JsonElement content) + protected void ProcessScript(OriginationOperation origination, JsonElement content, MichelineArray code, IMicheline storageValue) { var contract = origination.Contract; - - var code = Micheline.FromJson(content.Required("script").Required("code")) as MichelineArray; var micheParameter = code.First(x => x is MichelinePrim p && p.Prim == PrimType.parameter); var micheStorage = code.First(x => x is MichelinePrim p && p.Prim == PrimType.storage); var micheCode = code.First(x => x is MichelinePrim p && p.Prim == PrimType.code); @@ -528,17 +544,16 @@ protected async Task ProcessScript(OriginationOperation origination, JsonElement contract.Kind = ContractKind.Asset; } - IMicheline storageValue; - if (HasLazyStorages(micheStorage)) - { - // get from RPC because we don't know the IDs of allocated big_maps and sapling_states - var rawContract = await Proto.Rpc.GetContractAsync(origination.Level, contract.Address); - storageValue = Micheline.FromJson(rawContract.Required("script").Required("storage")); - } - else + if (BigMapDiffs != null) { - storageValue = Micheline.FromJson(content.Required("script").Required("storage")); + var ind = 0; + var ptrs = BigMapDiffs.Where(x => x.Action <= BigMapDiffAction.Copy && x.Ptr >= 0).Select(x => x.Ptr).ToList(); + var view = script.Schema.Storage.Schema.ToTreeView(storageValue); + + foreach (var bigmap in view.Nodes().Where(x => x.Schema.Prim == PrimType.big_map)) + storageValue = storageValue.Replace(bigmap.Value, new MichelineInt(ptrs[^++ind])); } + var storage = new Storage { Level = origination.Level, @@ -571,19 +586,39 @@ protected async Task RevertScript(OriginationOperation origination) Cache.Storages.Remove(contract); } - protected bool HasLazyStorages(IMicheline micheline) + protected virtual IEnumerable ParseBigMapDiffs(OriginationOperation origination, JsonElement result, MichelineArray code, IMicheline storage) { - if (micheline is MichelinePrim prim) - { - if (prim.Prim == PrimType.big_map || prim.Prim == PrimType.sapling_state) - return true; + List res = null; + + var micheStorage = code.First(x => x is MichelinePrim p && p.Prim == PrimType.storage) as MichelinePrim; + var schema = new StorageSchema(micheStorage); + var tree = schema.Schema.ToTreeView(storage); + var bigmap = tree.Nodes().FirstOrDefault(x => x.Schema.Prim == PrimType.big_map); - if (prim.Args != null) - foreach (var arg in prim.Args) - if (HasLazyStorages(arg)) - return true; + if (bigmap != null) + { + res = new List + { + new AllocDiff { Ptr = origination.Contract.Id } + }; + if (bigmap.Value is MichelineArray items && items.Count > 0) + { + foreach (var item in items) + { + var key = (item as MichelinePrim).Args[0]; + var value = (item as MichelinePrim).Args[1]; + res.Add(new UpdateDiff + { + Ptr = res[0].Ptr, + Key = key, + Value = value, + KeyHash = (bigmap.Schema as BigMapSchema).GetKeyHash(key) + }); + } + } } - return false; + + return res; } } } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs index 3cabb09a7..c9a9a2251 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs @@ -1,9 +1,11 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using Netezos.Contracts; using Netezos.Encoding; using Tzkt.Data.Models; @@ -14,6 +16,7 @@ namespace Tzkt.Sync.Protocols.Proto1 class TransactionsCommit : ProtocolCommit { public TransactionOperation Transaction { get; private set; } + public IEnumerable BigMapDiffs { get; private set; } public TransactionsCommit(ProtocolHandler protocol) : base(protocol) { } @@ -131,7 +134,10 @@ await Spend(sender, await ResetGracePeriod(transaction); if (result.TryGetProperty("storage", out var storage)) + { + BigMapDiffs = ParseBigMapDiffs(transaction, result); await ProcessStorage(transaction, storage); + } } #endregion @@ -257,7 +263,10 @@ await Spend(parentSender, await ResetGracePeriod(transaction); if (result.TryGetProperty("storage", out var storage)) + { + BigMapDiffs = ParseBigMapDiffs(transaction, result); await ProcessStorage(transaction, storage); + } } #endregion @@ -521,6 +530,7 @@ protected virtual async Task ProcessStorage(TransactionOperation transaction, Js var currentStorage = await Cache.Storages.GetAsync(contract); var newStorageMicheline = schema.OptimizeStorage(Micheline.FromJson(storage), false); + newStorageMicheline = NormalizeStorage(transaction, newStorageMicheline, schema); var newStorageBytes = newStorageMicheline.ToBytes(); if (newStorageBytes.IsEqual(currentStorage.RawValue)) @@ -569,5 +579,72 @@ public async Task RevertStorage(TransactionOperation transaction) Db.Storages.Remove(storage); } } + + protected virtual IMicheline NormalizeStorage(TransactionOperation transaction, IMicheline storage, ContractScript schema) + { + var view = schema.Storage.Schema.ToTreeView(storage); + var bigmap = view.Nodes().FirstOrDefault(x => x.Schema.Prim == PrimType.big_map); + if (bigmap != null) + storage = storage.Replace(bigmap.Value, new MichelineInt(transaction.Target.Id)); + return storage; + } + + protected virtual IEnumerable ParseBigMapDiffs(TransactionOperation transaction, JsonElement result) + { + if (transaction.Level != 5993) + return null; + // It seems there were no big_map diffs at all in proto 1 + // thus there was no an adequate way to track big_map updates, + // so the only way to handle this single big_map update is hardcoding + return new List + { + new UpdateDiff + { + Ptr = transaction.Target.Id, + KeyHash = "exprteAx9hWkXvYSQ4nN9SqjJGVR1sTneHQS1QEcSdzckYdXZVvsqY", + Key = new MichelineString("KT1R3uoZ6W1ZxEwzqtv75Ro7DhVY6UAcxuK2"), + Value = new MichelinePrim + { + Prim = PrimType.Pair, + Args = new List + { + new MichelineString("Aliases Contract"), + new MichelinePrim + { + Prim = PrimType.Pair, + Args = new List + { + new MichelinePrim { Prim = PrimType.None }, + new MichelinePrim + { + Prim = PrimType.Pair, + Args = new List + { + new MichelineInt(0), + new MichelinePrim + { + Prim = PrimType.Pair, + Args = new List + { + new MichelinePrim + { + Prim = PrimType.Left, + Args = new List + { + new MichelinePrim { Prim = PrimType.Unit } + } + }, + new MichelineInt(1530741267) + } + } + } + } + } + } + } + }, + } + }; + } } } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Proto1Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Proto1Handler.cs index ec8d09e8e..5cc651954 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Proto1Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Proto1Handler.cs @@ -83,6 +83,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -99,11 +101,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -112,7 +119,10 @@ public override async Task Commit(JsonElement block) switch (internalContent.RequiredString("kind")) { case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -127,6 +137,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -229,6 +241,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/BigMapCommit.cs new file mode 100644 index 000000000..96344d04c --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto2 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/Operations/TransactionsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/Operations/TransactionsCommit.cs index f09bd8224..a6d086981 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/Operations/TransactionsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto2/Commits/Operations/TransactionsCommit.cs @@ -1,7 +1,29 @@ -namespace Tzkt.Sync.Protocols.Proto2 +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Netezos.Encoding; +using Tzkt.Data.Models; + +namespace Tzkt.Sync.Protocols.Proto2 { class TransactionsCommit : Proto1.TransactionsCommit { public TransactionsCommit(ProtocolHandler protocol) : base(protocol) { } + + protected override IEnumerable ParseBigMapDiffs(TransactionOperation transaction, JsonElement result) + { + if (!result.TryGetProperty("big_map_diff", out var diffs)) + return null; + + return diffs.RequiredArray().EnumerateArray().Select(x => new UpdateDiff + { + Ptr = transaction.Target.Id, + KeyHash = x.RequiredString("key_hash"), + Key = Micheline.FromJson(x.Required("key")), + Value = x.TryGetProperty("value", out var value) + ? Micheline.FromJson(value) + : null, + }); + } } } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto2/Proto2Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto2/Proto2Handler.cs index 9049a4ea8..41275e75f 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto2/Proto2Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto2/Proto2Handler.cs @@ -89,6 +89,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -105,11 +107,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -118,7 +125,10 @@ public override async Task Commit(JsonElement block) switch (internalContent.RequiredString("kind")) { case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -133,6 +143,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -222,6 +234,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/BigMapCommit.cs new file mode 100644 index 000000000..90650d1ff --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto3 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/Operations/TransactionsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/Operations/TransactionsCommit.cs index 3b4f62923..26d419a67 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/Operations/TransactionsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto3/Commits/Operations/TransactionsCommit.cs @@ -2,7 +2,7 @@ namespace Tzkt.Sync.Protocols.Proto3 { - class TransactionsCommit : Proto1.TransactionsCommit + class TransactionsCommit : Proto2.TransactionsCommit { public TransactionsCommit(ProtocolHandler protocol) : base(protocol) { } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto3/Proto3Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto3/Proto3Handler.cs index 47f2a7ee0..6f3eecdfe 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto3/Proto3Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto3/Proto3Handler.cs @@ -105,6 +105,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -121,11 +123,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -134,7 +141,10 @@ public override async Task Commit(JsonElement block) switch (internalContent.RequiredString("kind")) { case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -149,6 +159,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -238,6 +250,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/BigMapCommit.cs new file mode 100644 index 000000000..9d2c67000 --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto4 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto4/Proto4Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto4/Proto4Handler.cs index 09f4cdceb..867461483 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto4/Proto4Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto4/Proto4Handler.cs @@ -108,6 +108,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -124,11 +126,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -137,7 +144,10 @@ public override async Task Commit(JsonElement block) switch (internalContent.RequiredString("kind")) { case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -152,6 +162,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -244,6 +256,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Handlers/Proto5/Activation/ProtoActivator.cs b/Tzkt.Sync/Protocols/Handlers/Proto5/Activation/ProtoActivator.cs index 86197d2d6..7d275b0c4 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto5/Activation/ProtoActivator.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto5/Activation/ProtoActivator.cs @@ -183,6 +183,40 @@ protected override async Task MigrateContext(AppState state) Db.Storages.Add(newStorage); Cache.Storages.Add(contract, newStorage); + + var tree = script.Schema.Storage.Schema.ToTreeView(Micheline.FromBytes(storage.RawValue)); + var bigmap = tree.Nodes().FirstOrDefault(x => x.Schema.Prim == PrimType.big_map); + if (bigmap != null) + { + var newTree = newScript.Schema.Storage.Schema.ToTreeView(Micheline.FromBytes(newStorage.RawValue)); + var newBigmap = newTree.Nodes().FirstOrDefault(x => x.Schema.Prim == PrimType.big_map); + if (newBigmap.Value is not MichelineInt mi) + throw new System.Exception("Expected micheline int"); + var newPtr = (int)mi.Value; + + if (newBigmap.Path != bigmap.Path) + await Db.Database.ExecuteSqlRawAsync($@" + UPDATE ""BigMaps"" SET ""StoragePath"" = '{newBigmap.Path}' WHERE ""Ptr"" = {contract.Id}; + "); + + await Db.Database.ExecuteSqlRawAsync($@" + UPDATE ""BigMaps"" SET ""Ptr"" = {newPtr} WHERE ""Ptr"" = {contract.Id}; + UPDATE ""BigMapKeys"" SET ""BigMapPtr"" = {newPtr} WHERE ""BigMapPtr"" = {contract.Id}; + UPDATE ""BigMapUpdates"" SET ""BigMapPtr"" = {newPtr} WHERE ""BigMapPtr"" = {contract.Id}; + "); + + var storages = await Db.Storages.Where(x => x.ContractId == contract.Id).ToListAsync(); + foreach (var prevStorage in storages) + { + var prevValue = Micheline.FromBytes(prevStorage.RawValue); + var prevTree = script.Schema.Storage.Schema.ToTreeView(prevValue); + var prevBigmap = prevTree.Nodes().First(x => x.Schema.Prim == PrimType.big_map); + (prevBigmap.Value as MichelineInt).Value = newPtr; + + prevStorage.RawValue = prevValue.ToBytes(); + prevStorage.JsonValue = script.Schema.HumanizeStorage(prevValue); + } + } } #endregion } @@ -242,6 +276,41 @@ protected override async Task RevertContext(AppState state) var contract = change.Account as Contract; Cache.Accounts.Add(contract); + var tree = change.NewScript.Schema.Storage.Schema.ToTreeView(Micheline.FromBytes(change.NewStorage.RawValue)); + var bigmap = tree.Nodes().FirstOrDefault(x => x.Schema.Prim == PrimType.big_map); + if (bigmap != null) + { + var oldTree = change.OldScript.Schema.Storage.Schema.ToTreeView(Micheline.FromBytes(change.OldStorage.RawValue)); + var oldBigmap = oldTree.Nodes().FirstOrDefault(x => x.Schema.Prim == PrimType.big_map); + + if (bigmap.Value is not MichelineInt mi) + throw new System.Exception("Expected micheline int"); + var newPtr = (int)mi.Value; + + if (oldBigmap.Path != bigmap.Path) + await Db.Database.ExecuteSqlRawAsync($@" + UPDATE ""BigMaps"" SET ""StoragePath"" = '{oldBigmap.Path}' WHERE ""Ptr"" = {newPtr}; + "); + + await Db.Database.ExecuteSqlRawAsync($@" + UPDATE ""BigMaps"" SET ""Ptr"" = {contract.Id} WHERE ""Ptr"" = {newPtr}; + UPDATE ""BigMapKeys"" SET ""BigMapPtr"" = {contract.Id} WHERE ""BigMapPtr"" = {newPtr}; + UPDATE ""BigMapUpdates"" SET ""BigMapPtr"" = {contract.Id} WHERE ""BigMapPtr"" = {newPtr}; + "); + + var storages = await Db.Storages.Where(x => x.ContractId == contract.Id && x.Level < change.Level).ToListAsync(); + foreach (var prevStorage in storages) + { + var prevValue = Micheline.FromBytes(prevStorage.RawValue); + var prevTree = change.OldScript.Schema.Storage.Schema.ToTreeView(prevValue); + var prevBigmap = prevTree.Nodes().First(x => x.Schema.Prim == PrimType.big_map); + (prevBigmap.Value as MichelineInt).Value = contract.Id; + + prevStorage.RawValue = prevValue.ToBytes(); + prevStorage.JsonValue = change.OldScript.Schema.HumanizeStorage(prevValue); + } + } + change.OldScript.Current = true; Cache.Schemas.Add(contract, change.OldScript.Schema); diff --git a/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/BigMapCommit.cs new file mode 100644 index 000000000..5c259034d --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto5 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/OriginationsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/OriginationsCommit.cs index d2b11a5c6..c1822e7fd 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/OriginationsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/OriginationsCommit.cs @@ -1,5 +1,8 @@ -using System.Text.Json; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; using System.Threading.Tasks; +using Netezos.Encoding; using Tzkt.Data.Models; namespace Tzkt.Sync.Protocols.Proto5 @@ -30,5 +33,12 @@ protected override BlockEvents GetBlockEvents(Contract contract) } protected override bool? GetSpendable(JsonElement content) => null; + + protected override IEnumerable ParseBigMapDiffs(OriginationOperation origination, JsonElement result, MichelineArray code, IMicheline storage) + { + return result.TryGetProperty("big_map_diff", out var diffs) + ? diffs.RequiredArray().EnumerateArray().Select(BigMapDiff.Parse) + : null; + } } } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/TransactionsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/TransactionsCommit.cs index 169a89f9b..0d6eca45b 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/TransactionsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto5/Commits/Operations/TransactionsCommit.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -55,5 +57,17 @@ protected override async Task ProcessParameters(TransactionOperation transaction transaction.RawParameters = rawParam.ToBytes(); } } + + protected override IMicheline NormalizeStorage(TransactionOperation transaction, IMicheline storage, Netezos.Contracts.ContractScript schema) + { + return storage; + } + + protected override IEnumerable ParseBigMapDiffs(TransactionOperation transaction, JsonElement result) + { + return result.TryGetProperty("big_map_diff", out var diffs) + ? diffs.RequiredArray().EnumerateArray().Select(BigMapDiff.Parse) + : null; + } } } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto5/Proto5Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto5/Proto5Handler.cs index f4792d242..5fe413fb4 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto5/Proto5Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto5/Proto5Handler.cs @@ -110,6 +110,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -126,11 +128,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -142,10 +149,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); break; case "origination": - await new OriginationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalOrig = new OriginationsCommit(this); + await internalOrig.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalOrig.BigMapDiffs != null) + bigMapCommit.Append(internalOrig.Origination, internalOrig.Origination.Contract, internalOrig.BigMapDiffs); break; case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -160,6 +173,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -252,7 +267,8 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); - + await new BigMapCommit(this).Revert(currBlock); + foreach (var operation in operations.OrderByDescending(x => x.Id)) { switch (operation) diff --git a/Tzkt.Sync/Protocols/Handlers/Proto6/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto6/Commits/BigMapCommit.cs new file mode 100644 index 000000000..d0fa662d4 --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto6/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto6 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto6/Proto6Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto6/Proto6Handler.cs index 4574d10a8..c6c5a3152 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto6/Proto6Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto6/Proto6Handler.cs @@ -110,6 +110,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -126,11 +128,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -142,10 +149,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); break; case "origination": - await new OriginationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalOrig = new OriginationsCommit(this); + await internalOrig.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalOrig.BigMapDiffs != null) + bigMapCommit.Append(internalOrig.Origination, internalOrig.Origination.Contract, internalOrig.BigMapDiffs); break; case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -160,6 +173,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -252,6 +267,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Handlers/Proto7/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto7/Commits/BigMapCommit.cs new file mode 100644 index 000000000..53349bc35 --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto7/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto7 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto7/Proto7Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto7/Proto7Handler.cs index 4f365f430..7ea5c582e 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto7/Proto7Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto7/Proto7Handler.cs @@ -110,6 +110,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -126,11 +128,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -142,10 +149,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); break; case "origination": - await new OriginationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalOrig = new OriginationsCommit(this); + await internalOrig.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalOrig.BigMapDiffs != null) + bigMapCommit.Append(internalOrig.Origination, internalOrig.Origination.Contract, internalOrig.BigMapDiffs); break; case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -160,6 +173,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -252,6 +267,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Handlers/Proto8/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto8/Commits/BigMapCommit.cs new file mode 100644 index 000000000..9bb575158 --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto8/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto8 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto8/Proto8Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto8/Proto8Handler.cs index b77c282c2..bc6a9aff3 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto8/Proto8Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto8/Proto8Handler.cs @@ -110,6 +110,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -126,11 +128,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -142,10 +149,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); break; case "origination": - await new OriginationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalOrig = new OriginationsCommit(this); + await internalOrig.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalOrig.BigMapDiffs != null) + bigMapCommit.Append(internalOrig.Origination, internalOrig.Origination.Contract, internalOrig.BigMapDiffs); break; case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -160,6 +173,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -252,6 +267,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { diff --git a/Tzkt.Sync/Protocols/Helpers/BigMapDiff.cs b/Tzkt.Sync/Protocols/Helpers/BigMapDiff.cs new file mode 100644 index 000000000..9b602e8fa --- /dev/null +++ b/Tzkt.Sync/Protocols/Helpers/BigMapDiff.cs @@ -0,0 +1,76 @@ +using System.Text.Json; +using Netezos.Encoding; + +namespace Tzkt.Sync.Protocols +{ + public class AllocDiff : BigMapDiff + { + public override BigMapDiffAction Action => BigMapDiffAction.Alloc; + } + + public class CopyDiff : BigMapDiff + { + public override BigMapDiffAction Action => BigMapDiffAction.Copy; + public int SourcePtr { get; set; } + } + + public class UpdateDiff : BigMapDiff + { + public override BigMapDiffAction Action => BigMapDiffAction.Update; + + public string KeyHash { get; set; } + public IMicheline Key { get; set; } + public IMicheline Value { get; set; } + } + + public class RemoveDiff : BigMapDiff + { + public override BigMapDiffAction Action => BigMapDiffAction.Remove; + } + + public abstract class BigMapDiff + { + #region static + public static BigMapDiff Parse(JsonElement diff) + { + return diff.RequiredString("action") switch + { + "alloc" => new AllocDiff + { + Ptr = diff.RequiredInt32("big_map") + }, + "copy" => new CopyDiff + { + Ptr = diff.RequiredInt32("destination_big_map"), + SourcePtr = diff.RequiredInt32("source_big_map") + }, + "update" => new UpdateDiff + { + Ptr = diff.RequiredInt32("big_map"), + KeyHash = diff.RequiredString("key_hash"), + Key = Micheline.FromJson(diff.Required("key")), + Value = diff.TryGetProperty("value", out var v) + ? Micheline.FromJson(v) + : null + }, + "remove" => new RemoveDiff + { + Ptr = diff.RequiredInt32("big_map") + }, + _ => throw new ValidationException($"Unknown big_map_diff action") + }; + } + #endregion + + public abstract BigMapDiffAction Action { get; } + public int Ptr { get; set; } + } + + public enum BigMapDiffAction + { + Alloc, + Copy, + Update, + Remove + } +} diff --git a/Tzkt.Sync/Services/Cache/AppStateCache.cs b/Tzkt.Sync/Services/Cache/AppStateCache.cs index 847b8b082..604bcfbcb 100644 --- a/Tzkt.Sync/Services/Cache/AppStateCache.cs +++ b/Tzkt.Sync/Services/Cache/AppStateCache.cs @@ -59,6 +59,24 @@ public int NextOperationId() return ++AppState.OperationCounter; } + public int NextBigMapId() + { + Db.TryAttach(AppState); + return ++AppState.BigMapCounter; + } + + public int NextBigMapKeyId() + { + Db.TryAttach(AppState); + return ++AppState.BigMapKeyCounter; + } + + public int NextBigMapUpdateId() + { + Db.TryAttach(AppState); + return ++AppState.BigMapUpdateCounter; + } + public int GetManagerCounter() { return AppState.ManagerCounter; diff --git a/Tzkt.Sync/Services/Cache/BigMapKeysCache.cs b/Tzkt.Sync/Services/Cache/BigMapKeysCache.cs new file mode 100644 index 000000000..e65d1ab60 --- /dev/null +++ b/Tzkt.Sync/Services/Cache/BigMapKeysCache.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +using Tzkt.Data; +using Tzkt.Data.Models; + +namespace Tzkt.Sync.Services.Cache +{ + public class BigMapKeysCache + { + public const int MaxItems = 4096; //TODO: set limits in app settings + + static readonly Dictionary Cached = new Dictionary(4097); + + readonly TzktContext Db; + + public BigMapKeysCache(TzktContext db) + { + Db = db; + } + + public void Reset() + { + Cached.Clear(); + } + + public async Task Prefetch(IEnumerable<(int ptr, string hash)> keys) + { + var missed = keys.Where(x => !Cached.ContainsKey(x.ptr + x.hash)).ToList(); + if (missed.Count > 0) + { + #region check space + if (Cached.Count + missed.Count > MaxItems) + { + var pinned = keys.Select(x => x.ptr + x.hash).ToHashSet(); + var toRemove = Cached + .Where(kv => !pinned.Contains(kv.Key)) + .OrderBy(x => x.Value.LastLevel) + .Select(x => x.Key) + .Take(Math.Max(MaxItems / 4, Cached.Count - MaxItems * 3 / 4)) + .ToList(); + + foreach (var key in toRemove) + Cached.Remove(key); + } + #endregion + + var ptrHashes = string.Join(',', missed.Select(x => $"({x.ptr}, '{x.hash}')")); // TODO: use parameters + var loaded = await Db.BigMapKeys + .FromSqlRaw($@" + SELECT * FROM ""{nameof(TzktContext.BigMapKeys)}"" + WHERE (""{nameof(BigMapKey.BigMapPtr)}"", ""{nameof(BigMapKey.KeyHash)}"") IN ({ptrHashes})") + .ToListAsync(); + + foreach (var item in loaded) + Cached.Add(item.BigMapPtr + item.KeyHash, item); + } + } + + public bool TryGet(int ptr, string hash, out BigMapKey key) + { + return Cached.TryGetValue(ptr + hash, out key); + } + + public void Cache(BigMapKey key) + { + Cached[key.BigMapPtr + key.KeyHash] = key; + } + + public void Cache(IEnumerable keys) + { + foreach (var key in keys) + Cached[key.BigMapPtr + key.KeyHash] = key; + } + + public void Remove(BigMapKey key) + { + Cached.Remove(key.BigMapPtr + key.KeyHash); + } + } +} diff --git a/Tzkt.Sync/Services/Cache/BigMapsCache.cs b/Tzkt.Sync/Services/Cache/BigMapsCache.cs new file mode 100644 index 000000000..72f91a075 --- /dev/null +++ b/Tzkt.Sync/Services/Cache/BigMapsCache.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; + +using Tzkt.Data; +using Tzkt.Data.Models; + +namespace Tzkt.Sync.Services.Cache +{ + public class BigMapsCache + { + public const int MaxItems = 1024; //TODO: set limits in app settings + + static readonly Dictionary Cached = new Dictionary(1027); + + readonly TzktContext Db; + + public BigMapsCache(TzktContext db) + { + Db = db; + } + + public void Reset() + { + Cached.Clear(); + } + + public async Task Prefetch(IEnumerable ptrs) + { + var missed = ptrs.Where(x => !Cached.ContainsKey(x)).ToHashSet(); + if (missed.Count > 0) + { + #region check space + if (Cached.Count + missed.Count > MaxItems) + { + var pinned = ptrs.ToHashSet(); + var toRemove = Cached + .Where(kv => !pinned.Contains(kv.Key)) + .OrderBy(x => x.Value.LastLevel) + .Select(x => x.Key) + .Take(Math.Max(MaxItems / 4, Cached.Count - MaxItems * 3 / 4)) + .ToList(); + + foreach (var key in toRemove) + Cached.Remove(key); + } + #endregion + + var items = await Db.BigMaps + .Where(x => missed.Contains(x.Ptr)) + .ToListAsync(); + + foreach (var item in items) + Cached.Add(item.Ptr, item); + } + } + + public BigMap Get(int ptr) + { + if (!Cached.TryGetValue(ptr, out var bigMap)) + throw new Exception($"BigMap #{ptr} doesn't exist"); + return bigMap; + } + + public void Cache(BigMap bigmap) + { + Cached[bigmap.Ptr] = bigmap; + } + + public void Remove(BigMap bigmap) + { + Cached.Remove(bigmap.Ptr); + } + } +} diff --git a/Tzkt.Sync/Services/Cache/CacheService.cs b/Tzkt.Sync/Services/Cache/CacheService.cs index f5b32c73a..1b4424e8a 100644 --- a/Tzkt.Sync/Services/Cache/CacheService.cs +++ b/Tzkt.Sync/Services/Cache/CacheService.cs @@ -20,6 +20,8 @@ public class CacheService public SoftwareCache Software { get; private set; } public SchemasCache Schemas { get; private set; } public StoragesCache Storages { get; private set; } + public BigMapsCache BigMaps { get; private set; } + public BigMapKeysCache BigMapKeys { get; private set; } public CacheService(TzktContext db) { @@ -35,6 +37,8 @@ public CacheService(TzktContext db) Software = new SoftwareCache(db); Schemas = new SchemasCache(db); Storages = new StoragesCache(db); + BigMaps = new BigMapsCache(db); + BigMapKeys = new BigMapKeysCache(db); } public async Task ResetAsync() @@ -49,6 +53,8 @@ public async Task ResetAsync() Software.Reset(); Schemas.Reset(); Storages.Reset(); + BigMaps.Reset(); + BigMapKeys.Reset(); await AppState.ResetAsync(); await Accounts.ResetAsync(); diff --git a/Tzkt.Sync/Services/Cache/SchemasCache.cs b/Tzkt.Sync/Services/Cache/SchemasCache.cs index 92b958a4c..7a31ece01 100644 --- a/Tzkt.Sync/Services/Cache/SchemasCache.cs +++ b/Tzkt.Sync/Services/Cache/SchemasCache.cs @@ -12,9 +12,9 @@ namespace Tzkt.Sync.Services.Cache { public class SchemasCache { - public const int MaxItems = 256; //TODO: set limits in app settings + public const int MaxItems = 1024; //TODO: set limits in app settings - static readonly Dictionary CachedById = new Dictionary(257); + static readonly Dictionary CachedById = new Dictionary(1027); readonly TzktContext Db; @@ -40,7 +40,7 @@ public async Task GetAsync(Contract contract) if (!CachedById.TryGetValue(contract.Id, out var item)) { - item = (await Db.Scripts.FirstOrDefaultAsync(x => x.ContractId == contract.Id && x.Current)).Schema + item = (await Db.Scripts.FirstOrDefaultAsync(x => x.ContractId == contract.Id && x.Current))?.Schema ?? throw new Exception($"Script for contract #{contract.Id} doesn't exist"); Add(contract, item); @@ -59,6 +59,7 @@ void CheckSpace() if (CachedById.Count >= MaxItems) { var oldest = CachedById.Keys + .OrderBy(x => x) .Take(MaxItems / 4) .ToList(); diff --git a/Tzkt.Sync/Services/Cache/StoragesCache.cs b/Tzkt.Sync/Services/Cache/StoragesCache.cs index 064d2f49e..93e9363ae 100644 --- a/Tzkt.Sync/Services/Cache/StoragesCache.cs +++ b/Tzkt.Sync/Services/Cache/StoragesCache.cs @@ -11,9 +11,9 @@ namespace Tzkt.Sync.Services.Cache { public class StoragesCache { - public const int MaxItems = 256; //TODO: set limits in app settings + public const int MaxItems = 1024; //TODO: set limits in app settings - static readonly Dictionary CachedByContractId = new Dictionary(257); + static readonly Dictionary CachedByContractId = new Dictionary(1027); readonly TzktContext Db; @@ -58,9 +58,12 @@ void CheckSpace() if (CachedByContractId.Count >= MaxItems) { var oldest = CachedByContractId.Values - .Take(MaxItems / 4); + .OrderBy(x => x.Level) + .Take(MaxItems / 4) + .Select(x => x.ContractId) + .ToList(); - foreach (var key in oldest.Select(x => x.ContractId).ToList()) + foreach (var key in oldest) CachedByContractId.Remove(key); } } diff --git a/Tzkt.Sync/Tzkt.Sync.csproj b/Tzkt.Sync/Tzkt.Sync.csproj index a90b71953..d04da81df 100644 --- a/Tzkt.Sync/Tzkt.Sync.csproj +++ b/Tzkt.Sync/Tzkt.Sync.csproj @@ -6,12 +6,12 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + From dde2fef413023868f7338f2838cf25a6fe321ae3 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Wed, 31 Mar 2021 18:58:28 +0300 Subject: [PATCH 02/35] Extend data models with off-chain metadata --- Tzkt.Data/Models/Accounts/Account.cs | 11 +++++++++++ Tzkt.Data/Models/Blocks/Protocol.cs | 6 ++++++ Tzkt.Data/Models/Blocks/Software.cs | 16 ++++------------ Tzkt.Data/Models/Voting/Proposal.cs | 6 ++++++ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/Tzkt.Data/Models/Accounts/Account.cs b/Tzkt.Data/Models/Accounts/Account.cs index 53bab2c5c..397437376 100644 --- a/Tzkt.Data/Models/Accounts/Account.cs +++ b/Tzkt.Data/Models/Accounts/Account.cs @@ -28,6 +28,8 @@ public abstract class Account public int? DelegationLevel { get; set; } public bool Staked { get; set; } + public string Metadata { get; set; } + #region relations [ForeignKey(nameof(DelegateId))] public Delegate Delegate { get; set; } @@ -60,6 +62,11 @@ public static void BuildAccountModel(this ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(x => x.DelegateId); + + modelBuilder.Entity() + .HasIndex(x => x.Metadata) + .HasMethod("GIN") + .HasOperators("jsonb_path_ops"); #endregion #region keys @@ -83,6 +90,10 @@ public static void BuildAccountModel(this ModelBuilder modelBuilder) .IsFixedLength(true) .HasMaxLength(36) .IsRequired(); + + modelBuilder.Entity() + .Property(x => x.Metadata) + .HasColumnType("jsonb"); #endregion #region relations diff --git a/Tzkt.Data/Models/Blocks/Protocol.cs b/Tzkt.Data/Models/Blocks/Protocol.cs index e85ccc38c..c5301d2e1 100644 --- a/Tzkt.Data/Models/Blocks/Protocol.cs +++ b/Tzkt.Data/Models/Blocks/Protocol.cs @@ -44,6 +44,8 @@ public class Protocol public int ProposalQuorum { get; set; } public int BallotQuorumMin { get; set; } public int BallotQuorumMax { get; set; } + + public string Metadata { get; set; } } public static class ProtocolModel @@ -64,6 +66,10 @@ public static void BuildProtocolModel(this ModelBuilder modelBuilder) .IsFixedLength(true) .HasMaxLength(51) .IsRequired(); + + modelBuilder.Entity() + .Property(x => x.Metadata) + .HasColumnType("jsonb"); #endregion } } diff --git a/Tzkt.Data/Models/Blocks/Software.cs b/Tzkt.Data/Models/Blocks/Software.cs index 51ec7f84e..b1b3fabae 100644 --- a/Tzkt.Data/Models/Blocks/Software.cs +++ b/Tzkt.Data/Models/Blocks/Software.cs @@ -1,6 +1,4 @@ -using System; -using System.Collections.Generic; -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; namespace Tzkt.Data.Models { @@ -12,12 +10,7 @@ public class Software public int LastLevel { get; set; } public string ShortHash { get; set; } - #region off-chain - public DateTime? CommitDate { get; set; } - public string CommitHash { get; set; } - public string Version { get; set; } - public List Tags { get; set; } - #endregion + public string Metadata { get; set; } } public static class SoftwareModel @@ -37,9 +30,8 @@ public static void BuildSoftwareModel(this ModelBuilder modelBuilder) .IsRequired(); modelBuilder.Entity() - .Property(x => x.CommitHash) - .IsFixedLength(true) - .HasMaxLength(40); + .Property(x => x.Metadata) + .HasColumnType("jsonb"); #endregion } } diff --git a/Tzkt.Data/Models/Voting/Proposal.cs b/Tzkt.Data/Models/Voting/Proposal.cs index 52733c94d..e072d603f 100644 --- a/Tzkt.Data/Models/Voting/Proposal.cs +++ b/Tzkt.Data/Models/Voting/Proposal.cs @@ -14,6 +14,8 @@ public class Proposal public int Upvotes { get; set; } public int Rolls { get; set; } public ProposalStatus Status { get; set; } + + public string Metadata { get; set; } } public static class ProposalModel @@ -38,6 +40,10 @@ public static void BuildProposalModel(this ModelBuilder modelBuilder) .Property(nameof(Proposal.Hash)) .IsFixedLength(true) .HasMaxLength(51); + + modelBuilder.Entity() + .Property(x => x.Metadata) + .HasColumnType("jsonb"); #endregion } } From 7fe3faf53a8bd2b6a239379cdda48c3e091041e6 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Wed, 31 Mar 2021 18:59:49 +0300 Subject: [PATCH 03/35] Add internals count to transaction model --- Tzkt.Data/Models/Operations/Base/Operations.cs | 10 ---------- Tzkt.Data/Models/Operations/TransactionOperation.cs | 5 ++++- .../Proto1/Commits/Operations/DelegationsCommit.cs | 3 ++- .../Proto1/Commits/Operations/OriginationsCommit.cs | 3 ++- .../Proto1/Commits/Operations/TransactionsCommit.cs | 3 ++- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Tzkt.Data/Models/Operations/Base/Operations.cs b/Tzkt.Data/Models/Operations/Base/Operations.cs index df5faecf4..1161f1afe 100644 --- a/Tzkt.Data/Models/Operations/Base/Operations.cs +++ b/Tzkt.Data/Models/Operations/Base/Operations.cs @@ -26,14 +26,4 @@ public enum Operations RevelationPenalty = 0b_0001_0000_0000_0000, Baking = 0b_0010_0000_0000_0000 } - - [Flags] - public enum InternalOperations : byte - { - None = 0b_0000, - - Delegations = 0b_0001, - Originations = 0b_0010, - Transactions = 0b_0100 - } } diff --git a/Tzkt.Data/Models/Operations/TransactionOperation.cs b/Tzkt.Data/Models/Operations/TransactionOperation.cs index 0bd608482..5ac0bd3b2 100644 --- a/Tzkt.Data/Models/Operations/TransactionOperation.cs +++ b/Tzkt.Data/Models/Operations/TransactionOperation.cs @@ -17,7 +17,10 @@ public class TransactionOperation : InternalOperation public int? StorageId { get; set; } - public InternalOperations? InternalOperations { get; set; } + public short? InternalOperations { get; set; } + public short? InternalDelegations { get; set; } + public short? InternalOriginations { get; set; } + public short? InternalTransactions { get; set; } #region relations [ForeignKey(nameof(TargetId))] diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/DelegationsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/DelegationsCommit.cs index a3576e061..b969417bd 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/DelegationsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/DelegationsCommit.cs @@ -187,7 +187,8 @@ public virtual async Task ApplyInternal(Block block, TransactionOperation parent #endregion #region apply operation - parentTx.InternalOperations = (parentTx.InternalOperations ?? InternalOperations.None) | InternalOperations.Delegations; + parentTx.InternalOperations = (short?)((parentTx.InternalOperations ?? 0) + 1); + parentTx.InternalDelegations = (short?)((parentTx.InternalDelegations ?? 0) + 1); sender.DelegationsCount++; if (prevDelegate != null && prevDelegate != sender) prevDelegate.DelegationsCount++; diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs index c4733eb33..43615b744 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/OriginationsCommit.cs @@ -254,7 +254,8 @@ public virtual async Task ApplyInternal(Block block, TransactionOperation parent #endregion #region apply operation - parentTx.InternalOperations = (parentTx.InternalOperations ?? InternalOperations.None) | InternalOperations.Originations; + parentTx.InternalOperations = (short?)((parentTx.InternalOperations ?? 0) + 1); + parentTx.InternalOriginations = (short?)((parentTx.InternalOriginations ?? 0) + 1); sender.OriginationsCount++; if (contractManager != null && contractManager != sender) contractManager.OriginationsCount++; diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs index c9a9a2251..f8e160de5 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/Operations/TransactionsCommit.cs @@ -223,7 +223,8 @@ public virtual async Task ApplyInternal(Block block, TransactionOperation parent #endregion #region apply operation - parentTx.InternalOperations = (parentTx.InternalOperations ?? InternalOperations.None) | InternalOperations.Transactions; + parentTx.InternalOperations = (short?)((parentTx.InternalOperations ?? 0) + 1); + parentTx.InternalTransactions = (short?)((parentTx.InternalTransactions ?? 0) + 1); sender.TransactionsCount++; if (target != null && target != sender) target.TransactionsCount++; From 91c5fc9445f10672528232798369ef04ffef2fc2 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Wed, 31 Mar 2021 19:00:39 +0300 Subject: [PATCH 04/35] Add cycle to app state model --- Tzkt.Data/Models/AppState.cs | 2 ++ .../Protocols/Handlers/Genesis/GenesisHandler.cs | 2 ++ .../Handlers/Initiator/InitiatorHandler.cs | 2 ++ .../Handlers/Proto1/Commits/StateCommit.cs | 14 ++++++++------ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Tzkt.Data/Models/AppState.cs b/Tzkt.Data/Models/AppState.cs index e338b0a4e..16e8836b1 100644 --- a/Tzkt.Data/Models/AppState.cs +++ b/Tzkt.Data/Models/AppState.cs @@ -10,6 +10,7 @@ public class AppState public int KnownHead { get; set; } public DateTime LastSync { get; set; } + public int Cycle { get; set; } public int Level { get; set; } public DateTime Timestamp { get; set; } public string Protocol { get; set; } @@ -72,6 +73,7 @@ public static void BuildAppStateModel(this ModelBuilder modelBuilder) new AppState { Id = -1, + Cycle = -1, Level = -1, Timestamp = DateTime.MinValue, Protocol = "", diff --git a/Tzkt.Sync/Protocols/Handlers/Genesis/GenesisHandler.cs b/Tzkt.Sync/Protocols/Handlers/Genesis/GenesisHandler.cs index be1b8f5e3..2934309c0 100644 --- a/Tzkt.Sync/Protocols/Handlers/Genesis/GenesisHandler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Genesis/GenesisHandler.cs @@ -64,6 +64,7 @@ public override Task Commit(JsonElement rawBlock) #region update state var state = Cache.AppState.Get(); + state.Cycle = -1; state.Level = block.Level; state.Timestamp = block.Timestamp; state.Protocol = block.Protocol.Hash; @@ -89,6 +90,7 @@ await Db.Database.ExecuteSqlRawAsync(@" #region update state var state = Cache.AppState.Get(); + state.Cycle = -1; state.Level = -1; state.Timestamp = DateTime.MinValue; state.Protocol = ""; diff --git a/Tzkt.Sync/Protocols/Handlers/Initiator/InitiatorHandler.cs b/Tzkt.Sync/Protocols/Handlers/Initiator/InitiatorHandler.cs index 9b2aef725..c3337182a 100644 --- a/Tzkt.Sync/Protocols/Handlers/Initiator/InitiatorHandler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Initiator/InitiatorHandler.cs @@ -70,6 +70,7 @@ public override Task Commit(JsonElement rawBlock) #region update state var state = Cache.AppState.Get(); + state.Cycle = 0; state.Level = block.Level; state.Timestamp = block.Timestamp; state.Protocol = block.Protocol.Hash; @@ -103,6 +104,7 @@ await Db.Database.ExecuteSqlRawAsync($@" #region update state var state = Cache.AppState.Get(); + state.Cycle = -1; state.Level = prev.Level; state.Timestamp = prev.Timestamp; state.Protocol = prev.Protocol.Hash; diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/StateCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/StateCommit.cs index 4baa22349..1067385cc 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/StateCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/StateCommit.cs @@ -20,6 +20,7 @@ public virtual Task Apply(Block block, JsonElement rawBlock) var state = appState; #endregion + state.Cycle = (block.Level - 1) / block.Protocol.BlocksPerCycle; state.Level = block.Level; state.Timestamp = block.Timestamp; state.Protocol = block.Protocol.Hash; @@ -82,14 +83,15 @@ public virtual async Task Revert(Block block) #region entities var state = appState; var prevBlock = await Cache.Blocks.PreviousAsync(); - if (prevBlock != null) prevBlock.Protocol ??= await Cache.Protocols.GetAsync(prevBlock.ProtoCode); + prevBlock.Protocol ??= await Cache.Protocols.GetAsync(prevBlock.ProtoCode); #endregion - state.Level = prevBlock?.Level ?? -1; - state.Timestamp = prevBlock?.Timestamp ?? DateTime.MinValue; - state.Protocol = prevBlock?.Protocol.Hash ?? ""; - state.NextProtocol = prevBlock == null ? "" : nextProtocol; - state.Hash = prevBlock?.Hash ?? ""; + state.Cycle = (prevBlock.Level - 1) / Math.Max(prevBlock.Protocol.BlocksPerCycle, 1); + state.Level = prevBlock.Level; + state.Timestamp = prevBlock.Timestamp; + state.Protocol = prevBlock.Protocol.Hash; + state.NextProtocol = nextProtocol; + state.Hash = prevBlock.Hash; state.BlocksCount--; if (block.Events.HasFlag(BlockEvents.ProtocolBegin)) state.ProtocolsCount--; From a1211580dd8a3f75794954587aa65d7e574fcec3 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Wed, 31 Mar 2021 19:56:21 +0300 Subject: [PATCH 05/35] Add ETH quote --- Tzkt.Api/Controllers/AccountsController.cs | 6 ++- Tzkt.Api/Models/Quote.cs | 5 +++ Tzkt.Api/Models/QuoteShort.cs | 8 +++- Tzkt.Api/Models/State.cs | 20 ++++++++++ Tzkt.Api/Repositories/QuotesRepository.cs | 14 ++++++- Tzkt.Api/Repositories/ReportRepository.cs | 24 +++++++++++- Tzkt.Api/Repositories/StateRepository.cs | 6 ++- Tzkt.Api/Services/Cache/Quotes/QuotesCache.cs | 15 ++++++-- .../Cache/State/RawModels/RawState.cs | 4 ++ Tzkt.Data/Models/AppState.cs | 1 + Tzkt.Data/Models/Quotes/IQuote.cs | 3 +- Tzkt.Data/Models/Quotes/Quote.cs | 5 ++- .../Providers/Coingecko/CoingeckoProvider.cs | 3 ++ .../Quotes/Providers/DefaultQuotesProvider.cs | 38 ++++++++++++++++++- .../TzktQuotes/TzktQuotesProvider.cs | 6 +++ Tzkt.Sync/Services/Quotes/QuotesService.cs | 27 ++++++++----- 16 files changed, 161 insertions(+), 24 deletions(-) diff --git a/Tzkt.Api/Controllers/AccountsController.cs b/Tzkt.Api/Controllers/AccountsController.cs index f0107320b..93ca68b5f 100644 --- a/Tzkt.Api/Controllers/AccountsController.cs +++ b/Tzkt.Api/Controllers/AccountsController.cs @@ -426,7 +426,7 @@ public Task GetMetadata([Address] string address) /// End of the datetime range to filter by (ISO 8601, e.g. 2019-12-31) /// Column delimiter (`comma`, `semicolon`) /// Decimal separator (`comma`, `point`) - /// Currency to convert amounts to (`btc`, `eur`, `usd`) + /// Currency to convert amounts to (`btc`, `eur`, `usd`, `cny`, `jpy`, `krw`, `eth`) /// `true` if you want to use historical prices, `false` to use current price /// [HttpGet("{address}/report")] @@ -475,6 +475,10 @@ public async Task GetBalanceReport( "btc" => 0, "eur" => 1, "usd" => 2, + "cny" => 3, + "jpy" => 4, + "krw" => 5, + "eth" => 6, _ => -1 }; #endregion diff --git a/Tzkt.Api/Models/Quote.cs b/Tzkt.Api/Models/Quote.cs index 05fa9d1a5..2922e7782 100644 --- a/Tzkt.Api/Models/Quote.cs +++ b/Tzkt.Api/Models/Quote.cs @@ -43,5 +43,10 @@ public class Quote /// XTZ/KRW price /// public double Krw { get; set; } + + /// + /// XTZ/ETH price + /// + public double Eth { get; set; } } } diff --git a/Tzkt.Api/Models/QuoteShort.cs b/Tzkt.Api/Models/QuoteShort.cs index abcce71db..017d8474f 100644 --- a/Tzkt.Api/Models/QuoteShort.cs +++ b/Tzkt.Api/Models/QuoteShort.cs @@ -35,6 +35,11 @@ public class QuoteShort /// XTZ/KRW price /// public double? Krw { get; set; } + + /// + /// XTZ/ETH price + /// + public double? Eth { get; set; } } [Flags] @@ -47,6 +52,7 @@ public enum Symbols Usd = 4, Cny = 8, Jpy = 16, - Krw = 32 + Krw = 32, + Eth = 64 } } diff --git a/Tzkt.Api/Models/State.cs b/Tzkt.Api/Models/State.cs index d13499c65..78acd44dc 100644 --- a/Tzkt.Api/Models/State.cs +++ b/Tzkt.Api/Models/State.cs @@ -71,5 +71,25 @@ public class State /// Last known XTZ/USD price /// public double QuoteUsd { get; set; } + + /// + /// Last known XTZ/CNY price + /// + public double QuoteCny { get; set; } + + /// + /// Last known XTZ/JPY price + /// + public double QuoteJpy { get; set; } + + /// + /// Last known XTZ/KRW price + /// + public double QuoteKrw { get; set; } + + /// + /// Last known XTZ/ETH price + /// + public double QuoteEth { get; set; } } } diff --git a/Tzkt.Api/Repositories/QuotesRepository.cs b/Tzkt.Api/Repositories/QuotesRepository.cs index 34bea3acc..ba1254a14 100644 --- a/Tzkt.Api/Repositories/QuotesRepository.cs +++ b/Tzkt.Api/Repositories/QuotesRepository.cs @@ -46,7 +46,8 @@ public Quote GetLast() Usd = Quotes.Get(2), Cny = Quotes.Get(3), Jpy = Quotes.Get(4), - Krw = Quotes.Get(5) + Krw = Quotes.Get(5), + Eth = Quotes.Get(6) }; } @@ -79,6 +80,7 @@ public async Task> Get( Cny = row.Cny, Jpy = row.Jpy, Krw = row.Krw, + Eth = row.Eth }); } @@ -104,6 +106,7 @@ public async Task Get( case "cny": columns.Add(@"""Cny"""); break; case "jpy": columns.Add(@"""Jpy"""); break; case "krw": columns.Add(@"""Krw"""); break; + case "eth": columns.Add(@"""Eth"""); break; } } @@ -162,6 +165,10 @@ public async Task Get( foreach (var row in rows) result[j++][i] = row.Krw; break; + case "eth": + foreach (var row in rows) + result[j++][i] = row.Eth; + break; } } @@ -187,6 +194,7 @@ public async Task Get( case "cny": columns.Add(@"""Cny"""); break; case "jpy": columns.Add(@"""Jpy"""); break; case "krw": columns.Add(@"""Krw"""); break; + case "eth": columns.Add(@"""Eth"""); break; } if (columns.Count == 0) @@ -242,6 +250,10 @@ public async Task Get( foreach (var row in rows) result[j++] = row.Krw; break; + case "eth": + foreach (var row in rows) + result[j++] = row.Eth; + break; } return result; diff --git a/Tzkt.Api/Repositories/ReportRepository.cs b/Tzkt.Api/Repositories/ReportRepository.cs index 39ad12a41..d77e8f9cc 100644 --- a/Tzkt.Api/Repositories/ReportRepository.cs +++ b/Tzkt.Api/Repositories/ReportRepository.cs @@ -170,7 +170,17 @@ public async Task Write(StreamWriter csv, string address, DateTime from, DateTim var rows = await db.QueryAsync(sql.ToString(), new { account = account.Id, from, to, limit }); #region write header - var symbolName = symbol == 2 ? "USD" : symbol == 1 ? "EUR" : "BTC"; + var symbolName = symbol switch + { + 0 => "BTC", + 1 => "EUR", + 2 => "USD", + 3 => "CNY", + 4 => "JPY", + 5 => "KRW", + 6 => "ETH", + _ => "" + }; csv.Write("Block level"); csv.Write(delimiter); @@ -302,7 +312,17 @@ public async Task WriteHistorical(StreamWriter csv, string address, DateTime fro var rows = await db.QueryAsync(sql.ToString(), new { account = account.Id, from, to, limit }); #region write header - var symbolName = symbol == 2 ? "USD" : symbol == 1 ? "EUR" : "BTC"; + var symbolName = symbol switch + { + 0 => "BTC", + 1 => "EUR", + 2 => "USD", + 3 => "CNY", + 4 => "JPY", + 5 => "KRW", + 6 => "ETH", + _ => "" + }; csv.Write("Block level"); csv.Write(delimiter); diff --git a/Tzkt.Api/Repositories/StateRepository.cs b/Tzkt.Api/Repositories/StateRepository.cs index 53af52575..4d1583b55 100644 --- a/Tzkt.Api/Repositories/StateRepository.cs +++ b/Tzkt.Api/Repositories/StateRepository.cs @@ -34,7 +34,11 @@ public State Get() QuoteLevel = appState.QuoteLevel, QuoteBtc = appState.QuoteBtc, QuoteEur = appState.QuoteEur, - QuoteUsd = appState.QuoteUsd + QuoteUsd = appState.QuoteUsd, + QuoteCny = appState.QuoteCny, + QuoteJpy = appState.QuoteJpy, + QuoteKrw = appState.QuoteKrw, + QuoteEth = appState.QuoteEth }; } } diff --git a/Tzkt.Api/Services/Cache/Quotes/QuotesCache.cs b/Tzkt.Api/Services/Cache/Quotes/QuotesCache.cs index 7a7b39bf8..97a18af5b 100644 --- a/Tzkt.Api/Services/Cache/Quotes/QuotesCache.cs +++ b/Tzkt.Api/Services/Cache/Quotes/QuotesCache.cs @@ -20,12 +20,12 @@ public QuotesCache(StateCache state, IConfiguration config, ILogger { logger.LogDebug("Initializing quotes cache..."); - Quotes = new List[6]; + Quotes = new List[7]; State = state; Logger = logger; var sql = @" - SELECT ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"" + SELECT ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"", ""Eth"" FROM ""Quotes"" ORDER BY ""Level"""; @@ -44,6 +44,7 @@ public QuotesCache(StateCache state, IConfiguration config, ILogger Quotes[3].Add(row.Cny); Quotes[4].Add(row.Jpy); Quotes[5].Add(row.Krw); + Quotes[6].Add(row.Eth); } logger.LogInformation("Loaded {1} quotes", Quotes[0].Count); @@ -53,7 +54,7 @@ public async Task UpdateAsync() { Logger.LogDebug("Updating quotes cache"); var sql = $@" - SELECT ""Level"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"" + SELECT ""Level"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"", ""Eth"" FROM ""Quotes"" WHERE ""Level"" > @fromLevel ORDER BY ""Level"""; @@ -71,6 +72,7 @@ public async Task UpdateAsync() Quotes[3][row.Level] = row.Cny; Quotes[4][row.Level] = row.Jpy; Quotes[5][row.Level] = row.Krw; + Quotes[6][row.Level] = row.Eth; } else { @@ -80,6 +82,7 @@ public async Task UpdateAsync() Quotes[3].Add(row.Cny); Quotes[4].Add(row.Jpy); Quotes[5].Add(row.Krw); + Quotes[6].Add(row.Eth); } } Logger.LogDebug("{1} quotes updates", rows.Count()); @@ -121,6 +124,9 @@ public QuoteShort Get(Symbols symbols, int level) if (symbols.HasFlag(Symbols.Krw)) quote.Krw = Quotes[5][^1]; + + if (symbols.HasFlag(Symbols.Eth)) + quote.Eth = Quotes[6][^1]; } else { @@ -141,6 +147,9 @@ public QuoteShort Get(Symbols symbols, int level) if (symbols.HasFlag(Symbols.Krw)) quote.Krw = Quotes[5][level]; + + if (symbols.HasFlag(Symbols.Eth)) + quote.Eth = Quotes[6][level]; } return quote; diff --git a/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs b/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs index 4bf60847b..63a27ab45 100644 --- a/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs +++ b/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs @@ -51,6 +51,10 @@ public class RawState public double QuoteBtc { get; set; } public double QuoteEur { get; set; } public double QuoteUsd { get; set; } + public double QuoteCny { get; set; } + public double QuoteJpy { get; set; } + public double QuoteKrw { get; set; } + public double QuoteEth { get; set; } #endregion } } diff --git a/Tzkt.Data/Models/AppState.cs b/Tzkt.Data/Models/AppState.cs index 16e8836b1..cc8aaa7da 100644 --- a/Tzkt.Data/Models/AppState.cs +++ b/Tzkt.Data/Models/AppState.cs @@ -62,6 +62,7 @@ public class AppState public double QuoteCny { get; set; } public double QuoteJpy { get; set; } public double QuoteKrw { get; set; } + public double QuoteEth { get; set; } #endregion } diff --git a/Tzkt.Data/Models/Quotes/IQuote.cs b/Tzkt.Data/Models/Quotes/IQuote.cs index 405140e0d..56c8fa799 100644 --- a/Tzkt.Data/Models/Quotes/IQuote.cs +++ b/Tzkt.Data/Models/Quotes/IQuote.cs @@ -1,6 +1,4 @@ using System; -using System.Collections.Generic; -using System.Text; namespace Tzkt.Data.Models { @@ -15,5 +13,6 @@ public interface IQuote double Cny { get; set; } double Jpy { get; set; } double Krw { get; set; } + double Eth { get; set; } } } diff --git a/Tzkt.Data/Models/Quotes/Quote.cs b/Tzkt.Data/Models/Quotes/Quote.cs index d334dd917..221a916bc 100644 --- a/Tzkt.Data/Models/Quotes/Quote.cs +++ b/Tzkt.Data/Models/Quotes/Quote.cs @@ -1,5 +1,5 @@ -using Microsoft.EntityFrameworkCore; -using System; +using System; +using Microsoft.EntityFrameworkCore; namespace Tzkt.Data.Models { @@ -15,6 +15,7 @@ public class Quote : IQuote public double Cny { get; set; } public double Jpy { get; set; } public double Krw { get; set; } + public double Eth { get; set; } } public static class QuoteModel diff --git a/Tzkt.Sync/Services/Quotes/Providers/Coingecko/CoingeckoProvider.cs b/Tzkt.Sync/Services/Quotes/Providers/Coingecko/CoingeckoProvider.cs index b57cc01c1..3afe4ab03 100644 --- a/Tzkt.Sync/Services/Quotes/Providers/Coingecko/CoingeckoProvider.cs +++ b/Tzkt.Sync/Services/Quotes/Providers/Coingecko/CoingeckoProvider.cs @@ -41,6 +41,9 @@ public override async Task> GetJpy(DateTime from, Dat public override async Task> GetKrw(DateTime from, DateTime to) => await GetQuotes("krw", from, to); + public override async Task> GetEth(DateTime from, DateTime to) + => await GetQuotes("eth", from, to); + async Task> GetQuotes(string currency, DateTime from, DateTime to) { var _from = (long)(from - DateTime.UnixEpoch).TotalSeconds; diff --git a/Tzkt.Sync/Services/Quotes/Providers/DefaultQuotesProvider.cs b/Tzkt.Sync/Services/Quotes/Providers/DefaultQuotesProvider.cs index d79a36c1e..1248e4ea1 100644 --- a/Tzkt.Sync/Services/Quotes/Providers/DefaultQuotesProvider.cs +++ b/Tzkt.Sync/Services/Quotes/Providers/DefaultQuotesProvider.cs @@ -16,7 +16,8 @@ public async Task FillQuotes(IEnumerable quotes, IQuote last) FillUsdQuotes(quotes, last), FillCnyQuotes(quotes, last), FillJpyQuotes(quotes, last), - FillKrwQuotes(quotes, last)); + FillKrwQuotes(quotes, last), + FillEthQuotes(quotes, last)); return filled.Min(); } @@ -213,6 +214,38 @@ async Task FillKrwQuotes(IEnumerable quotes, IQuote last) return quotes.Count(); } + async Task FillEthQuotes(IEnumerable quotes, IQuote last) + { + var res = (await GetEth( + quotes.First().Timestamp.AddMinutes(-30), + quotes.Last().Timestamp)).ToList(); + + if (res.Count == 0) + { + foreach (var quote in quotes) + quote.Eth = last?.Eth ?? 0; + } + else + { + var i = 0; + foreach (var quote in quotes) + { + if (quote.Timestamp < res[0].Timestamp) + { + quote.Eth = last?.Eth ?? 0; + } + else + { + while (i < res.Count - 1 && quote.Timestamp >= res[i + 1].Timestamp) i++; + + quote.Eth = res[i].Price; + } + } + } + + return quotes.Count(); + } + #region virtual public virtual Task> GetBtc(DateTime from, DateTime to) => Task.FromResult(Enumerable.Empty()); @@ -231,6 +264,9 @@ public virtual Task> GetJpy(DateTime from, DateTime t public virtual Task> GetKrw(DateTime from, DateTime to) => Task.FromResult(Enumerable.Empty()); + + public virtual Task> GetEth(DateTime from, DateTime to) + => Task.FromResult(Enumerable.Empty()); #endregion } diff --git a/Tzkt.Sync/Services/Quotes/Providers/TzktQuotes/TzktQuotesProvider.cs b/Tzkt.Sync/Services/Quotes/Providers/TzktQuotes/TzktQuotesProvider.cs index 53416c1b6..c815c71a2 100644 --- a/Tzkt.Sync/Services/Quotes/Providers/TzktQuotes/TzktQuotesProvider.cs +++ b/Tzkt.Sync/Services/Quotes/Providers/TzktQuotes/TzktQuotesProvider.cs @@ -39,6 +39,7 @@ public async Task FillQuotes(IEnumerable quotes, IQuote last) quote.Cny = last?.Cny ?? 0; quote.Jpy = last?.Jpy ?? 0; quote.Krw = last?.Krw ?? 0; + quote.Eth = last?.Eth ?? 0; } } else @@ -54,6 +55,7 @@ public async Task FillQuotes(IEnumerable quotes, IQuote last) quote.Cny = last?.Cny ?? 0; quote.Jpy = last?.Jpy ?? 0; quote.Krw = last?.Krw ?? 0; + quote.Eth = last?.Eth ?? 0; } else { @@ -65,6 +67,7 @@ public async Task FillQuotes(IEnumerable quotes, IQuote last) quote.Cny = res[i].Cny; quote.Jpy = res[i].Jpy; quote.Krw = res[i].Krw; + quote.Eth = res[i].Eth; } } } @@ -108,6 +111,9 @@ public class TzktQuote : IQuote [JsonPropertyName("krw")] public double Krw { get; set; } + [JsonPropertyName("eth")] + public double Eth { get; set; } + int IQuote.Level => throw new NotImplementedException(); } } diff --git a/Tzkt.Sync/Services/Quotes/QuotesService.cs b/Tzkt.Sync/Services/Quotes/QuotesService.cs index ec3b1b13c..8f961f94b 100644 --- a/Tzkt.Sync/Services/Quotes/QuotesService.cs +++ b/Tzkt.Sync/Services/Quotes/QuotesService.cs @@ -176,7 +176,7 @@ void SaveQuotes(IEnumerable quotes) { var conn = Db.Database.GetDbConnection() as NpgsqlConnection; if (conn.State != System.Data.ConnectionState.Open) conn.Open(); - using var writer = conn.BeginBinaryImport(@"COPY ""Quotes"" (""Level"", ""Timestamp"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"") FROM STDIN (FORMAT BINARY)"); + using var writer = conn.BeginBinaryImport(@"COPY ""Quotes"" (""Level"", ""Timestamp"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"", ""Eth"") FROM STDIN (FORMAT BINARY)"); foreach (var q in quotes) { @@ -189,6 +189,7 @@ void SaveQuotes(IEnumerable quotes) writer.Write(q.Cny); writer.Write(q.Jpy); writer.Write(q.Krw); + writer.Write(q.Eth); } writer.Complete(); @@ -197,8 +198,8 @@ void SaveQuotes(IEnumerable quotes) void UpdateState(AppState state, Quote quote) { Db.Database.ExecuteSqlRaw($@" - UPDATE ""AppState"" SET ""QuoteLevel"" = {{0}}, ""QuoteBtc"" = {{1}}, ""QuoteEur"" = {{2}}, ""QuoteUsd"" = {{3}}, ""QuoteCny"" = {{4}}, ""QuoteJpy"" = {{5}}, ""QuoteKrw"" = {{6}};", - quote.Level, quote.Btc, quote.Eur, quote.Usd, quote.Cny, quote.Jpy, quote.Krw); + UPDATE ""AppState"" SET ""QuoteLevel"" = {{0}}, ""QuoteBtc"" = {{1}}, ""QuoteEur"" = {{2}}, ""QuoteUsd"" = {{3}}, ""QuoteCny"" = {{4}}, ""QuoteJpy"" = {{5}}, ""QuoteKrw"" = {{6}}, ""QuoteEth"" = {{7}};", + quote.Level, quote.Btc, quote.Eur, quote.Usd, quote.Cny, quote.Jpy, quote.Krw, quote.Eth); state.QuoteLevel = quote.Level; state.QuoteBtc = quote.Btc; @@ -207,14 +208,15 @@ void UpdateState(AppState state, Quote quote) state.QuoteCny = quote.Cny; state.QuoteJpy = quote.Jpy; state.QuoteKrw = quote.Krw; + state.QuoteEth = quote.Eth; } void SaveAndUpdate(AppState state, Quote quote) { Db.Database.ExecuteSqlRaw($@" - INSERT INTO ""Quotes"" (""Level"", ""Timestamp"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"") VALUES ({{0}}, {{1}}, {{2}}, {{3}}, {{4}}, {{5}}, {{6}}, {{7}}); - UPDATE ""AppState"" SET ""QuoteLevel"" = {{0}}, ""QuoteBtc"" = {{2}}, ""QuoteEur"" = {{3}}, ""QuoteUsd"" = {{4}}, ""QuoteCny"" = {{5}}, ""QuoteJpy"" = {{6}}, ""QuoteKrw"" = {{7}};", - quote.Level, quote.Timestamp, quote.Btc, quote.Eur, quote.Usd, quote.Cny, quote.Jpy, quote.Krw); + INSERT INTO ""Quotes"" (""Level"", ""Timestamp"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"", ""Eth"") VALUES ({{0}}, {{1}}, {{2}}, {{3}}, {{4}}, {{5}}, {{6}}, {{7}}, {{8}}); + UPDATE ""AppState"" SET ""QuoteLevel"" = {{0}}, ""QuoteBtc"" = {{2}}, ""QuoteEur"" = {{3}}, ""QuoteUsd"" = {{4}}, ""QuoteCny"" = {{5}}, ""QuoteJpy"" = {{6}}, ""QuoteKrw"" = {{7}}, ""QuoteEth"" = {{8}};", + quote.Level, quote.Timestamp, quote.Btc, quote.Eur, quote.Usd, quote.Cny, quote.Jpy, quote.Krw, quote.Eth); state.QuoteLevel = quote.Level; state.QuoteBtc = quote.Btc; @@ -223,6 +225,7 @@ void SaveAndUpdate(AppState state, Quote quote) state.QuoteCny = quote.Cny; state.QuoteJpy = quote.Jpy; state.QuoteKrw = quote.Krw; + state.QuoteEth = quote.Eth; } void SaveAndUpdate(AppState state, IEnumerable quotes) @@ -232,8 +235,8 @@ void SaveAndUpdate(AppState state, IEnumerable quotes) var sql = new StringBuilder(); sql.AppendLine($@" - UPDATE ""AppState"" SET ""QuoteLevel"" = {last.Level}, ""QuoteBtc"" = {{0}}, ""QuoteEur"" = {{1}}, ""QuoteUsd"" = {{2}}, ""QuoteCny"" = {{3}}, ""QuoteJpy"" = {{4}}, ""QuoteKrw"" = {{5}}; - INSERT INTO ""Quotes"" (""Level"", ""Timestamp"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"") VALUES"); + UPDATE ""AppState"" SET ""QuoteLevel"" = {last.Level}, ""QuoteBtc"" = {{0}}, ""QuoteEur"" = {{1}}, ""QuoteUsd"" = {{2}}, ""QuoteCny"" = {{3}}, ""QuoteJpy"" = {{4}}, ""QuoteKrw"" = {{5}}, ""QuoteEth"" = {{6}}; + INSERT INTO ""Quotes"" (""Level"", ""Timestamp"", ""Btc"", ""Eur"", ""Usd"", ""Cny"", ""Jpy"", ""Krw"", ""Eth"") VALUES"); var param = new List(cnt * 7 + 6); param.Add(last.Btc); @@ -242,13 +245,14 @@ void SaveAndUpdate(AppState state, IEnumerable quotes) param.Add(last.Cny); param.Add(last.Jpy); param.Add(last.Krw); + param.Add(last.Eth); var p = 6; var i = 0; foreach (var q in quotes) { - sql.Append($"({q.Level}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}})"); + sql.Append($"({q.Level}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}}, {{{p++}}})"); if (++i < cnt) sql.AppendLine(","); else sql.AppendLine(";"); @@ -259,6 +263,7 @@ void SaveAndUpdate(AppState state, IEnumerable quotes) param.Add(q.Cny); param.Add(q.Jpy); param.Add(q.Krw); + param.Add(q.Eth); } Db.Database.ExecuteSqlRaw(sql.ToString(), param); @@ -270,6 +275,7 @@ void SaveAndUpdate(AppState state, IEnumerable quotes) state.QuoteCny = last.Cny; state.QuoteJpy = last.Jpy; state.QuoteKrw = last.Krw; + state.QuoteEth = last.Eth; } IQuote LastQuote(AppState state) => state.QuoteLevel == -1 ? null : new Quote @@ -279,7 +285,8 @@ void SaveAndUpdate(AppState state, IEnumerable quotes) Usd = state.QuoteUsd, Cny = state.QuoteCny, Jpy = state.QuoteJpy, - Krw = state.QuoteKrw + Krw = state.QuoteKrw, + Eth = state.QuoteEth }; } From fcd193316811aaa88e9cb33a2391b3db45c1cd69 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Wed, 31 Mar 2021 21:25:50 +0300 Subject: [PATCH 06/35] Update software repo and cache to use metadata --- Tzkt.Api/Controllers/SoftwareController.cs | 2 +- Tzkt.Api/Models/Software.cs | 21 ++------- Tzkt.Api/Repositories/SoftwareRepository.cs | 47 +++---------------- .../Software/SoftwareMetadataService.cs | 10 ++-- 4 files changed, 17 insertions(+), 63 deletions(-) diff --git a/Tzkt.Api/Controllers/SoftwareController.cs b/Tzkt.Api/Controllers/SoftwareController.cs index 91aabcaec..cacc95dcd 100644 --- a/Tzkt.Api/Controllers/SoftwareController.cs +++ b/Tzkt.Api/Controllers/SoftwareController.cs @@ -31,7 +31,7 @@ public SoftwareController(SoftwareRepository software) /// Maximum number of items to return /// [HttpGet] - public async Task>> Get( + public async Task>> Get( SelectParameter select, SortParameter sort, OffsetParameter offset, diff --git a/Tzkt.Api/Models/Software.cs b/Tzkt.Api/Models/Software.cs index 9e6815b19..81eb855db 100644 --- a/Tzkt.Api/Models/Software.cs +++ b/Tzkt.Api/Models/Software.cs @@ -1,5 +1,5 @@ using System; -using System.Collections.Generic; +using System.Text.Json; namespace Tzkt.Api.Models { @@ -36,23 +36,8 @@ public class Software public int BlocksCount { get; set; } /// - /// Offchain data: commit date + /// Offchain metadata /// - public DateTime? CommitDate { get; set; } - - /// - /// Offchain data: commit hash - /// - public string CommitHash { get; set; } - - /// - /// Offchain data: software version (commit tag) - /// - public string Version { get; set; } - - /// - /// Offchain data: software tags, e.g. `docker`, `staging` etc. - /// - public List Tags { get; set; } + public JsonString Metadata { get; set; } } } diff --git a/Tzkt.Api/Repositories/SoftwareRepository.cs b/Tzkt.Api/Repositories/SoftwareRepository.cs index 5eb3711ef..bae64993f 100644 --- a/Tzkt.Api/Repositories/SoftwareRepository.cs +++ b/Tzkt.Api/Repositories/SoftwareRepository.cs @@ -42,15 +42,12 @@ public async Task> Get(SortParameter sort, OffsetParameter return rows.Select(row => new Software { BlocksCount = row.BlocksCount, - CommitDate = row.CommitDate, - CommitHash = row.CommitHash, FirstLevel = row.FirstLevel, FirstTime = Time[row.FirstLevel], LastLevel = row.LastLevel, LastTime = Time[row.LastLevel], ShortHash = row.ShortHash, - Tags = row.Tags == null ? null : new List(row.Tags), - Version = row.Version + Metadata = new JsonString(row.Metadata) }); } @@ -62,15 +59,12 @@ public async Task Get(SortParameter sort, OffsetParameter offset, in switch (field) { case "blocksCount": columns.Add(@"""BlocksCount"""); break; - case "commitDate": columns.Add(@"""CommitDate"""); break; - case "commitHash": columns.Add(@"""CommitHash"""); break; case "firstLevel": columns.Add(@"""FirstLevel"""); break; case "firstTime": columns.Add(@"""FirstLevel"""); break; case "lastLevel": columns.Add(@"""LastLevel"""); break; case "lastTime": columns.Add(@"""LastLevel"""); break; case "shortHash": columns.Add(@"""ShortHash"""); break; - case "tags": columns.Add(@"""Tags"""); break; - case "version": columns.Add(@"""Version"""); break; + case "metadata": columns.Add(@"""Metadata"""); break; } } @@ -101,14 +95,6 @@ public async Task Get(SortParameter sort, OffsetParameter offset, in foreach (var row in rows) result[j++][i] = row.BlocksCount; break; - case "commitDate": - foreach (var row in rows) - result[j++][i] = row.CommitDate; - break; - case "commitHash": - foreach (var row in rows) - result[j++][i] = row.CommitHash; - break; case "firstLevel": foreach (var row in rows) result[j++][i] = row.FirstLevel; @@ -129,13 +115,9 @@ public async Task Get(SortParameter sort, OffsetParameter offset, in foreach (var row in rows) result[j++][i] = row.ShortHash; break; - case "tags": - foreach (var row in rows) - result[j++][i] = row.Tags == null ? null : new List(row.Tags); - break; - case "version": + case "metadata": foreach (var row in rows) - result[j++][i] = row.Version; + result[j++][i] = new JsonString(row.Metadata); break; } } @@ -149,15 +131,12 @@ public async Task Get(SortParameter sort, OffsetParameter offset, int switch (field) { case "blocksCount": columns.Add(@"""BlocksCount"""); break; - case "commitDate": columns.Add(@"""CommitDate"""); break; - case "commitHash": columns.Add(@"""CommitHash"""); break; case "firstLevel": columns.Add(@"""FirstLevel"""); break; case "firstTime": columns.Add(@"""FirstLevel"""); break; case "lastLevel": columns.Add(@"""LastLevel"""); break; case "lastTime": columns.Add(@"""LastLevel"""); break; case "shortHash": columns.Add(@"""ShortHash"""); break; - case "tags": columns.Add(@"""Tags"""); break; - case "version": columns.Add(@"""Version"""); break; + case "metadata": columns.Add(@"""Metadata"""); break; } if (columns.Count == 0) @@ -184,14 +163,6 @@ public async Task Get(SortParameter sort, OffsetParameter offset, int foreach (var row in rows) result[j++] = row.BlocksCount; break; - case "commitDate": - foreach (var row in rows) - result[j++] = row.CommitDate; - break; - case "commitHash": - foreach (var row in rows) - result[j++] = row.CommitHash; - break; case "firstLevel": foreach (var row in rows) result[j++] = row.FirstLevel; @@ -212,13 +183,9 @@ public async Task Get(SortParameter sort, OffsetParameter offset, int foreach (var row in rows) result[j++] = row.ShortHash; break; - case "tags": - foreach (var row in rows) - result[j++] = row.Tags == null ? null : new List(row.Tags); - break; - case "version": + case "metadata": foreach (var row in rows) - result[j++] = row.Version; + result[j++] = new JsonString(row.Metadata); break; } diff --git a/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs b/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs index 623469b9b..167a580d9 100644 --- a/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs +++ b/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs @@ -25,12 +25,14 @@ public SoftwareMetadataService(TimeCache time, IConfiguration config, ILogger>'version' as ""Version"", ""Metadata""->>'commitDate' as ""CommitDate"" + FROM ""Software"""); Aliases = rows.ToDictionary(row => (int)row.Id, row => new SoftwareAlias { Version = row.Version, - Date = row.CommitDate ?? Time[row.FirstLevel] + Date = DateTime.TryParse(row.CommitDate, out DateTime dt) ? dt : Time[row.FirstLevel] }); Logger.LogDebug($"Loaded {Aliases.Count} software metadata"); @@ -46,14 +48,14 @@ public SoftwareAlias this[int id] { using var db = GetConnection(); var row = db.QueryFirst($@" - SELECT ""Id"", ""FirstLevel"", ""Version"", ""CommitDate"" + SELECT ""Id"", ""FirstLevel"", ""Metadata""->>'version' as ""Version"", ""Metadata""->>'commitDate' as ""CommitDate"" FROM ""Software"" WHERE ""Id"" = {id}"); alias = new SoftwareAlias { Version = row.Version, - Date = row.CommitDate ?? Time[row.FirstLevel] + Date = DateTime.TryParse(row.CommitDate, out DateTime dt) ? dt : Time[row.FirstLevel] }; Aliases.Add(id, alias); From 3381a1d0d6b5eb6b882fa65d007e081ac195f99b Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Wed, 31 Mar 2021 23:47:52 +0300 Subject: [PATCH 07/35] Add start and end time to cycle model --- Tzkt.Api/Models/Baking/Cycle.cs | 20 +++++++ Tzkt.Api/Repositories/CyclesRepository.cs | 52 ++++++++++++++++++- Tzkt.Data/Models/Baking/Cycle.cs | 2 + .../Activation/ProtoActivator.Cycles.cs | 2 + .../Handlers/Proto1/Commits/CycleCommit.cs | 2 + .../Handlers/Proto4/Commits/CycleCommit.cs | 2 + 6 files changed, 79 insertions(+), 1 deletion(-) diff --git a/Tzkt.Api/Models/Baking/Cycle.cs b/Tzkt.Api/Models/Baking/Cycle.cs index e391bd1cf..f16259d18 100644 --- a/Tzkt.Api/Models/Baking/Cycle.cs +++ b/Tzkt.Api/Models/Baking/Cycle.cs @@ -9,6 +9,26 @@ public class Cycle /// public int Index { get; set; } + /// + /// Level of the first block in the cycle + /// + public int FirstLevel { get; set; } + + /// + /// Timestamp of the first block in the cycle + /// + public DateTime StartTime { get; set; } + + /// + /// Level of the last block in the cycle + /// + public int LastLevel { get; set; } + + /// + /// Timestamp of the last block in the cycle + /// + public DateTime EndTime { get; set; } + /// /// Index of the snapshot /// diff --git a/Tzkt.Api/Repositories/CyclesRepository.cs b/Tzkt.Api/Repositories/CyclesRepository.cs index 489633330..ea0b8149a 100644 --- a/Tzkt.Api/Repositories/CyclesRepository.cs +++ b/Tzkt.Api/Repositories/CyclesRepository.cs @@ -14,11 +14,13 @@ public class CyclesRepository : DbConnection { readonly ProtocolsCache Protocols; readonly QuotesCache Quotes; + readonly TimeCache Times; - public CyclesRepository(ProtocolsCache protocols, QuotesCache quotes, IConfiguration config) : base(config) + public CyclesRepository(ProtocolsCache protocols, QuotesCache quotes, TimeCache times, IConfiguration config) : base(config) { Protocols = protocols; Quotes = quotes; + Times = times; } public async Task GetCount() @@ -43,6 +45,10 @@ public async Task Get(int index, Symbols quote) return new Cycle { Index = row.Index, + FirstLevel = row.FirstLevel, + StartTime = Times[row.FirstLevel], + LastLevel = row.LastLevel, + EndTime = Times[row.LastLevel], RandomSeed = row.Seed, SnapshotIndex = row.SnapshotIndex, SnapshotLevel = row.SnapshotLevel, @@ -73,6 +79,10 @@ public async Task> Get( return rows.Select(row => new Cycle { Index = row.Index, + FirstLevel = row.FirstLevel, + StartTime = Times[row.FirstLevel], + LastLevel = row.LastLevel, + EndTime = Times[row.LastLevel], RandomSeed = row.Seed, SnapshotIndex = row.SnapshotIndex, SnapshotLevel = row.SnapshotLevel, @@ -99,6 +109,10 @@ public async Task Get( switch (field) { case "index": columns.Add(@"""Index"""); break; + case "firstLevel": columns.Add(@"""FirstLevel"""); break; + case "startTime": columns.Add(@"""FirstLevel"""); break; + case "lastLevel": columns.Add(@"""LastLevel"""); break; + case "endTime": columns.Add(@"""LastLevel"""); break; case "randomSeed": columns.Add(@"""Seed"""); break; case "snapshotIndex": columns.Add(@"""SnapshotIndex"""); break; case "snapshotLevel": columns.Add(@"""SnapshotLevel"""); break; @@ -133,6 +147,22 @@ public async Task Get( foreach (var row in rows) result[j++][i] = row.Index; break; + case "firstLevel": + foreach (var row in rows) + result[j++][i] = row.FirstLevel; + break; + case "startTime": + foreach (var row in rows) + result[j++][i] = Times[row.FirstLevel]; + break; + case "lastLevel": + foreach (var row in rows) + result[j++][i] = row.LastLevel; + break; + case "endTime": + foreach (var row in rows) + result[j++][i] = Times[row.LastLevel]; + break; case "randomSeed": foreach (var row in rows) result[j++][i] = row.Seed; @@ -188,6 +218,10 @@ public async Task Get( switch (field) { case "index": columns.Add(@"""Index"""); break; + case "firstLevel": columns.Add(@"""FirstLevel"""); break; + case "startTime": columns.Add(@"""FirstLevel"""); break; + case "lastLevel": columns.Add(@"""LastLevel"""); break; + case "endTime": columns.Add(@"""LastLevel"""); break; case "randomSeed": columns.Add(@"""Seed"""); break; case "snapshotIndex": columns.Add(@"""SnapshotIndex"""); break; case "snapshotLevel": columns.Add(@"""SnapshotLevel"""); break; @@ -219,6 +253,22 @@ public async Task Get( foreach (var row in rows) result[j++] = row.Index; break; + case "firstLevel": + foreach (var row in rows) + result[j++] = row.FirstLevel; + break; + case "startTime": + foreach (var row in rows) + result[j++] = Times[row.FirstLevel]; + break; + case "lastLevel": + foreach (var row in rows) + result[j++] = row.LastLevel; + break; + case "endTime": + foreach (var row in rows) + result[j++] = Times[row.LastLevel]; + break; case "randomSeed": foreach (var row in rows) result[j++] = row.Seed; diff --git a/Tzkt.Data/Models/Baking/Cycle.cs b/Tzkt.Data/Models/Baking/Cycle.cs index 199fb85b4..9bb1eb425 100644 --- a/Tzkt.Data/Models/Baking/Cycle.cs +++ b/Tzkt.Data/Models/Baking/Cycle.cs @@ -6,6 +6,8 @@ public class Cycle { public int Id { get; set; } public int Index { get; set; } + public int FirstLevel { get; set; } + public int LastLevel { get; set; } public int SnapshotIndex { get; set; } public int SnapshotLevel { get; set; } public int TotalRolls { get; set; } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Activation/ProtoActivator.Cycles.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Activation/ProtoActivator.Cycles.cs index 093d39bce..3cd65f77d 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Activation/ProtoActivator.Cycles.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Activation/ProtoActivator.Cycles.cs @@ -26,6 +26,8 @@ public async Task BootstrapCycles(Protocol protocol, List accounts) Db.Cycles.Add(new Cycle { Index = cycle, + FirstLevel = cycle * protocol.BlocksPerCycle + 1, + LastLevel = (cycle + 1) * protocol.BlocksPerCycle, SnapshotIndex = 0, SnapshotLevel = 1, TotalRolls = totalRolls, diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/CycleCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/CycleCommit.cs index 617ccb308..fdfbb614d 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/CycleCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/CycleCommit.cs @@ -56,6 +56,8 @@ public virtual async Task Apply(Block block) FutureCycle = new Cycle { Index = futureCycle, + FirstLevel = futureCycle * block.Protocol.BlocksPerCycle + 1, + LastLevel = (futureCycle + 1) * block.Protocol.BlocksPerCycle, SnapshotIndex = rawCycle.RequiredInt32("roll_snapshot"), SnapshotLevel = snapshotLevel, TotalRolls = Snapshots.Values.Sum(x => (int)(x.StakingBalance / block.Protocol.TokensPerRoll)), diff --git a/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/CycleCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/CycleCommit.cs index 823c142bb..18b6b9bd8 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/CycleCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto4/Commits/CycleCommit.cs @@ -59,6 +59,8 @@ public override async Task Apply(Block block) FutureCycle = new Cycle { Index = futureCycle, + FirstLevel = futureCycle * block.Protocol.BlocksPerCycle + 1, + LastLevel = (futureCycle + 1) * block.Protocol.BlocksPerCycle, SnapshotIndex = rawCycle.RequiredInt32("roll_snapshot"), SnapshotLevel = snapshotLevel, TotalRolls = Snapshots.Values.Sum(x => (int)(x.StakingBalance / snapshotProtocol.TokensPerRoll)), From afb648527d66d03da5957f774bd4894aeb9a301c Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 1 Apr 2021 00:07:13 +0300 Subject: [PATCH 08/35] DB migration --- .../20210331210536_Bigmaps.Designer.cs | 2778 +++++++++++++++++ .../Migrations/20210331210536_Bigmaps.cs | 386 +++ .../Migrations/TzktContextModelSnapshot.cs | 299 +- 3 files changed, 3414 insertions(+), 49 deletions(-) create mode 100644 Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs create mode 100644 Tzkt.Data/Migrations/20210331210536_Bigmaps.cs diff --git a/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs b/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs new file mode 100644 index 000000000..59727b2dd --- /dev/null +++ b/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs @@ -0,0 +1,2778 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Tzkt.Data; + +namespace Tzkt.Data.Migrations +{ + [DbContext(typeof(TzktContext))] + [Migration("20210331210536_Bigmaps")] + partial class Bigmaps + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Relational:MaxIdentifierLength", 63) + .HasAnnotation("ProductVersion", "5.0.4") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + modelBuilder.Entity("Tzkt.Data.Models.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Address") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character(36)") + .IsFixedLength(true); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("ContractsCount") + .HasColumnType("integer"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("DelegationLevel") + .HasColumnType("integer"); + + b.Property("DelegationsCount") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("MigrationsCount") + .HasColumnType("integer"); + + b.Property("OriginationsCount") + .HasColumnType("integer"); + + b.Property("RevealsCount") + .HasColumnType("integer"); + + b.Property("Staked") + .HasColumnType("boolean"); + + b.Property("TransactionsCount") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("smallint"); + + b.HasKey("Id"); + + b.HasIndex("Address") + .IsUnique(); + + b.HasIndex("DelegateId"); + + b.HasIndex("FirstLevel"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Metadata") + .HasMethod("GIN") + .HasOperators(new[] { "jsonb_path_ops" }); + + b.HasIndex("Staked"); + + b.HasIndex("Type"); + + b.ToTable("Accounts"); + + b.HasDiscriminator("Type"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ActivationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccountId") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.ToTable("ActivationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.AppState", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountCounter") + .HasColumnType("integer"); + + b.Property("AccountsCount") + .HasColumnType("integer"); + + b.Property("ActivationOpsCount") + .HasColumnType("integer"); + + b.Property("BallotOpsCount") + .HasColumnType("integer"); + + b.Property("BigMapCounter") + .HasColumnType("integer"); + + b.Property("BigMapKeyCounter") + .HasColumnType("integer"); + + b.Property("BigMapUpdateCounter") + .HasColumnType("integer"); + + b.Property("BlocksCount") + .HasColumnType("integer"); + + b.Property("CommitmentsCount") + .HasColumnType("integer"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("CyclesCount") + .HasColumnType("integer"); + + b.Property("DelegationOpsCount") + .HasColumnType("integer"); + + b.Property("DoubleBakingOpsCount") + .HasColumnType("integer"); + + b.Property("DoubleEndorsingOpsCount") + .HasColumnType("integer"); + + b.Property("EndorsementOpsCount") + .HasColumnType("integer"); + + b.Property("Hash") + .HasColumnType("text"); + + b.Property("KnownHead") + .HasColumnType("integer"); + + b.Property("LastSync") + .HasColumnType("timestamp without time zone"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("ManagerCounter") + .HasColumnType("integer"); + + b.Property("MigrationOpsCount") + .HasColumnType("integer"); + + b.Property("NextProtocol") + .HasColumnType("text"); + + b.Property("NonceRevelationOpsCount") + .HasColumnType("integer"); + + b.Property("OperationCounter") + .HasColumnType("integer"); + + b.Property("OriginationOpsCount") + .HasColumnType("integer"); + + b.Property("ProposalOpsCount") + .HasColumnType("integer"); + + b.Property("ProposalsCount") + .HasColumnType("integer"); + + b.Property("Protocol") + .HasColumnType("text"); + + b.Property("ProtocolsCount") + .HasColumnType("integer"); + + b.Property("QuoteBtc") + .HasColumnType("double precision"); + + b.Property("QuoteCny") + .HasColumnType("double precision"); + + b.Property("QuoteEth") + .HasColumnType("double precision"); + + b.Property("QuoteEur") + .HasColumnType("double precision"); + + b.Property("QuoteJpy") + .HasColumnType("double precision"); + + b.Property("QuoteKrw") + .HasColumnType("double precision"); + + b.Property("QuoteLevel") + .HasColumnType("integer"); + + b.Property("QuoteUsd") + .HasColumnType("double precision"); + + b.Property("RevealOpsCount") + .HasColumnType("integer"); + + b.Property("RevelationPenaltyOpsCount") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("TransactionOpsCount") + .HasColumnType("integer"); + + b.Property("VotingEpoch") + .HasColumnType("integer"); + + b.Property("VotingPeriod") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("AppState"); + + b.HasData( + new + { + Id = -1, + AccountCounter = 0, + AccountsCount = 0, + ActivationOpsCount = 0, + BallotOpsCount = 0, + BigMapCounter = 0, + BigMapKeyCounter = 0, + BigMapUpdateCounter = 0, + BlocksCount = 0, + CommitmentsCount = 0, + Cycle = -1, + CyclesCount = 0, + DelegationOpsCount = 0, + DoubleBakingOpsCount = 0, + DoubleEndorsingOpsCount = 0, + EndorsementOpsCount = 0, + Hash = "", + KnownHead = 0, + LastSync = new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + Level = -1, + ManagerCounter = 0, + MigrationOpsCount = 0, + NextProtocol = "", + NonceRevelationOpsCount = 0, + OperationCounter = 0, + OriginationOpsCount = 0, + ProposalOpsCount = 0, + ProposalsCount = 0, + Protocol = "", + ProtocolsCount = 0, + QuoteBtc = 0.0, + QuoteCny = 0.0, + QuoteEth = 0.0, + QuoteEur = 0.0, + QuoteJpy = 0.0, + QuoteKrw = 0.0, + QuoteLevel = -1, + QuoteUsd = 0.0, + RevealOpsCount = 0, + RevelationPenaltyOpsCount = 0, + Timestamp = new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + TransactionOpsCount = 0, + VotingEpoch = -1, + VotingPeriod = -1 + }); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BakerCycle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("BlockDeposits") + .HasColumnType("bigint"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("DelegatedBalance") + .HasColumnType("bigint"); + + b.Property("DelegatorsCount") + .HasColumnType("integer"); + + b.Property("DoubleBakingLostDeposits") + .HasColumnType("bigint"); + + b.Property("DoubleBakingLostFees") + .HasColumnType("bigint"); + + b.Property("DoubleBakingLostRewards") + .HasColumnType("bigint"); + + b.Property("DoubleBakingRewards") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingLostDeposits") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingLostFees") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingLostRewards") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingRewards") + .HasColumnType("bigint"); + + b.Property("EndorsementDeposits") + .HasColumnType("bigint"); + + b.Property("EndorsementRewards") + .HasColumnType("bigint"); + + b.Property("Endorsements") + .HasColumnType("integer"); + + b.Property("ExpectedBlocks") + .HasColumnType("double precision"); + + b.Property("ExpectedEndorsements") + .HasColumnType("double precision"); + + b.Property("ExtraBlockFees") + .HasColumnType("bigint"); + + b.Property("ExtraBlockRewards") + .HasColumnType("bigint"); + + b.Property("ExtraBlocks") + .HasColumnType("integer"); + + b.Property("FutureBlockDeposits") + .HasColumnType("bigint"); + + b.Property("FutureBlockRewards") + .HasColumnType("bigint"); + + b.Property("FutureBlocks") + .HasColumnType("integer"); + + b.Property("FutureEndorsementDeposits") + .HasColumnType("bigint"); + + b.Property("FutureEndorsementRewards") + .HasColumnType("bigint"); + + b.Property("FutureEndorsements") + .HasColumnType("integer"); + + b.Property("MissedEndorsementRewards") + .HasColumnType("bigint"); + + b.Property("MissedEndorsements") + .HasColumnType("integer"); + + b.Property("MissedExtraBlockFees") + .HasColumnType("bigint"); + + b.Property("MissedExtraBlockRewards") + .HasColumnType("bigint"); + + b.Property("MissedExtraBlocks") + .HasColumnType("integer"); + + b.Property("MissedOwnBlockFees") + .HasColumnType("bigint"); + + b.Property("MissedOwnBlockRewards") + .HasColumnType("bigint"); + + b.Property("MissedOwnBlocks") + .HasColumnType("integer"); + + b.Property("OwnBlockFees") + .HasColumnType("bigint"); + + b.Property("OwnBlockRewards") + .HasColumnType("bigint"); + + b.Property("OwnBlocks") + .HasColumnType("integer"); + + b.Property("RevelationLostFees") + .HasColumnType("bigint"); + + b.Property("RevelationLostRewards") + .HasColumnType("bigint"); + + b.Property("RevelationRewards") + .HasColumnType("bigint"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("StakingBalance") + .HasColumnType("bigint"); + + b.Property("UncoveredEndorsementRewards") + .HasColumnType("bigint"); + + b.Property("UncoveredEndorsements") + .HasColumnType("integer"); + + b.Property("UncoveredExtraBlockFees") + .HasColumnType("bigint"); + + b.Property("UncoveredExtraBlockRewards") + .HasColumnType("bigint"); + + b.Property("UncoveredExtraBlocks") + .HasColumnType("integer"); + + b.Property("UncoveredOwnBlockFees") + .HasColumnType("bigint"); + + b.Property("UncoveredOwnBlockRewards") + .HasColumnType("bigint"); + + b.Property("UncoveredOwnBlocks") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Cycle"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Cycle", "BakerId") + .IsUnique(); + + b.ToTable("BakerCycles"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BakingRight", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Priority") + .HasColumnType("integer"); + + b.Property("Slots") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("Type") + .HasColumnType("smallint"); + + b.HasKey("Id"); + + b.HasIndex("Cycle"); + + b.HasIndex("Level"); + + b.HasIndex("Cycle", "BakerId"); + + b.ToTable("BakingRights"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BallotOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Period") + .HasColumnType("integer"); + + b.Property("ProposalId") + .HasColumnType("integer"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("Vote") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Epoch"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("Period"); + + b.HasIndex("ProposalId"); + + b.HasIndex("SenderId"); + + b.ToTable("BallotOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMap", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Active") + .HasColumnType("boolean"); + + b.Property("ActiveKeys") + .HasColumnType("integer"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("KeyType") + .HasColumnType("bytea"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Ptr") + .HasColumnType("integer"); + + b.Property("StoragePath") + .HasColumnType("text"); + + b.Property("TotalKeys") + .HasColumnType("integer"); + + b.Property("Updates") + .HasColumnType("integer"); + + b.Property("ValueType") + .HasColumnType("bytea"); + + b.HasKey("Id"); + + b.HasAlternateKey("Ptr"); + + b.HasIndex("ContractId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Ptr") + .IsUnique(); + + b.ToTable("BigMaps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMapKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Active") + .HasColumnType("boolean"); + + b.Property("BigMapPtr") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("JsonKey") + .HasColumnType("jsonb"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("KeyHash") + .HasMaxLength(54) + .HasColumnType("character varying(54)"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("RawKey") + .HasColumnType("bytea"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("Updates") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BigMapPtr"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("LastLevel"); + + b.HasIndex("BigMapPtr", "Active") + .HasFilter("\"Active\" = true"); + + b.HasIndex("BigMapPtr", "KeyHash"); + + b.ToTable("BigMapKeys"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMapUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("BigMapKeyId") + .HasColumnType("integer"); + + b.Property("BigMapPtr") + .HasColumnType("integer"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("TransactionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BigMapKeyId") + .HasFilter("\"BigMapKeyId\" is not null"); + + b.HasIndex("BigMapPtr"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("OriginationId") + .HasFilter("\"OriginationId\" is not null"); + + b.HasIndex("TransactionId") + .HasFilter("\"TransactionId\" is not null"); + + b.ToTable("BigMapUpdates"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Block", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Deposit") + .HasColumnType("bigint"); + + b.Property("Events") + .HasColumnType("integer"); + + b.Property("Fees") + .HasColumnType("bigint"); + + b.Property("Hash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Operations") + .HasColumnType("integer"); + + b.Property("Priority") + .HasColumnType("integer"); + + b.Property("ProtoCode") + .HasColumnType("integer"); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("RevelationId") + .HasColumnType("integer"); + + b.Property("Reward") + .HasColumnType("bigint"); + + b.Property("SoftwareId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("Validations") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Hash") + .IsUnique(); + + b.HasIndex("Level") + .IsUnique(); + + b.HasIndex("ProtoCode"); + + b.HasIndex("RevelationId") + .IsUnique(); + + b.HasIndex("SoftwareId"); + + b.ToTable("Blocks"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Commitment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("Address") + .IsRequired() + .HasMaxLength(37) + .HasColumnType("character(37)") + .IsFixedLength(true); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Address") + .IsUnique(); + + b.HasIndex("Id") + .IsUnique(); + + b.ToTable("Commitments"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Cycle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("Index") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Seed") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character(64)") + .IsFixedLength(true); + + b.Property("SnapshotIndex") + .HasColumnType("integer"); + + b.Property("SnapshotLevel") + .HasColumnType("integer"); + + b.Property("TotalBakers") + .HasColumnType("integer"); + + b.Property("TotalDelegated") + .HasColumnType("bigint"); + + b.Property("TotalDelegators") + .HasColumnType("integer"); + + b.Property("TotalRolls") + .HasColumnType("integer"); + + b.Property("TotalStaking") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasAlternateKey("Index"); + + b.HasIndex("Index") + .IsUnique(); + + b.ToTable("Cycles"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DelegationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("Amount") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Nonce") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("PrevDelegateId") + .HasColumnType("integer"); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("DelegateId"); + + b.HasIndex("InitiatorId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("PrevDelegateId"); + + b.HasIndex("SenderId"); + + b.ToTable("DelegationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DelegatorCycle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("DelegatorId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Cycle"); + + b.HasIndex("DelegatorId"); + + b.HasIndex("Cycle", "BakerId"); + + b.HasIndex("Cycle", "DelegatorId") + .IsUnique(); + + b.ToTable("DelegatorCycles"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleBakingOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccusedLevel") + .HasColumnType("integer"); + + b.Property("AccuserId") + .HasColumnType("integer"); + + b.Property("AccuserReward") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OffenderId") + .HasColumnType("integer"); + + b.Property("OffenderLostDeposit") + .HasColumnType("bigint"); + + b.Property("OffenderLostFee") + .HasColumnType("bigint"); + + b.Property("OffenderLostReward") + .HasColumnType("bigint"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccuserId"); + + b.HasIndex("Level"); + + b.HasIndex("OffenderId"); + + b.HasIndex("OpHash"); + + b.ToTable("DoubleBakingOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleEndorsingOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccusedLevel") + .HasColumnType("integer"); + + b.Property("AccuserId") + .HasColumnType("integer"); + + b.Property("AccuserReward") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OffenderId") + .HasColumnType("integer"); + + b.Property("OffenderLostDeposit") + .HasColumnType("bigint"); + + b.Property("OffenderLostFee") + .HasColumnType("bigint"); + + b.Property("OffenderLostReward") + .HasColumnType("bigint"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccuserId"); + + b.HasIndex("Level"); + + b.HasIndex("OffenderId"); + + b.HasIndex("OpHash"); + + b.ToTable("DoubleEndorsingOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.EndorsementOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Deposit") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("Reward") + .HasColumnType("bigint"); + + b.Property("Slots") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("DelegateId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.ToTable("EndorsementOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.MigrationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("BalanceChange") + .HasColumnType("bigint"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("NewScriptId") + .HasColumnType("integer"); + + b.Property("NewStorageId") + .HasColumnType("integer"); + + b.Property("OldScriptId") + .HasColumnType("integer"); + + b.Property("OldStorageId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.HasIndex("Level"); + + b.HasIndex("NewScriptId"); + + b.HasIndex("NewStorageId"); + + b.HasIndex("OldScriptId"); + + b.HasIndex("OldStorageId"); + + b.ToTable("MigrationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.NonceRevelationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("RevealedLevel") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("SenderId"); + + b.ToTable("NonceRevelationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.OriginationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("ManagerId") + .HasColumnType("integer"); + + b.Property("Nonce") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("ScriptId") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageId") + .HasColumnType("integer"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("ContractId"); + + b.HasIndex("DelegateId"); + + b.HasIndex("InitiatorId"); + + b.HasIndex("Level"); + + b.HasIndex("ManagerId"); + + b.HasIndex("OpHash"); + + b.HasIndex("ScriptId"); + + b.HasIndex("SenderId"); + + b.HasIndex("StorageId"); + + b.ToTable("OriginationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Proposal", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("FirstPeriod") + .HasColumnType("integer"); + + b.Property("Hash") + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("LastPeriod") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Upvotes") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Epoch"); + + b.HasIndex("Hash"); + + b.ToTable("Proposals"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ProposalOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Duplicated") + .HasColumnType("boolean"); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Period") + .HasColumnType("integer"); + + b.Property("ProposalId") + .HasColumnType("integer"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("Epoch"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("Period"); + + b.HasIndex("ProposalId"); + + b.HasIndex("SenderId"); + + b.ToTable("ProposalOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Protocol", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BallotQuorumMax") + .HasColumnType("integer"); + + b.Property("BallotQuorumMin") + .HasColumnType("integer"); + + b.Property("BlockDeposit") + .HasColumnType("bigint"); + + b.Property("BlockReward0") + .HasColumnType("bigint"); + + b.Property("BlockReward1") + .HasColumnType("bigint"); + + b.Property("BlocksPerCommitment") + .HasColumnType("integer"); + + b.Property("BlocksPerCycle") + .HasColumnType("integer"); + + b.Property("BlocksPerSnapshot") + .HasColumnType("integer"); + + b.Property("BlocksPerVoting") + .HasColumnType("integer"); + + b.Property("ByteCost") + .HasColumnType("integer"); + + b.Property("Code") + .HasColumnType("integer"); + + b.Property("EndorsementDeposit") + .HasColumnType("bigint"); + + b.Property("EndorsementReward0") + .HasColumnType("bigint"); + + b.Property("EndorsementReward1") + .HasColumnType("bigint"); + + b.Property("EndorsersPerBlock") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("HardBlockGasLimit") + .HasColumnType("integer"); + + b.Property("HardOperationGasLimit") + .HasColumnType("integer"); + + b.Property("HardOperationStorageLimit") + .HasColumnType("integer"); + + b.Property("Hash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("NoRewardCycles") + .HasColumnType("integer"); + + b.Property("OriginationSize") + .HasColumnType("integer"); + + b.Property("PreservedCycles") + .HasColumnType("integer"); + + b.Property("ProposalQuorum") + .HasColumnType("integer"); + + b.Property("RampUpCycles") + .HasColumnType("integer"); + + b.Property("RevelationReward") + .HasColumnType("bigint"); + + b.Property("TimeBetweenBlocks") + .HasColumnType("integer"); + + b.Property("TokensPerRoll") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("Protocols"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Quote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Btc") + .HasColumnType("double precision"); + + b.Property("Cny") + .HasColumnType("double precision"); + + b.Property("Eth") + .HasColumnType("double precision"); + + b.Property("Eur") + .HasColumnType("double precision"); + + b.Property("Jpy") + .HasColumnType("double precision"); + + b.Property("Krw") + .HasColumnType("double precision"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("Usd") + .HasColumnType("double precision"); + + b.HasKey("Id"); + + b.HasIndex("Level") + .IsUnique(); + + b.ToTable("Quotes"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevealOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("SenderId"); + + b.ToTable("RevealOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevelationPenaltyOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("LostFees") + .HasColumnType("bigint"); + + b.Property("LostReward") + .HasColumnType("bigint"); + + b.Property("MissedLevel") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Level"); + + b.ToTable("RevelationPenaltyOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Script", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("CodeSchema") + .HasColumnType("bytea"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("Current") + .HasColumnType("boolean"); + + b.Property("MigrationId") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("ParameterSchema") + .HasColumnType("bytea"); + + b.Property("StorageSchema") + .HasColumnType("bytea"); + + b.HasKey("Id"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("ContractId", "Current") + .HasFilter("\"Current\" = true"); + + b.ToTable("Scripts"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.SnapshotBalance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Level"); + + b.ToTable("SnapshotBalances"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BlocksCount") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("ShortHash") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character(8)") + .IsFixedLength(true); + + b.HasKey("Id"); + + b.ToTable("Software"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Statistics", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("Date") + .HasColumnType("timestamp without time zone"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("TotalActivated") + .HasColumnType("bigint"); + + b.Property("TotalBootstrapped") + .HasColumnType("bigint"); + + b.Property("TotalBurned") + .HasColumnType("bigint"); + + b.Property("TotalCommitments") + .HasColumnType("bigint"); + + b.Property("TotalCreated") + .HasColumnType("bigint"); + + b.Property("TotalFrozen") + .HasColumnType("bigint"); + + b.Property("TotalVested") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("Cycle") + .IsUnique() + .HasFilter("\"Cycle\" IS NOT NULL"); + + b.HasIndex("Date") + .IsUnique() + .HasFilter("\"Date\" IS NOT NULL"); + + b.HasIndex("Level") + .IsUnique(); + + b.ToTable("Statistics"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Storage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("Current") + .HasColumnType("boolean"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("MigrationId") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("TransactionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ContractId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("ContractId", "Current") + .HasFilter("\"Current\" = true"); + + b.ToTable("Storages"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.TransactionOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("Amount") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("Entrypoint") + .HasColumnType("text"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("InternalDelegations") + .HasColumnType("smallint"); + + b.Property("InternalOperations") + .HasColumnType("smallint"); + + b.Property("InternalOriginations") + .HasColumnType("smallint"); + + b.Property("InternalTransactions") + .HasColumnType("smallint"); + + b.Property("JsonParameters") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Nonce") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("RawParameters") + .HasColumnType("bytea"); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageId") + .HasColumnType("integer"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("TargetId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("InitiatorId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("SenderId"); + + b.HasIndex("StorageId"); + + b.HasIndex("TargetId"); + + b.ToTable("TransactionOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.VotingPeriod", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BallotsQuorum") + .HasColumnType("integer"); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("Index") + .HasColumnType("integer"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("NayBallots") + .HasColumnType("integer"); + + b.Property("NayRolls") + .HasColumnType("integer"); + + b.Property("ParticipationEma") + .HasColumnType("integer"); + + b.Property("PassBallots") + .HasColumnType("integer"); + + b.Property("PassRolls") + .HasColumnType("integer"); + + b.Property("ProposalsCount") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Supermajority") + .HasColumnType("integer"); + + b.Property("TopRolls") + .HasColumnType("integer"); + + b.Property("TopUpvotes") + .HasColumnType("integer"); + + b.Property("TotalBakers") + .HasColumnType("integer"); + + b.Property("TotalRolls") + .HasColumnType("integer"); + + b.Property("UpvotesQuorum") + .HasColumnType("integer"); + + b.Property("YayBallots") + .HasColumnType("integer"); + + b.Property("YayRolls") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasAlternateKey("Index"); + + b.HasIndex("Epoch"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Index") + .IsUnique(); + + b.ToTable("VotingPeriods"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.VotingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Period") + .HasColumnType("integer"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Period"); + + b.HasIndex("Period", "BakerId") + .IsUnique(); + + b.ToTable("VotingSnapshots"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Contract", b => + { + b.HasBaseType("Tzkt.Data.Models.Account"); + + b.Property("CreatorId") + .HasColumnType("integer"); + + b.Property("Kind") + .HasColumnType("smallint"); + + b.Property("ManagerId") + .HasColumnType("integer"); + + b.Property("Spendable") + .HasColumnType("boolean"); + + b.Property("Tzips") + .HasColumnType("integer"); + + b.Property("WeirdDelegateId") + .HasColumnType("integer"); + + b.HasIndex("CreatorId"); + + b.HasIndex("ManagerId"); + + b.HasIndex("WeirdDelegateId"); + + b.HasIndex("Type", "Kind") + .HasFilter("\"Type\" = 2"); + + b.HasDiscriminator().HasValue((byte)2); + }); + + modelBuilder.Entity("Tzkt.Data.Models.User", b => + { + b.HasBaseType("Tzkt.Data.Models.Account"); + + b.Property("Activated") + .HasColumnType("boolean"); + + b.Property("PublicKey") + .HasMaxLength(55) + .HasColumnType("character varying(55)"); + + b.Property("Revealed") + .HasColumnType("boolean"); + + b.HasDiscriminator().HasValue((byte)0); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Delegate", b => + { + b.HasBaseType("Tzkt.Data.Models.User"); + + b.Property("ActivationLevel") + .HasColumnType("integer"); + + b.Property("BallotsCount") + .HasColumnType("integer"); + + b.Property("BlocksCount") + .HasColumnType("integer"); + + b.Property("DeactivationLevel") + .HasColumnType("integer"); + + b.Property("DelegatorsCount") + .HasColumnType("integer"); + + b.Property("DoubleBakingCount") + .HasColumnType("integer"); + + b.Property("DoubleEndorsingCount") + .HasColumnType("integer"); + + b.Property("EndorsementsCount") + .HasColumnType("integer"); + + b.Property("FrozenDeposits") + .HasColumnType("bigint"); + + b.Property("FrozenFees") + .HasColumnType("bigint"); + + b.Property("FrozenRewards") + .HasColumnType("bigint"); + + b.Property("NonceRevelationsCount") + .HasColumnType("integer"); + + b.Property("ProposalsCount") + .HasColumnType("integer"); + + b.Property("RevelationPenaltiesCount") + .HasColumnType("integer"); + + b.Property("SoftwareId") + .HasColumnType("integer"); + + b.Property("StakingBalance") + .HasColumnType("bigint"); + + b.HasIndex("SoftwareId"); + + b.HasIndex("Type", "Staked") + .HasFilter("\"Type\" = 1"); + + b.HasDiscriminator().HasValue((byte)1); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Account", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany("DelegatedAccounts") + .HasForeignKey("DelegateId"); + + b.HasOne("Tzkt.Data.Models.Block", "FirstBlock") + .WithMany("CreatedAccounts") + .HasForeignKey("FirstLevel") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Delegate"); + + b.Navigation("FirstBlock"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ActivationOperation", b => + { + b.HasOne("Tzkt.Data.Models.User", "Account") + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Activations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Account"); + + b.Navigation("Block"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BallotOperation", b => + { + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Ballots") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Proposal", "Proposal") + .WithMany() + .HasForeignKey("ProposalId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Proposal"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Block", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Baker") + .WithMany() + .HasForeignKey("BakerId"); + + b.HasOne("Tzkt.Data.Models.Protocol", "Protocol") + .WithMany() + .HasForeignKey("ProtoCode") + .HasPrincipalKey("Code") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.NonceRevelationOperation", "Revelation") + .WithOne("RevealedBlock") + .HasForeignKey("Tzkt.Data.Models.Block", "RevelationId") + .HasPrincipalKey("Tzkt.Data.Models.NonceRevelationOperation", "RevealedLevel"); + + b.HasOne("Tzkt.Data.Models.Software", "Software") + .WithMany() + .HasForeignKey("SoftwareId"); + + b.Navigation("Baker"); + + b.Navigation("Protocol"); + + b.Navigation("Revelation"); + + b.Navigation("Software"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DelegationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany() + .HasForeignKey("DelegateId"); + + b.HasOne("Tzkt.Data.Models.Account", "Initiator") + .WithMany() + .HasForeignKey("InitiatorId"); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Delegations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "PrevDelegate") + .WithMany() + .HasForeignKey("PrevDelegateId"); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Delegate"); + + b.Navigation("Initiator"); + + b.Navigation("PrevDelegate"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleBakingOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Accuser") + .WithMany() + .HasForeignKey("AccuserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("DoubleBakings") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Offender") + .WithMany() + .HasForeignKey("OffenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Accuser"); + + b.Navigation("Block"); + + b.Navigation("Offender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleEndorsingOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Accuser") + .WithMany() + .HasForeignKey("AccuserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("DoubleEndorsings") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Offender") + .WithMany() + .HasForeignKey("OffenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Accuser"); + + b.Navigation("Block"); + + b.Navigation("Offender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.EndorsementOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany() + .HasForeignKey("DelegateId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Endorsements") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Delegate"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.MigrationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Account", "Account") + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Migrations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Script", "NewScript") + .WithMany() + .HasForeignKey("NewScriptId"); + + b.HasOne("Tzkt.Data.Models.Storage", "NewStorage") + .WithMany() + .HasForeignKey("NewStorageId"); + + b.HasOne("Tzkt.Data.Models.Script", "OldScript") + .WithMany() + .HasForeignKey("OldScriptId"); + + b.HasOne("Tzkt.Data.Models.Storage", "OldStorage") + .WithMany() + .HasForeignKey("OldStorageId"); + + b.Navigation("Account"); + + b.Navigation("Block"); + + b.Navigation("NewScript"); + + b.Navigation("NewStorage"); + + b.Navigation("OldScript"); + + b.Navigation("OldStorage"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.NonceRevelationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Baker") + .WithMany() + .HasForeignKey("BakerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Revelations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Baker"); + + b.Navigation("Block"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.OriginationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Contract", "Contract") + .WithMany() + .HasForeignKey("ContractId"); + + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany() + .HasForeignKey("DelegateId"); + + b.HasOne("Tzkt.Data.Models.Account", "Initiator") + .WithMany() + .HasForeignKey("InitiatorId"); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Originations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.User", "Manager") + .WithMany() + .HasForeignKey("ManagerId"); + + b.HasOne("Tzkt.Data.Models.Script", "Script") + .WithMany() + .HasForeignKey("ScriptId"); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Storage", "Storage") + .WithMany() + .HasForeignKey("StorageId"); + + b.Navigation("Block"); + + b.Navigation("Contract"); + + b.Navigation("Delegate"); + + b.Navigation("Initiator"); + + b.Navigation("Manager"); + + b.Navigation("Script"); + + b.Navigation("Sender"); + + b.Navigation("Storage"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ProposalOperation", b => + { + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Proposals") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Proposal", "Proposal") + .WithMany() + .HasForeignKey("ProposalId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Proposal"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevealOperation", b => + { + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Reveals") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevelationPenaltyOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Baker") + .WithMany() + .HasForeignKey("BakerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("RevelationPenalties") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Baker"); + + b.Navigation("Block"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.TransactionOperation", b => + { + b.HasOne("Tzkt.Data.Models.Account", "Initiator") + .WithMany() + .HasForeignKey("InitiatorId"); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Transactions") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Storage", "Storage") + .WithMany() + .HasForeignKey("StorageId"); + + b.HasOne("Tzkt.Data.Models.Account", "Target") + .WithMany() + .HasForeignKey("TargetId"); + + b.Navigation("Block"); + + b.Navigation("Initiator"); + + b.Navigation("Sender"); + + b.Navigation("Storage"); + + b.Navigation("Target"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Contract", b => + { + b.HasOne("Tzkt.Data.Models.Account", "Creator") + .WithMany() + .HasForeignKey("CreatorId"); + + b.HasOne("Tzkt.Data.Models.User", "Manager") + .WithMany() + .HasForeignKey("ManagerId"); + + b.HasOne("Tzkt.Data.Models.User", "WeirdDelegate") + .WithMany() + .HasForeignKey("WeirdDelegateId"); + + b.Navigation("Creator"); + + b.Navigation("Manager"); + + b.Navigation("WeirdDelegate"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Delegate", b => + { + b.HasOne("Tzkt.Data.Models.Software", "Software") + .WithMany() + .HasForeignKey("SoftwareId"); + + b.Navigation("Software"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Block", b => + { + b.Navigation("Activations"); + + b.Navigation("Ballots"); + + b.Navigation("CreatedAccounts"); + + b.Navigation("Delegations"); + + b.Navigation("DoubleBakings"); + + b.Navigation("DoubleEndorsings"); + + b.Navigation("Endorsements"); + + b.Navigation("Migrations"); + + b.Navigation("Originations"); + + b.Navigation("Proposals"); + + b.Navigation("Reveals"); + + b.Navigation("RevelationPenalties"); + + b.Navigation("Revelations"); + + b.Navigation("Transactions"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.NonceRevelationOperation", b => + { + b.Navigation("RevealedBlock"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Delegate", b => + { + b.Navigation("DelegatedAccounts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs b/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs new file mode 100644 index 000000000..640b12076 --- /dev/null +++ b/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs @@ -0,0 +1,386 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +namespace Tzkt.Data.Migrations +{ + public partial class Bigmaps : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "CommitDate", + table: "Software"); + + migrationBuilder.DropColumn( + name: "CommitHash", + table: "Software"); + + migrationBuilder.DropColumn( + name: "Tags", + table: "Software"); + + migrationBuilder.DropColumn( + name: "Version", + table: "Software"); + + migrationBuilder.AddColumn( + name: "InternalDelegations", + table: "TransactionOps", + type: "smallint", + nullable: true); + + migrationBuilder.AddColumn( + name: "InternalOriginations", + table: "TransactionOps", + type: "smallint", + nullable: true); + + migrationBuilder.AddColumn( + name: "InternalTransactions", + table: "TransactionOps", + type: "smallint", + nullable: true); + + migrationBuilder.AddColumn( + name: "Metadata", + table: "Software", + type: "jsonb", + nullable: true); + + migrationBuilder.AddColumn( + name: "Eth", + table: "Quotes", + type: "double precision", + nullable: false, + defaultValue: 0.0); + + migrationBuilder.AddColumn( + name: "Metadata", + table: "Protocols", + type: "jsonb", + nullable: true); + + migrationBuilder.AddColumn( + name: "Metadata", + table: "Proposals", + type: "jsonb", + nullable: true); + + migrationBuilder.AddColumn( + name: "FirstLevel", + table: "Cycles", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "LastLevel", + table: "Cycles", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "BigMapCounter", + table: "AppState", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "BigMapKeyCounter", + table: "AppState", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "BigMapUpdateCounter", + table: "AppState", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "Cycle", + table: "AppState", + type: "integer", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "QuoteEth", + table: "AppState", + type: "double precision", + nullable: false, + defaultValue: 0.0); + + migrationBuilder.AddColumn( + name: "Metadata", + table: "Accounts", + type: "jsonb", + nullable: true); + + migrationBuilder.CreateTable( + name: "BigMapKeys", + columns: table => new + { + Id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + BigMapPtr = table.Column(type: "integer", nullable: false), + FirstLevel = table.Column(type: "integer", nullable: false), + LastLevel = table.Column(type: "integer", nullable: false), + Updates = table.Column(type: "integer", nullable: false), + Active = table.Column(type: "boolean", nullable: false), + KeyHash = table.Column(type: "character varying(54)", maxLength: 54, nullable: true), + RawKey = table.Column(type: "bytea", nullable: true), + JsonKey = table.Column(type: "jsonb", nullable: true), + RawValue = table.Column(type: "bytea", nullable: true), + JsonValue = table.Column(type: "jsonb", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_BigMapKeys", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "BigMaps", + columns: table => new + { + Id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + Ptr = table.Column(type: "integer", nullable: false), + ContractId = table.Column(type: "integer", nullable: false), + StoragePath = table.Column(type: "text", nullable: true), + Active = table.Column(type: "boolean", nullable: false), + KeyType = table.Column(type: "bytea", nullable: true), + ValueType = table.Column(type: "bytea", nullable: true), + FirstLevel = table.Column(type: "integer", nullable: false), + LastLevel = table.Column(type: "integer", nullable: false), + TotalKeys = table.Column(type: "integer", nullable: false), + ActiveKeys = table.Column(type: "integer", nullable: false), + Updates = table.Column(type: "integer", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_BigMaps", x => x.Id); + table.UniqueConstraint("AK_BigMaps_Ptr", x => x.Ptr); + }); + + migrationBuilder.CreateTable( + name: "BigMapUpdates", + columns: table => new + { + Id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + BigMapPtr = table.Column(type: "integer", nullable: false), + Action = table.Column(type: "integer", nullable: false), + Level = table.Column(type: "integer", nullable: false), + OriginationId = table.Column(type: "integer", nullable: true), + TransactionId = table.Column(type: "integer", nullable: true), + BigMapKeyId = table.Column(type: "integer", nullable: true), + RawValue = table.Column(type: "bytea", nullable: true), + JsonValue = table.Column(type: "jsonb", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_BigMapUpdates", x => x.Id); + }); + + migrationBuilder.UpdateData( + table: "AppState", + keyColumn: "Id", + keyValue: -1, + column: "Cycle", + value: -1); + + migrationBuilder.CreateIndex( + name: "IX_Accounts_Metadata", + table: "Accounts", + column: "Metadata") + .Annotation("Npgsql:IndexMethod", "GIN") + .Annotation("Npgsql:IndexOperators", new[] { "jsonb_path_ops" }); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_BigMapPtr", + table: "BigMapKeys", + column: "BigMapPtr"); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_BigMapPtr_Active", + table: "BigMapKeys", + columns: new[] { "BigMapPtr", "Active" }, + filter: "\"Active\" = true"); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_BigMapPtr_KeyHash", + table: "BigMapKeys", + columns: new[] { "BigMapPtr", "KeyHash" }); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_Id", + table: "BigMapKeys", + column: "Id", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_LastLevel", + table: "BigMapKeys", + column: "LastLevel"); + + migrationBuilder.CreateIndex( + name: "IX_BigMaps_ContractId", + table: "BigMaps", + column: "ContractId"); + + migrationBuilder.CreateIndex( + name: "IX_BigMaps_Id", + table: "BigMaps", + column: "Id", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BigMaps_Ptr", + table: "BigMaps", + column: "Ptr", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BigMapUpdates_BigMapKeyId", + table: "BigMapUpdates", + column: "BigMapKeyId", + filter: "\"BigMapKeyId\" is not null"); + + migrationBuilder.CreateIndex( + name: "IX_BigMapUpdates_BigMapPtr", + table: "BigMapUpdates", + column: "BigMapPtr"); + + migrationBuilder.CreateIndex( + name: "IX_BigMapUpdates_Id", + table: "BigMapUpdates", + column: "Id", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BigMapUpdates_Level", + table: "BigMapUpdates", + column: "Level"); + + migrationBuilder.CreateIndex( + name: "IX_BigMapUpdates_OriginationId", + table: "BigMapUpdates", + column: "OriginationId", + filter: "\"OriginationId\" is not null"); + + migrationBuilder.CreateIndex( + name: "IX_BigMapUpdates_TransactionId", + table: "BigMapUpdates", + column: "TransactionId", + filter: "\"TransactionId\" is not null"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "BigMapKeys"); + + migrationBuilder.DropTable( + name: "BigMaps"); + + migrationBuilder.DropTable( + name: "BigMapUpdates"); + + migrationBuilder.DropIndex( + name: "IX_Accounts_Metadata", + table: "Accounts"); + + migrationBuilder.DropColumn( + name: "InternalDelegations", + table: "TransactionOps"); + + migrationBuilder.DropColumn( + name: "InternalOriginations", + table: "TransactionOps"); + + migrationBuilder.DropColumn( + name: "InternalTransactions", + table: "TransactionOps"); + + migrationBuilder.DropColumn( + name: "Metadata", + table: "Software"); + + migrationBuilder.DropColumn( + name: "Eth", + table: "Quotes"); + + migrationBuilder.DropColumn( + name: "Metadata", + table: "Protocols"); + + migrationBuilder.DropColumn( + name: "Metadata", + table: "Proposals"); + + migrationBuilder.DropColumn( + name: "FirstLevel", + table: "Cycles"); + + migrationBuilder.DropColumn( + name: "LastLevel", + table: "Cycles"); + + migrationBuilder.DropColumn( + name: "BigMapCounter", + table: "AppState"); + + migrationBuilder.DropColumn( + name: "BigMapKeyCounter", + table: "AppState"); + + migrationBuilder.DropColumn( + name: "BigMapUpdateCounter", + table: "AppState"); + + migrationBuilder.DropColumn( + name: "Cycle", + table: "AppState"); + + migrationBuilder.DropColumn( + name: "QuoteEth", + table: "AppState"); + + migrationBuilder.DropColumn( + name: "Metadata", + table: "Accounts"); + + migrationBuilder.AddColumn( + name: "CommitDate", + table: "Software", + type: "timestamp without time zone", + nullable: true); + + migrationBuilder.AddColumn( + name: "CommitHash", + table: "Software", + type: "character(40)", + fixedLength: true, + maxLength: 40, + nullable: true); + + migrationBuilder.AddColumn( + name: "Tags", + table: "Software", + type: "text[]", + nullable: true); + + migrationBuilder.AddColumn( + name: "Version", + table: "Software", + type: "text", + nullable: true); + } + } +} diff --git a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs index ecd3aff34..6eb62075b 100644 --- a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs +++ b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs @@ -1,6 +1,5 @@ // using System; -using System.Collections.Generic; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; @@ -16,16 +15,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder - .UseIdentityByDefaultColumns() .HasAnnotation("Relational:MaxIdentifierLength", 63) - .HasAnnotation("ProductVersion", "5.0.2"); + .HasAnnotation("ProductVersion", "5.0.4") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); modelBuilder.Entity("Tzkt.Data.Models.Account", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("Address") .IsRequired() @@ -57,6 +56,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("LastLevel") .HasColumnType("integer"); + b.Property("Metadata") + .HasColumnType("jsonb"); + b.Property("MigrationsCount") .HasColumnType("integer"); @@ -87,6 +89,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Id") .IsUnique(); + b.HasIndex("Metadata") + .HasMethod("GIN") + .HasOperators(new[] { "jsonb_path_ops" }); + b.HasIndex("Staked"); b.HasIndex("Type"); @@ -101,7 +107,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccountId") .HasColumnType("integer"); @@ -138,7 +144,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccountCounter") .HasColumnType("integer"); @@ -152,12 +158,24 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("BallotOpsCount") .HasColumnType("integer"); + b.Property("BigMapCounter") + .HasColumnType("integer"); + + b.Property("BigMapKeyCounter") + .HasColumnType("integer"); + + b.Property("BigMapUpdateCounter") + .HasColumnType("integer"); + b.Property("BlocksCount") .HasColumnType("integer"); b.Property("CommitmentsCount") .HasColumnType("integer"); + b.Property("Cycle") + .HasColumnType("integer"); + b.Property("CyclesCount") .HasColumnType("integer"); @@ -221,6 +239,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("QuoteCny") .HasColumnType("double precision"); + b.Property("QuoteEth") + .HasColumnType("double precision"); + b.Property("QuoteEur") .HasColumnType("double precision"); @@ -266,8 +287,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) AccountsCount = 0, ActivationOpsCount = 0, BallotOpsCount = 0, + BigMapCounter = 0, + BigMapKeyCounter = 0, + BigMapUpdateCounter = 0, BlocksCount = 0, CommitmentsCount = 0, + Cycle = -1, CyclesCount = 0, DelegationOpsCount = 0, DoubleBakingOpsCount = 0, @@ -289,6 +314,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) ProtocolsCount = 0, QuoteBtc = 0.0, QuoteCny = 0.0, + QuoteEth = 0.0, QuoteEur = 0.0, QuoteJpy = 0.0, QuoteKrw = 0.0, @@ -308,7 +334,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); @@ -483,7 +509,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); @@ -522,7 +548,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("Epoch") .HasColumnType("integer"); @@ -571,12 +597,174 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("BallotOps"); }); + modelBuilder.Entity("Tzkt.Data.Models.BigMap", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Active") + .HasColumnType("boolean"); + + b.Property("ActiveKeys") + .HasColumnType("integer"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("KeyType") + .HasColumnType("bytea"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Ptr") + .HasColumnType("integer"); + + b.Property("StoragePath") + .HasColumnType("text"); + + b.Property("TotalKeys") + .HasColumnType("integer"); + + b.Property("Updates") + .HasColumnType("integer"); + + b.Property("ValueType") + .HasColumnType("bytea"); + + b.HasKey("Id"); + + b.HasAlternateKey("Ptr"); + + b.HasIndex("ContractId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Ptr") + .IsUnique(); + + b.ToTable("BigMaps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMapKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Active") + .HasColumnType("boolean"); + + b.Property("BigMapPtr") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("JsonKey") + .HasColumnType("jsonb"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("KeyHash") + .HasMaxLength(54) + .HasColumnType("character varying(54)"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("RawKey") + .HasColumnType("bytea"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("Updates") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BigMapPtr"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("LastLevel"); + + b.HasIndex("BigMapPtr", "Active") + .HasFilter("\"Active\" = true"); + + b.HasIndex("BigMapPtr", "KeyHash"); + + b.ToTable("BigMapKeys"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMapUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("BigMapKeyId") + .HasColumnType("integer"); + + b.Property("BigMapPtr") + .HasColumnType("integer"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("TransactionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BigMapKeyId") + .HasFilter("\"BigMapKeyId\" is not null"); + + b.HasIndex("BigMapPtr"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("OriginationId") + .HasFilter("\"OriginationId\" is not null"); + + b.HasIndex("TransactionId") + .HasFilter("\"TransactionId\" is not null"); + + b.ToTable("BigMapUpdates"); + }); + modelBuilder.Entity("Tzkt.Data.Models.Block", b => { b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); @@ -651,7 +839,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccountId") .HasColumnType("integer"); @@ -684,11 +872,17 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("FirstLevel") + .HasColumnType("integer"); b.Property("Index") .HasColumnType("integer"); + b.Property("LastLevel") + .HasColumnType("integer"); + b.Property("Seed") .IsRequired() .HasMaxLength(64) @@ -731,7 +925,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AllocationFee") .HasColumnType("bigint"); @@ -818,7 +1012,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); @@ -851,7 +1045,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccusedLevel") .HasColumnType("integer"); @@ -904,7 +1098,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccusedLevel") .HasColumnType("integer"); @@ -957,7 +1151,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("DelegateId") .HasColumnType("integer"); @@ -1002,7 +1196,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccountId") .HasColumnType("integer"); @@ -1053,7 +1247,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); @@ -1094,7 +1288,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AllocationFee") .HasColumnType("bigint"); @@ -1193,7 +1387,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("Epoch") .HasColumnType("integer"); @@ -1212,6 +1406,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("LastPeriod") .HasColumnType("integer"); + b.Property("Metadata") + .HasColumnType("jsonb"); + b.Property("Rolls") .HasColumnType("integer"); @@ -1235,7 +1432,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("Duplicated") .HasColumnType("boolean"); @@ -1289,7 +1486,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BallotQuorumMax") .HasColumnType("integer"); @@ -1357,6 +1554,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("LastLevel") .HasColumnType("integer"); + b.Property("Metadata") + .HasColumnType("jsonb"); + b.Property("NoRewardCycles") .HasColumnType("integer"); @@ -1391,7 +1591,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("Btc") .HasColumnType("double precision"); @@ -1399,6 +1599,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Cny") .HasColumnType("double precision"); + b.Property("Eth") + .HasColumnType("double precision"); + b.Property("Eur") .HasColumnType("double precision"); @@ -1430,7 +1633,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AllocationFee") .HasColumnType("bigint"); @@ -1493,7 +1696,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); @@ -1527,7 +1730,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("CodeSchema") .HasColumnType("bytea"); @@ -1566,7 +1769,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AccountId") .HasColumnType("integer"); @@ -1592,37 +1795,26 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BlocksCount") .HasColumnType("integer"); - b.Property("CommitDate") - .HasColumnType("timestamp without time zone"); - - b.Property("CommitHash") - .HasMaxLength(40) - .HasColumnType("character(40)") - .IsFixedLength(true); - b.Property("FirstLevel") .HasColumnType("integer"); b.Property("LastLevel") .HasColumnType("integer"); + b.Property("Metadata") + .HasColumnType("jsonb"); + b.Property("ShortHash") .IsRequired() .HasMaxLength(8) .HasColumnType("character(8)") .IsFixedLength(true); - b.Property>("Tags") - .HasColumnType("text[]"); - - b.Property("Version") - .HasColumnType("text"); - b.HasKey("Id"); b.ToTable("Software"); @@ -1633,7 +1825,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("Cycle") .HasColumnType("integer"); @@ -1686,7 +1878,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("ContractId") .HasColumnType("integer"); @@ -1732,7 +1924,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("AllocationFee") .HasColumnType("bigint"); @@ -1761,7 +1953,16 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("InitiatorId") .HasColumnType("integer"); - b.Property("InternalOperations") + b.Property("InternalDelegations") + .HasColumnType("smallint"); + + b.Property("InternalOperations") + .HasColumnType("smallint"); + + b.Property("InternalOriginations") + .HasColumnType("smallint"); + + b.Property("InternalTransactions") .HasColumnType("smallint"); b.Property("JsonParameters") @@ -1831,7 +2032,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BallotsQuorum") .HasColumnType("integer"); @@ -1916,7 +2117,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Id") .ValueGeneratedOnAdd() .HasColumnType("integer") - .UseIdentityByDefaultColumn(); + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); b.Property("BakerId") .HasColumnType("integer"); From 340cb6b73c2ff0b4c6e6d577d5d0b3c666f5da83 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 1 Apr 2021 02:08:07 +0300 Subject: [PATCH 09/35] Add /bigmaps API endpoint --- Tzkt.Api/Controllers/BigMapsController.cs | 121 ++++++++ Tzkt.Api/Models/BigMaps/BigMap.cs | 60 ++++ Tzkt.Api/Repositories/BigMapsRepository.cs | 343 +++++++++++++++++++++ Tzkt.Api/Startup.cs | 1 + 4 files changed, 525 insertions(+) create mode 100644 Tzkt.Api/Controllers/BigMapsController.cs create mode 100644 Tzkt.Api/Models/BigMaps/BigMap.cs create mode 100644 Tzkt.Api/Repositories/BigMapsRepository.cs diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs new file mode 100644 index 000000000..391e7af1f --- /dev/null +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Netezos.Encoding; +using Tzkt.Api.Models; +using Tzkt.Api.Repositories; + +namespace Tzkt.Api.Controllers +{ + [ApiController] + [Route("v1/bigmaps")] + public class BigMapsController : ControllerBase + { + private readonly BigMapsRepository BigMaps; + + public BigMapsController(BigMapsRepository bigMaps) + { + BigMaps = bigMaps; + } + + /// + /// Get bigmaps count + /// + /// + /// Returns the total number of bigmaps. + /// + /// + [HttpGet("count")] + public Task GetCount() + { + return BigMaps.GetCount(); + } + + /// + /// Get bigmaps + /// + /// + /// Returns a list of bigmaps. + /// + /// Filters bigmaps by smart contract address. + /// Filters bigmaps by status: `true` - active, `false` - removed. + /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. + /// Sorts bigmaps by specified field. Supported fields: `id` (default), `ptr`, `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the bigmap key and value type: `0` - JSON, `2` - raw micheline + /// + [HttpGet] + public async Task>> Get( + AccountParameter contract, + BoolParameter active, + SelectParameter select, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + #region validate + if (sort != null && !sort.Validate("id", "ptr", "firstLevel", "lastLevel", "totalKeys", "activeKeys", "updates")) + return new BadRequest($"{nameof(sort)}", "Sorting by the specified field is not allowed."); + #endregion + + if (select == null) + return Ok(await BigMaps.Get(contract, active, sort, offset, limit, micheline)); + + if (select.Values != null) + { + if (select.Values.Length == 1) + return Ok(await BigMaps.Get(contract, active, sort, offset, limit, select.Values[0], micheline)); + else + return Ok(await BigMaps.Get(contract, active, sort, offset, limit, select.Values, micheline)); + } + else + { + if (select.Fields.Length == 1) + return Ok(await BigMaps.Get(contract, active, sort, offset, limit, select.Fields[0], micheline)); + else + { + return Ok(new SelectionResponse + { + Cols = select.Fields, + Rows = await BigMaps.Get(contract, active, sort, offset, limit, select.Fields, micheline) + }); + } + } + } + + /// + /// Get bigmap by ptr + /// + /// + /// Returns a bigmap with the specified ptr. + /// + /// Bigmap pointer + /// Format of the bigmap key and value type: `0` - JSON, `2` - raw micheline + /// + [HttpGet("{ptr:int}")] + public Task GetByPtr( + [Min(0)] int ptr, + MichelineFormat micheline = MichelineFormat.Json) + { + return BigMaps.Get(ptr, micheline); + } + + /// + /// Get bigmap type + /// + /// + /// Returns a type of the bigmap with the specified ptr in Micheline format (with annotations). + /// + /// Bigmap pointer + /// + [HttpGet("{ptr:int}/type")] + public Task GetTypeByPtr([Min(0)] int ptr) + { + return BigMaps.GetMicheType(ptr); + } + } +} diff --git a/Tzkt.Api/Models/BigMaps/BigMap.cs b/Tzkt.Api/Models/BigMaps/BigMap.cs new file mode 100644 index 000000000..ba3304ace --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/BigMap.cs @@ -0,0 +1,60 @@ +namespace Tzkt.Api.Models +{ + public class BigMap + { + /// + /// Bigmap pointer + /// + public int Ptr { get; set; } + + /// + /// Smart contract in which's storage the bigmap is allocated + /// + public Alias Contract { get; set; } + + /// + /// Path in the JSON storage to the bigmap + /// + public string Path { get; set; } + + /// + /// Bigmap status: `true` - active, `false` - removed + /// + public bool Active { get; set; } + + /// + /// Level of the block where the bigmap was seen first time + /// + public int FirstLevel { get; set; } + + /// + /// Level of the block where the bigmap was seen last time + /// + public int LastLevel { get; set; } + + /// + /// Total number of keys ever added to the bigmap + /// + public int TotalKeys { get; set; } + + /// + /// Total number of active (current) keys + /// + public int ActiveKeys { get; set; } + + /// + /// Total number of actions with the bigmap + /// + public int Updates { get; set; } + + /// + /// Bigmap key type as JSON schema or Micheline, depending on the `micheline` query parameter. + /// + public object KeyType { get; set; } + + /// + /// Bigmap value type as JSON schema or Micheline, depending on the `micheline` query parameter. + /// + public object ValueType { get; set; } + } +} diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs new file mode 100644 index 000000000..5bb6ae226 --- /dev/null +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -0,0 +1,343 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Dapper; +using Netezos.Encoding; +using Netezos.Contracts; +using Tzkt.Api.Models; +using Tzkt.Api.Services.Cache; + +namespace Tzkt.Api.Repositories +{ + public class BigMapsRepository : DbConnection + { + readonly AccountsCache Accounts; + + public BigMapsRepository(AccountsCache accounts, IConfiguration config) : base(config) + { + Accounts = accounts; + } + + public async Task GetCount() + { + using var db = GetConnection(); + return await db.QueryFirstAsync(@"SELECT COUNT(*) FROM ""BigMaps"""); + } + + public async Task GetMicheType(int ptr) + { + var sql = @" + SELECT ""KeyType"", ""ValueType"" + FROM ""BigMaps"" + WHERE ""Ptr"" = @ptr + LIMIT 1"; + + using var db = GetConnection(); + var row = await db.QueryFirstOrDefaultAsync(sql, new { ptr }); + if (row == null) return null; + + return new MichelinePrim + { + Prim = PrimType.big_map, + Args = new List + { + Micheline.FromBytes(row.KeyType), + Micheline.FromBytes(row.ValueType), + } + }; + } + + public async Task Get(int ptr, MichelineFormat micheline) + { + var sql = @" + SELECT * + FROM ""BigMaps"" + WHERE ""Ptr"" = @ptr + LIMIT 1"; + + using var db = GetConnection(); + var row = await db.QueryFirstOrDefaultAsync(sql, new { ptr }); + if (row == null) return null; + + return new BigMap + { + Ptr = row.Ptr, + Contract = Accounts.GetAlias(row.ContractId), + Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), + Active = row.Active, + FirstLevel = row.FirstLevel, + LastLevel = row.LastLevel, + TotalKeys = row.TotalKeys, + ActiveKeys = row.ActiveKeys, + Updates = row.Updates, + KeyType = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.KeyType), + ValueType = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.ValueType) + }; + } + + public async Task> Get( + AccountParameter contract, + BoolParameter active, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + var sql = new SqlBuilder(@"SELECT * FROM ""BigMaps""") + .Filter("ContractId", contract) + .Filter("Active", active) + .Take(sort, offset, limit, x => x switch + { + "ptr" => ("Ptr", "Ptr"), + "firstLevel" => ("Id", "FirstLevel"), + "lastLevel" => ("LastLevel", "LastLevel"), + "totalKeys" => ("TotalKeys", "TotalKeys"), + "activeKeys" => ("ActiveKeys", "ActiveKeys"), + "updates" => ("Updates", "Updates"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + return rows.Select(row => new BigMap + { + Ptr = row.Ptr, + Contract = Accounts.GetAlias(row.ContractId), + Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), + Active = row.Active, + FirstLevel = row.FirstLevel, + LastLevel = row.LastLevel, + TotalKeys = row.TotalKeys, + ActiveKeys = row.ActiveKeys, + Updates = row.Updates, + KeyType = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.KeyType), + ValueType = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.ValueType) + }); + } + + public async Task Get( + AccountParameter contract, + BoolParameter active, + SortParameter sort, + OffsetParameter offset, + int limit, + string[] fields, + MichelineFormat micheline) + { + var columns = new HashSet(fields.Length); + foreach (var field in fields) + { + switch (field) + { + case "ptr": columns.Add(@"""Ptr"""); break; + case "contract": columns.Add(@"""ContractId"""); break; + case "path": columns.Add(@"""StoragePath"""); break; + case "active": columns.Add(@"""Active"""); break; + case "firstLevel": columns.Add(@"""FirstLevel"""); break; + case "lastLevel": columns.Add(@"""LastLevel"""); break; + case "totalKeys": columns.Add(@"""TotalKeys"""); break; + case "activeKeys": columns.Add(@"""ActiveKeys"""); break; + case "updates": columns.Add(@"""Updates"""); break; + case "keyType": columns.Add(@"""KeyType"""); break; + case "valueType": columns.Add(@"""ValueType"""); break; + } + } + + if (columns.Count == 0) + return Array.Empty(); + + var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMaps""") + .Filter("ContractId", contract) + .Filter("Active", active) + .Take(sort, offset, limit, x => x switch + { + "ptr" => ("Ptr", "Ptr"), + "firstLevel" => ("Id", "FirstLevel"), + "lastLevel" => ("LastLevel", "LastLevel"), + "totalKeys" => ("TotalKeys", "TotalKeys"), + "activeKeys" => ("ActiveKeys", "ActiveKeys"), + "updates" => ("Updates", "Updates"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + var result = new object[rows.Count()][]; + for (int i = 0; i < result.Length; i++) + result[i] = new object[fields.Length]; + + for (int i = 0, j = 0; i < fields.Length; j = 0, i++) + { + switch (fields[i]) + { + case "ptr": + foreach (var row in rows) + result[j++][i] = row.Ptr; + break; + case "contract": + foreach (var row in rows) + result[j++][i] = Accounts.GetAlias(row.ContractId); + break; + case "path": + foreach (var row in rows) + result[j++][i] = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'); + break; + case "active": + foreach (var row in rows) + result[j++][i] = row.Active; + break; + case "firstLevel": + foreach (var row in rows) + result[j++][i] = row.FirstLevel; + break; + case "lastLevel": + foreach (var row in rows) + result[j++][i] = row.LastLevel; + break; + case "totalKeys": + foreach (var row in rows) + result[j++][i] = row.TotalKeys; + break; + case "activeKeys": + foreach (var row in rows) + result[j++][i] = row.ActiveKeys; + break; + case "updates": + foreach (var row in rows) + result[j++][i] = row.Updates; + break; + case "keyType": + foreach (var row in rows) + result[j++][i] = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.KeyType); + break; + case "valueType": + foreach (var row in rows) + result[j++][i] = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.ValueType); + break; + } + } + + return result; + } + + public async Task Get( + AccountParameter contract, + BoolParameter active, + SortParameter sort, + OffsetParameter offset, + int limit, + string field, + MichelineFormat micheline) + { + var columns = new HashSet(1); + switch (field) + { + case "ptr": columns.Add(@"""Ptr"""); break; + case "contract": columns.Add(@"""ContractId"""); break; + case "path": columns.Add(@"""StoragePath"""); break; + case "active": columns.Add(@"""Active"""); break; + case "firstLevel": columns.Add(@"""FirstLevel"""); break; + case "lastLevel": columns.Add(@"""LastLevel"""); break; + case "totalKeys": columns.Add(@"""TotalKeys"""); break; + case "activeKeys": columns.Add(@"""ActiveKeys"""); break; + case "updates": columns.Add(@"""Updates"""); break; + case "keyType": columns.Add(@"""KeyType"""); break; + case "valueType": columns.Add(@"""ValueType"""); break; + } + + if (columns.Count == 0) + return Array.Empty(); + + var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMaps""") + .Filter("ContractId", contract) + .Filter("Active", active) + .Take(sort, offset, limit, x => x switch + { + "ptr" => ("Ptr", "Ptr"), + "firstLevel" => ("Id", "FirstLevel"), + "lastLevel" => ("LastLevel", "LastLevel"), + "totalKeys" => ("TotalKeys", "TotalKeys"), + "activeKeys" => ("ActiveKeys", "ActiveKeys"), + "updates" => ("Updates", "Updates"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + //TODO: optimize memory allocation + var result = new object[rows.Count()]; + var j = 0; + + switch (field) + { + case "ptr": + foreach (var row in rows) + result[j++] = row.Ptr; + break; + case "contract": + foreach (var row in rows) + result[j++] = Accounts.GetAlias(row.ContractId); + break; + case "path": + foreach (var row in rows) + result[j++] = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'); + break; + case "active": + foreach (var row in rows) + result[j++] = row.Active; + break; + case "firstLevel": + foreach (var row in rows) + result[j++] = row.FirstLevel; + break; + case "lastLevel": + foreach (var row in rows) + result[j++] = row.LastLevel; + break; + case "totalKeys": + foreach (var row in rows) + result[j++] = row.TotalKeys; + break; + case "activeKeys": + foreach (var row in rows) + result[j++] = row.ActiveKeys; + break; + case "updates": + foreach (var row in rows) + result[j++] = row.Updates; + break; + case "keyType": + foreach (var row in rows) + result[j++] = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.KeyType); + break; + case "valueType": + foreach (var row in rows) + result[j++] = (int)micheline < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : Micheline.FromBytes(row.ValueType); + break; + } + + return result; + } + } +} diff --git a/Tzkt.Api/Startup.cs b/Tzkt.Api/Startup.cs index a21660016..53d516dd9 100644 --- a/Tzkt.Api/Startup.cs +++ b/Tzkt.Api/Startup.cs @@ -58,6 +58,7 @@ public void ConfigureServices(IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddStateListener(); From a15c39b02cc09096b455ffb5844fbd6f3ea063b7 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Thu, 1 Apr 2021 20:22:58 +0300 Subject: [PATCH 10/35] Add /keys* API endpoints --- Tzkt.Api/Controllers/BigMapsController.cs | 127 ++++++- Tzkt.Api/Models/BigMaps/BigMapKey.cs | 41 +++ Tzkt.Api/Repositories/BigMapsRepository.cs | 369 ++++++++++++++++++--- Tzkt.Api/Utils/SqlBuilder.cs | 7 + 4 files changed, 502 insertions(+), 42 deletions(-) create mode 100644 Tzkt.Api/Models/BigMaps/BigMapKey.cs diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 391e7af1f..6ffde9dc4 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Netezos.Encoding; @@ -50,7 +51,7 @@ public Task GetCount() [HttpGet] public async Task>> Get( AccountParameter contract, - BoolParameter active, + bool? active, SelectParameter select, SortParameter sort, OffsetParameter offset, @@ -117,5 +118,129 @@ public Task GetTypeByPtr([Min(0)] int ptr) { return BigMaps.GetMicheType(ptr); } + + /// + /// Get bigmap keys + /// + /// + /// Returns a list of bigmap keys. + /// + /// Bigmap pointer + /// Filters keys by status: `true` - active, `false` - removed. + /// Filters keys by JSON key. Note, this query parameter supports the following format: `?key{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?key.token_id=...`. + /// Filters keys by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. + /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. + /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// + [HttpGet("{ptr:int}/keys")] + public async Task>> GetKeys( + [Min(0)] int ptr, + bool? active, + JsonParameter key, + JsonParameter value, + SelectParameter select, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + #region validate + if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "updates")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + if (select == null) + return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, micheline)); + + if (select.Values != null) + { + if (select.Values.Length == 1) + return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Values[0], micheline)); + else + return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Values, micheline)); + } + else + { + if (select.Fields.Length == 1) + return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Fields[0], micheline)); + else + { + return Ok(new SelectionResponse + { + Cols = select.Fields, + Rows = await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Fields, micheline) + }); + } + } + } + + /// + /// Get bigmap key + /// + /// + /// Returns a bigmap key with the specified key value. + /// + /// Bigmap pointer + /// Plain key, for example, `.../keys/abcde`. + /// If the key is complex (an object or an array), you can specify it as is, for example, `.../keys/{"address":"tz123","token":123}`. + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// + [HttpGet("{ptr:int}/keys/{key}")] + public async Task> GetKey( + [Min(0)] int ptr, + string key, + MichelineFormat micheline = MichelineFormat.Json) + { + try + { + switch (key[0]) + { + case '{': + case '[': + case '"': + case 't' when key == "true": + case 'f' when key == "false": + case 'n' when key == "null": + break; + default: + key = $"\"{key}\""; + break; + } + using var doc = JsonDocument.Parse(key); + return Ok(await BigMaps.GetKey(ptr, doc.RootElement.GetRawText(), micheline)); + } + catch (JsonException) + { + return new BadRequest(nameof(key), "invalid json value"); + } + catch + { + throw; + } + } + + /// + /// Get bigmap key by hash + /// + /// + /// Returns a bigmap key with the specified key hash. + /// + /// Bigmap pointer + /// Key hash + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// + [HttpGet("{ptr:int}/keys/{hash:regex(^expr[[0-9A-z]]{{50}}$)}")] + public Task GetKeyByHash( + [Min(0)] int ptr, + string hash, + MichelineFormat micheline = MichelineFormat.Json) + { + return BigMaps.GetKeyByHash(ptr, hash, micheline); + } } } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKey.cs b/Tzkt.Api/Models/BigMaps/BigMapKey.cs new file mode 100644 index 000000000..401b71e9f --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/BigMapKey.cs @@ -0,0 +1,41 @@ +namespace Tzkt.Api.Models +{ + public class BigMapKey + { + /// + /// Bigmap key status: `true` - active, `false` - removed + /// + public bool Active { get; set; } + + /// + /// Key hash + /// + public string Hash { get; set; } + + /// + /// Key in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Key { get; set; } + + /// + /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Value { get; set; } + + /// + /// Level of the block where the bigmap key was seen first time + /// + public int FirstLevel { get; set; } + + /// + /// Level of the block where the bigmap key was seen last time + /// + public int LastLevel { get; set; } + + /// + /// Total number of actions with the bigmap key + /// + public int Updates { get; set; } + + } +} diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 5bb6ae226..3f101e0ea 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -61,29 +61,12 @@ public async Task Get(int ptr, MichelineFormat micheline) var row = await db.QueryFirstOrDefaultAsync(sql, new { ptr }); if (row == null) return null; - return new BigMap - { - Ptr = row.Ptr, - Contract = Accounts.GetAlias(row.ContractId), - Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), - Active = row.Active, - FirstLevel = row.FirstLevel, - LastLevel = row.LastLevel, - TotalKeys = row.TotalKeys, - ActiveKeys = row.ActiveKeys, - Updates = row.Updates, - KeyType = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.KeyType), - ValueType = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.ValueType) - }; + return ReadBigMap(row, micheline); } public async Task> Get( AccountParameter contract, - BoolParameter active, + bool? active, SortParameter sort, OffsetParameter offset, int limit, @@ -106,29 +89,12 @@ public async Task> Get( using var db = GetConnection(); var rows = await db.QueryAsync(sql.Query, sql.Params); - return rows.Select(row => new BigMap - { - Ptr = row.Ptr, - Contract = Accounts.GetAlias(row.ContractId), - Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), - Active = row.Active, - FirstLevel = row.FirstLevel, - LastLevel = row.LastLevel, - TotalKeys = row.TotalKeys, - ActiveKeys = row.ActiveKeys, - Updates = row.Updates, - KeyType = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.KeyType), - ValueType = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.ValueType) - }); + return rows.Select(row => (BigMap)ReadBigMap(row, micheline)); } public async Task Get( AccountParameter contract, - BoolParameter active, + bool? active, SortParameter sort, OffsetParameter offset, int limit, @@ -222,13 +188,13 @@ public async Task Get( foreach (var row in rows) result[j++][i] = (int)micheline < 2 ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.KeyType); + : new JsonString(Micheline.ToJson(row.KeyType)); break; case "valueType": foreach (var row in rows) result[j++][i] = (int)micheline < 2 ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.ValueType); + : new JsonString(Micheline.ToJson(row.ValueType)); break; } } @@ -238,7 +204,7 @@ public async Task Get( public async Task Get( AccountParameter contract, - BoolParameter active, + bool? active, SortParameter sort, OffsetParameter offset, int limit, @@ -339,5 +305,326 @@ public async Task Get( return result; } + + public async Task GetKey( + int ptr, + string key, + MichelineFormat micheline) + { + var sql = @" + SELECT * + FROM ""BigMapKeys"" + WHERE ""BigMapPtr"" = @ptr + AND ""JsonKey"" = @key::jsonb + LIMIT 1"; + + using var db = GetConnection(); + var row = await db.QueryFirstOrDefaultAsync(sql, new { ptr, key }); + if (row == null) return null; + + return ReadBigMapKey(row, micheline); + } + + public async Task GetKeyByHash( + int ptr, + string hash, + MichelineFormat micheline) + { + var sql = @" + SELECT * + FROM ""BigMapKeys"" + WHERE ""BigMapPtr"" = @ptr + AND ""KeyHash"" = @hash::character(54) + LIMIT 1"; + + using var db = GetConnection(); + var row = await db.QueryFirstOrDefaultAsync(sql, new { ptr, hash }); + if (row == null) return null; + + return ReadBigMapKey(row, micheline); + } + + public async Task> GetKeys( + int ptr, + bool? active, + JsonParameter key, + JsonParameter value, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + var sql = new SqlBuilder(@"SELECT * FROM ""BigMapKeys""") + .Filter("BigMapPtr", ptr) + .Filter("Active", active) + .Filter("JsonKey", key) + .Filter("JsonValue", value) + .Take(sort, offset, limit, x => x switch + { + "firstLevel" => ("Id", "FirstLevel"), + "lastLevel" => ("LastLevel", "LastLevel"), + "updates" => ("Updates", "Updates"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + return rows.Select(row => (BigMapKey)ReadBigMapKey(row, micheline)); + } + + public async Task GetKeys( + int ptr, + bool? active, + JsonParameter key, + JsonParameter value, + SortParameter sort, + OffsetParameter offset, + int limit, + string[] fields, + MichelineFormat micheline) + { + var columns = new HashSet(fields.Length); + foreach (var field in fields) + { + switch (field) + { + case "active": columns.Add(@"""Active"""); break; + case "firstLevel": columns.Add(@"""FirstLevel"""); break; + case "lastLevel": columns.Add(@"""LastLevel"""); break; + case "updates": columns.Add(@"""Updates"""); break; + case "hash": columns.Add(@"""KeyHash"""); break; + case "key": + columns.Add((int)micheline < 2 ? @"""JsonKey""" : @"""RawKey"""); + break; + case "value": + columns.Add((int)micheline < 2 ? @"""JsonValue""" : @"""RawValue"""); + break; + } + } + + if (columns.Count == 0) + return Array.Empty(); + + var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMapKeys""") + .Filter("BigMapPtr", ptr) + .Filter("Active", active) + .Filter("JsonKey", key) + .Filter("JsonValue", value) + .Take(sort, offset, limit, x => x switch + { + "firstLevel" => ("Id", "FirstLevel"), + "lastLevel" => ("LastLevel", "LastLevel"), + "updates" => ("Updates", "Updates"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + var result = new object[rows.Count()][]; + for (int i = 0; i < result.Length; i++) + result[i] = new object[fields.Length]; + + for (int i = 0, j = 0; i < fields.Length; j = 0, i++) + { + switch (fields[i]) + { + case "active": + foreach (var row in rows) + result[j++][i] = row.Active; + break; + case "firstLevel": + foreach (var row in rows) + result[j++][i] = row.FirstLevel; + break; + case "lastLevel": + foreach (var row in rows) + result[j++][i] = row.LastLevel; + break; + case "updates": + foreach (var row in rows) + result[j++][i] = row.Updates; + break; + case "hash": + foreach (var row in rows) + result[j++][i] = row.KeyHash; + break; + case "key": + foreach (var row in rows) + result[j++][i] = micheline switch + { + MichelineFormat.Json => new JsonString(row.JsonKey), + MichelineFormat.JsonString => row.JsonKey, + MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawKey)), + MichelineFormat.RawString => Micheline.ToJson(row.RawKey), + _ => null + }; + break; + case "value": + foreach (var row in rows) + result[j++][i] = micheline switch + { + MichelineFormat.Json => new JsonString(row.JsonValue), + MichelineFormat.JsonString => row.JsonValue, + MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.RawString => Micheline.ToJson(row.RawValue), + _ => null + }; + break; + } + } + + return result; + } + + public async Task GetKeys( + int ptr, + bool? active, + JsonParameter key, + JsonParameter value, + SortParameter sort, + OffsetParameter offset, + int limit, + string field, + MichelineFormat micheline) + { + var columns = new HashSet(1); + switch (field) + { + case "active": columns.Add(@"""Active"""); break; + case "firstLevel": columns.Add(@"""FirstLevel"""); break; + case "lastLevel": columns.Add(@"""LastLevel"""); break; + case "updates": columns.Add(@"""Updates"""); break; + case "hash": columns.Add(@"""KeyHash"""); break; + case "key": + columns.Add((int)micheline < 2 ? @"""JsonKey""" : @"""RawKey"""); + break; + case "value": + columns.Add((int)micheline < 2 ? @"""JsonValue""" : @"""RawValue"""); + break; + } + + if (columns.Count == 0) + return Array.Empty(); + + var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMapKeys""") + .Filter("BigMapPtr", ptr) + .Filter("Active", active) + .Filter("JsonKey", key) + .Filter("JsonValue", value) + .Take(sort, offset, limit, x => x switch + { + "firstLevel" => ("Id", "FirstLevel"), + "lastLevel" => ("LastLevel", "LastLevel"), + "updates" => ("Updates", "Updates"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + //TODO: optimize memory allocation + var result = new object[rows.Count()]; + var j = 0; + + switch (field) + { + case "active": + foreach (var row in rows) + result[j++] = row.Active; + break; + case "firstLevel": + foreach (var row in rows) + result[j++] = row.FirstLevel; + break; + case "lastLevel": + foreach (var row in rows) + result[j++] = row.LastLevel; + break; + case "updates": + foreach (var row in rows) + result[j++] = row.Updates; + break; + case "hash": + foreach (var row in rows) + result[j++] = row.KeyHash; + break; + case "key": + foreach (var row in rows) + result[j++] = micheline switch + { + MichelineFormat.Json => new JsonString(row.JsonKey), + MichelineFormat.JsonString => row.JsonKey, + MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawKey)), + MichelineFormat.RawString => Micheline.ToJson(row.RawKey), + _ => null + }; + break; + case "value": + foreach (var row in rows) + result[j++] = micheline switch + { + MichelineFormat.Json => new JsonString(row.JsonValue), + MichelineFormat.JsonString => row.JsonValue, + MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.RawString => Micheline.ToJson(row.RawValue), + _ => null + }; + break; + } + + return result; + } + + BigMap ReadBigMap(dynamic row, MichelineFormat format) + { + return new BigMap + { + Ptr = row.Ptr, + Contract = Accounts.GetAlias(row.ContractId), + Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), + Active = row.Active, + FirstLevel = row.FirstLevel, + LastLevel = row.LastLevel, + TotalKeys = row.TotalKeys, + ActiveKeys = row.ActiveKeys, + Updates = row.Updates, + KeyType = (int)format < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : new JsonString(Micheline.ToJson(row.KeyType)), + ValueType = (int)format < 2 + ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : new JsonString(Micheline.ToJson(row.ValueType)) + }; + } + + BigMapKey ReadBigMapKey(dynamic row, MichelineFormat format) + { + return new BigMapKey + { + Active = row.Active, + FirstLevel = row.FirstLevel, + LastLevel = row.LastLevel, + Updates = row.Updates, + Hash = row.KeyHash, + Key = format switch + { + MichelineFormat.Json => new JsonString(row.JsonKey), + MichelineFormat.JsonString => row.JsonKey, + MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawKey)), + MichelineFormat.RawString => Micheline.ToJson(row.RawKey), + _ => null + }, + Value = format switch + { + MichelineFormat.Json => new JsonString(row.JsonValue), + MichelineFormat.JsonString => row.JsonValue, + MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.RawString => Micheline.ToJson(row.RawValue), + _ => null + } + }; + } } } diff --git a/Tzkt.Api/Utils/SqlBuilder.cs b/Tzkt.Api/Utils/SqlBuilder.cs index 4d557f207..25ff0b83f 100644 --- a/Tzkt.Api/Utils/SqlBuilder.cs +++ b/Tzkt.Api/Utils/SqlBuilder.cs @@ -58,6 +58,13 @@ public SqlBuilder FilterA(string column, int value) return this; } + public SqlBuilder Filter(string column, bool? value) + { + if (value == null) return this; + AppendFilter($@"""{column}"" = {value}"); + return this; + } + public SqlBuilder Filter(string column, AccountTypeParameter type) { if (type == null) return this; From d56c8b05cd5ba9994ab89f9fcf9bcbe73ed554c6 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Fri, 2 Apr 2021 00:50:29 +0300 Subject: [PATCH 11/35] Add /keys/history API endpoints --- Tzkt.Api/Controllers/BigMapsController.cs | 104 ++++++++++++-- Tzkt.Api/Controllers/ContractsController.cs | 4 +- Tzkt.Api/Models/BigMaps/BigMapKey.cs | 7 +- Tzkt.Api/Models/BigMaps/BigMapUpdate.cs | 32 +++++ Tzkt.Api/Models/Entrypoint.cs | 2 +- Tzkt.Api/Models/Software.cs | 2 +- Tzkt.Api/Models/StorageRecord.cs | 26 +--- Tzkt.Api/Repositories/AccountRepository.cs | 52 +++---- Tzkt.Api/Repositories/BigMapsRepository.cs | 140 +++++++++++++++---- Tzkt.Api/Repositories/OperationRepository.cs | 78 +++++------ Tzkt.Api/Repositories/SoftwareRepository.cs | 6 +- Tzkt.Api/Utils/Constants/BigMapActions.cs | 15 ++ Tzkt.Api/Utils/JsonString.cs | 30 ---- Tzkt.Api/Utils/RawJson.cs | 30 ++++ 14 files changed, 366 insertions(+), 162 deletions(-) create mode 100644 Tzkt.Api/Models/BigMaps/BigMapUpdate.cs create mode 100644 Tzkt.Api/Utils/Constants/BigMapActions.cs delete mode 100644 Tzkt.Api/Utils/JsonString.cs create mode 100644 Tzkt.Api/Utils/RawJson.cs diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 6ffde9dc4..38431cb4e 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -198,20 +198,7 @@ public async Task> GetKey( { try { - switch (key[0]) - { - case '{': - case '[': - case '"': - case 't' when key == "true": - case 'f' when key == "false": - case 'n' when key == "null": - break; - default: - key = $"\"{key}\""; - break; - } - using var doc = JsonDocument.Parse(key); + using var doc = JsonDocument.Parse(WrapKey(key)); return Ok(await BigMaps.GetKey(ptr, doc.RootElement.GetRawText(), micheline)); } catch (JsonException) @@ -224,6 +211,49 @@ public async Task> GetKey( } } + /// + /// Get key history + /// + /// + /// Returns updates history for the specified bigmap key. + /// + /// Bigmap pointer + /// Plain key, for example, `.../keys/abcde/history`. + /// If the key is complex (an object or an array), you can specify it as is, for example, `.../keys/{"address":"tz123","token":123}/history`. + /// Sorts bigmaps by specified field. Supported fields: `id` (default). + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// + [HttpGet("{ptr:int}/keys/{key}/history")] + public async Task>> GetKeyHistory( + [Min(0)] int ptr, + string key, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + #region validate + if (sort != null && !sort.Validate("id")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + try + { + using var doc = JsonDocument.Parse(WrapKey(key)); + return Ok(await BigMaps.GetKeyUpdates(ptr, doc.RootElement.GetRawText(), sort, offset, limit, micheline)); + } + catch (JsonException) + { + return new BadRequest(nameof(key), "invalid json value"); + } + catch + { + throw; + } + } + /// /// Get bigmap key by hash /// @@ -242,5 +272,51 @@ public Task GetKeyByHash( { return BigMaps.GetKeyByHash(ptr, hash, micheline); } + + /// + /// Get key by hash history + /// + /// + /// Returns updates history for the bigmap key with the specified key hash. + /// + /// Bigmap pointer + /// Key hash + /// Sorts bigmaps by specified field. Supported fields: `id` (default). + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// + [HttpGet("{ptr:int}/keys/{hash:regex(^expr[[0-9A-z]]{{50}}$)}/history")] + public async Task>> GetKeyByHashHistory( + [Min(0)] int ptr, + string hash, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + #region validate + if (sort != null && !sort.Validate("id")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + return Ok(await BigMaps.GetKeyByHashUpdates(ptr, hash, sort, offset, limit, micheline)); + } + + string WrapKey(string key) + { + switch (key[0]) + { + case '{': + case '[': + case '"': + case 't' when key == "true": + case 'f' when key == "false": + case 'n' when key == "null": + return key; + default: + return $"\"{key}\""; + } + } } } diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index 0f8432498..0fb10f14f 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -316,7 +316,7 @@ public async Task GetStorageSchema([Address] string address, [Min( /// Maximum number of items to return /// [HttpGet("{address}/storage/history")] - public Task>> GetStorageHistory([Address] string address, [Min(0)] int lastId = 0, [Range(0, 1000)] int limit = 10) + public Task> GetStorageHistory([Address] string address, [Min(0)] int lastId = 0, [Range(0, 1000)] int limit = 10) { return Accounts.GetStorageHistory(address, lastId, limit); } @@ -366,7 +366,7 @@ public Task GetRawStorageSchema([Address] string address, [Min(0)] i /// Maximum number of items to return /// [HttpGet("{address}/storage/raw/history")] - public Task>> GetRawStorageHistory([Address] string address, [Min(0)] int lastId = 0, [Range(0, 1000)] int limit = 10) + public Task> GetRawStorageHistory([Address] string address, [Min(0)] int lastId = 0, [Range(0, 1000)] int limit = 10) { return Accounts.GetRawStorageHistory(address, lastId, limit); } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKey.cs b/Tzkt.Api/Models/BigMaps/BigMapKey.cs index 401b71e9f..0ab5fba81 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapKey.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKey.cs @@ -1,7 +1,12 @@ -namespace Tzkt.Api.Models +using System.Text.Json.Serialization; + +namespace Tzkt.Api.Models { public class BigMapKey { + [JsonIgnore] + public int Id { get; set; } + /// /// Bigmap key status: `true` - active, `false` - removed /// diff --git a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs new file mode 100644 index 000000000..6c5521221 --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs @@ -0,0 +1,32 @@ +using System; + +namespace Tzkt.Api.Models +{ + public class BigMapUpdate + { + /// + /// Internal ID that can be used for pagination + /// + public int Id { get; set; } + + /// + /// Level of the block where the bigmap key was updated + /// + public int Level { get; set; } + + /// + /// Timestamp of the block where the bigmap key was updated + /// + public DateTime Timestamp { get; set; } + + /// + /// Action with the key + /// + public string Action { get; set; } + + /// + /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Value { get; set; } + } +} diff --git a/Tzkt.Api/Models/Entrypoint.cs b/Tzkt.Api/Models/Entrypoint.cs index 9b2c6b63a..67c9cf017 100644 --- a/Tzkt.Api/Models/Entrypoint.cs +++ b/Tzkt.Api/Models/Entrypoint.cs @@ -15,7 +15,7 @@ public class Entrypoint /// A kind of JSON schema, describing how parameters will look like in a human-readable JSON format /// [JsonSchemaType(typeof(object))] - public JsonString JsonParameters { get; set; } + public RawJson JsonParameters { get; set; } /// /// Parameters schema in micheline format diff --git a/Tzkt.Api/Models/Software.cs b/Tzkt.Api/Models/Software.cs index 81eb855db..ef156484d 100644 --- a/Tzkt.Api/Models/Software.cs +++ b/Tzkt.Api/Models/Software.cs @@ -38,6 +38,6 @@ public class Software /// /// Offchain metadata /// - public JsonString Metadata { get; set; } + public RawJson Metadata { get; set; } } } diff --git a/Tzkt.Api/Models/StorageRecord.cs b/Tzkt.Api/Models/StorageRecord.cs index 75307e1e3..26a042f01 100644 --- a/Tzkt.Api/Models/StorageRecord.cs +++ b/Tzkt.Api/Models/StorageRecord.cs @@ -1,9 +1,8 @@ using System; -using NJsonSchema.Annotations; namespace Tzkt.Api.Models { - public class StorageRecord + public class StorageRecord { /// /// Id of the record that can be used for pagination @@ -23,16 +22,15 @@ public class StorageRecord /// /// Operation that caused the storage change /// - public SourceOperation Operation { get; set; } + public SourceOperation Operation { get; set; } /// /// New storage value /// - [JsonSchemaType(typeof(object))] - public T Value { get; set; } + public object Value { get; set; } } - public class SourceOperation + public class SourceOperation { /// /// Operation type @@ -57,20 +55,6 @@ public class SourceOperation /// /// Transaction parameter, including called entrypoint and value passed to the entrypoint. /// - public SourceOperationParameter Parameter { get; set; } - } - - public class SourceOperationParameter - { - /// - /// Called entrypoint - /// - public string Entrypoint { get; set; } - - /// - /// Value passed to the entrypoint - /// - [JsonSchemaType(typeof(object))] - public T Value { get; set; } + public TxParameter Parameter { get; set; } } } diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index 251ea230e..772035f37 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -1799,7 +1799,7 @@ LEFT JOIN ""Storages"" AS st NumReveals = row.RevealsCount, NumMigrations = row.MigrationsCount, NumTransactions = row.TransactionsCount, - Storage = row.JsonValue == null ? null : new JsonString((string)row.JsonValue) + Storage = row.JsonValue == null ? null : new RawJson((string)row.JsonValue) }; }); } @@ -1996,7 +1996,7 @@ public async Task GetContracts( break; case "storage": foreach (var row in rows) - result[j++][i] = row.JsonValue == null ? null : new JsonString((string)row.JsonValue); + result[j++][i] = row.JsonValue == null ? null : new RawJson((string)row.JsonValue); break; } } @@ -2191,7 +2191,7 @@ public async Task GetContracts( break; case "storage": foreach (var row in rows) - result[j++] = row.JsonValue == null ? null : new JsonString((string)row.JsonValue); + result[j++] = row.JsonValue == null ? null : new RawJson((string)row.JsonValue); break; } @@ -2560,10 +2560,10 @@ ORDER BY ""ScriptLevel"" DESC return (Micheline.FromBytes(row.StorageSchema) as MichelinePrim).Args[0]; } - public async Task>> GetStorageHistory(string address, int lastId, int limit) + public async Task> GetStorageHistory(string address, int lastId, int limit) { var rawAccount = await Accounts.GetAsync(address); - if (rawAccount is not RawContract contract) return Enumerable.Empty>(); + if (rawAccount is not RawContract contract) return Enumerable.Empty(); using var db = GetConnection(); var rows = await db.QueryAsync($@" @@ -2594,28 +2594,30 @@ LEFT JOIN ""OriginationOps"" as o_op {(lastId > 0 ? $@"AND COALESCE(ss.""TransactionId"", ss.""OriginationId"", ss.""MigrationId"") < {lastId}" : "")} ORDER BY ss.""Level"" DESC, ss.""TransactionId"" DESC LIMIT {limit}"); - if (!rows.Any()) return Enumerable.Empty>(); + if (!rows.Any()) return Enumerable.Empty(); return rows.Select(row => { int id; DateTime timestamp; - SourceOperation source; + SourceOperation source; if (row.TransactionId != null) { id = row.TransactionId; timestamp = row.TransactionTimestamp; - source = new SourceOperation + source = new SourceOperation { Type = "transaction", Hash = row.TransactionHash, Counter = row.TransactionCounter, Nonce = row.TransactionNonce, - Parameter = row.TransactionEntrypoint == null ? null : new SourceOperationParameter + Parameter = row.TransactionEntrypoint == null ? null : new TxParameter { Entrypoint = row.TransactionEntrypoint, - Value = row.TransactionJsonParameters + Value = row.TransactionJsonParameters != null + ? new RawJson(row.TransactionJsonParameters) + : null } }; } @@ -2623,7 +2625,7 @@ LEFT JOIN ""OriginationOps"" as o_op { id = row.OriginationId; timestamp = row.OriginationTimestamp; - source = new SourceOperation + source = new SourceOperation { Type = "origination", Hash = row.OriginationHash, @@ -2635,27 +2637,27 @@ LEFT JOIN ""OriginationOps"" as o_op { id = row.MigrationId; timestamp = row.MigrationTimestamp; - source = new SourceOperation + source = new SourceOperation { Type = "migration" }; } - return new StorageRecord + return new StorageRecord { Id = id, Timestamp = timestamp, Operation = source, Level = row.Level, - Value = row.JsonValue, + Value = new RawJson(row.JsonValue), }; }); } - public async Task>> GetRawStorageHistory(string address, int lastId, int limit) + public async Task> GetRawStorageHistory(string address, int lastId, int limit) { var rawAccount = await Accounts.GetAsync(address); - if (rawAccount is not RawContract contract) return Enumerable.Empty>(); + if (rawAccount is not RawContract contract) return Enumerable.Empty(); using var db = GetConnection(); var rows = await db.QueryAsync($@" @@ -2686,29 +2688,29 @@ LEFT JOIN ""OriginationOps"" as o_op {(lastId > 0 ? $@"AND COALESCE(ss.""TransactionId"", ss.""OriginationId"", ss.""MigrationId"") < {lastId}" : "")} ORDER BY ss.""Level"" DESC, ss.""TransactionId"" DESC LIMIT {limit}"); - if (!rows.Any()) return Enumerable.Empty>(); + if (!rows.Any()) return Enumerable.Empty(); return rows.Select(row => { int id; DateTime timestamp; - SourceOperation source; + SourceOperation source; if (row.TransactionId != null) { id = row.TransactionId; timestamp = row.TransactionTimestamp; - source = new SourceOperation + source = new SourceOperation { Type = "transaction", Hash = row.TransactionHash, Counter = row.TransactionCounter, Nonce = row.TransactionNonce, - Parameter = row.TransactionEntrypoint == null ? null : new SourceOperationParameter + Parameter = row.TransactionEntrypoint == null ? null : new TxParameter { Entrypoint = row.TransactionEntrypoint, Value = row.TransactionRawParameters != null - ? Micheline.FromBytes(row.TransactionRawParameters) + ? new RawJson(Micheline.ToJson(row.TransactionRawParameters)) : null } }; @@ -2717,7 +2719,7 @@ LEFT JOIN ""OriginationOps"" as o_op { id = row.OriginationId; timestamp = row.OriginationTimestamp; - source = new SourceOperation + source = new SourceOperation { Type = "origination", Hash = row.OriginationHash, @@ -2729,19 +2731,19 @@ LEFT JOIN ""OriginationOps"" as o_op { id = row.MigrationId; timestamp = row.MigrationTimestamp; - source = new SourceOperation + source = new SourceOperation { Type = "migration" }; } - return new StorageRecord + return new StorageRecord { Id = id, Timestamp = timestamp, Operation = source, Level = row.Level, - Value = Micheline.FromBytes(row.RawValue), + Value = new RawJson(Micheline.ToJson(row.RawValue)), }; }); } diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 3f101e0ea..4405a7b0d 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -14,12 +14,15 @@ namespace Tzkt.Api.Repositories public class BigMapsRepository : DbConnection { readonly AccountsCache Accounts; + readonly TimeCache Times; - public BigMapsRepository(AccountsCache accounts, IConfiguration config) : base(config) + public BigMapsRepository(AccountsCache accounts, TimeCache times, IConfiguration config) : base(config) { Accounts = accounts; + Times = times; } + #region bigmaps public async Task GetCount() { using var db = GetConnection(); @@ -187,14 +190,14 @@ public async Task Get( case "keyType": foreach (var row in rows) result[j++][i] = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) - : new JsonString(Micheline.ToJson(row.KeyType)); + ? new RawJson(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : new RawJson(Micheline.ToJson(row.KeyType)); break; case "valueType": foreach (var row in rows) result[j++][i] = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : new JsonString(Micheline.ToJson(row.ValueType)); + ? new RawJson(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : new RawJson(Micheline.ToJson(row.ValueType)); break; } } @@ -292,20 +295,22 @@ public async Task Get( case "keyType": foreach (var row in rows) result[j++] = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.KeyType); + ? new RawJson(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : new RawJson(Micheline.ToJson(row.KeyType)); break; case "valueType": foreach (var row in rows) result[j++] = (int)micheline < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : Micheline.FromBytes(row.ValueType); + ? new RawJson(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : new RawJson(Micheline.ToJson(row.ValueType)); break; } return result; } + #endregion + #region bigmap keys public async Task GetKey( int ptr, string key, @@ -454,9 +459,9 @@ public async Task GetKeys( foreach (var row in rows) result[j++][i] = micheline switch { - MichelineFormat.Json => new JsonString(row.JsonKey), + MichelineFormat.Json => new RawJson(row.JsonKey), MichelineFormat.JsonString => row.JsonKey, - MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawKey)), + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), MichelineFormat.RawString => Micheline.ToJson(row.RawKey), _ => null }; @@ -465,9 +470,9 @@ public async Task GetKeys( foreach (var row in rows) result[j++][i] = micheline switch { - MichelineFormat.Json => new JsonString(row.JsonValue), + MichelineFormat.Json => new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => Micheline.ToJson(row.RawValue), _ => null }; @@ -554,9 +559,9 @@ public async Task GetKeys( foreach (var row in rows) result[j++] = micheline switch { - MichelineFormat.Json => new JsonString(row.JsonKey), + MichelineFormat.Json => new RawJson(row.JsonKey), MichelineFormat.JsonString => row.JsonKey, - MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawKey)), + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), MichelineFormat.RawString => Micheline.ToJson(row.RawKey), _ => null }; @@ -565,9 +570,9 @@ public async Task GetKeys( foreach (var row in rows) result[j++] = micheline switch { - MichelineFormat.Json => new JsonString(row.JsonValue), + MichelineFormat.Json => new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => Micheline.ToJson(row.RawValue), _ => null }; @@ -576,6 +581,63 @@ public async Task GetKeys( return result; } + #endregion + + #region bigmap key updates + public async Task> GetKeyUpdates( + int ptr, + string key, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + using var db = GetConnection(); + var keyRow = await db.QueryFirstOrDefaultAsync(@" + SELECT ""Id"" + FROM ""BigMapKeys"" + WHERE ""BigMapPtr"" = @ptr + AND ""JsonKey"" = @key::jsonb + LIMIT 1", + new { ptr, key }); + + if (keyRow == null) return null; + + var sql = new SqlBuilder(@"SELECT * FROM ""BigMapUpdates""") + .Filter("BigMapKeyId", (int)keyRow.Id) + .Take(sort, offset, limit, x => ("Id", "Id")); + + var rows = await db.QueryAsync(sql.Query, sql.Params); + return rows.Select(row => (BigMapUpdate)ReadBigMapUpdate(row, micheline)); + } + + public async Task> GetKeyByHashUpdates( + int ptr, + string hash, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + using var db = GetConnection(); + var keyRow = await db.QueryFirstOrDefaultAsync(@" + SELECT ""Id"" + FROM ""BigMapKeys"" + WHERE ""BigMapPtr"" = @ptr + AND ""KeyHash"" = @hash::character(54) + LIMIT 1", + new { ptr, hash }); + + if (keyRow == null) return null; + + var sql = new SqlBuilder(@"SELECT * FROM ""BigMapUpdates""") + .Filter("BigMapKeyId", (int)keyRow.Id) + .Take(sort, offset, limit, x => ("Id", "Id")); + + var rows = await db.QueryAsync(sql.Query, sql.Params); + return rows.Select(row => (BigMapUpdate)ReadBigMapUpdate(row, micheline)); + } + #endregion BigMap ReadBigMap(dynamic row, MichelineFormat format) { @@ -591,11 +653,11 @@ BigMap ReadBigMap(dynamic row, MichelineFormat format) ActiveKeys = row.ActiveKeys, Updates = row.Updates, KeyType = (int)format < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) - : new JsonString(Micheline.ToJson(row.KeyType)), + ? new RawJson(Schema.Create(Micheline.FromBytes(row.KeyType) as MichelinePrim).Humanize()) + : new RawJson(Micheline.ToJson(row.KeyType)), ValueType = (int)format < 2 - ? new JsonString(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : new JsonString(Micheline.ToJson(row.ValueType)) + ? new RawJson(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) + : new RawJson(Micheline.ToJson(row.ValueType)) }; } @@ -603,6 +665,7 @@ BigMapKey ReadBigMapKey(dynamic row, MichelineFormat format) { return new BigMapKey { + Id = row.Id, Active = row.Active, FirstLevel = row.FirstLevel, LastLevel = row.LastLevel, @@ -610,21 +673,48 @@ BigMapKey ReadBigMapKey(dynamic row, MichelineFormat format) Hash = row.KeyHash, Key = format switch { - MichelineFormat.Json => new JsonString(row.JsonKey), + MichelineFormat.Json => new RawJson(row.JsonKey), MichelineFormat.JsonString => row.JsonKey, - MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawKey)), + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), MichelineFormat.RawString => Micheline.ToJson(row.RawKey), _ => null }, Value = format switch { - MichelineFormat.Json => new JsonString(row.JsonValue), + MichelineFormat.Json => new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => Micheline.ToJson(row.RawValue), _ => null } }; } + + BigMapUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) + { + return new BigMapUpdate + { + Id = row.Id, + Level = row.Level, + Timestamp = Times[row.Level], + Action = UpdateAction((int)row.Action), + Value = format switch + { + MichelineFormat.Json => new RawJson(row.JsonValue), + MichelineFormat.JsonString => row.JsonValue, + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), + MichelineFormat.RawString => Micheline.ToJson(row.RawValue), + _ => null + } + }; + } + + string UpdateAction(int action) => action switch + { + 1 => BigMapActions.AddKey, + 2 => BigMapActions.UpdateKey, + 3 => BigMapActions.RemoveKey, + _ => "unknown" + }; } } diff --git a/Tzkt.Api/Repositories/OperationRepository.cs b/Tzkt.Api/Repositories/OperationRepository.cs index 87bafbc80..a785a410b 100644 --- a/Tzkt.Api/Repositories/OperationRepository.cs +++ b/Tzkt.Api/Repositories/OperationRepository.cs @@ -3242,9 +3242,9 @@ LEFT JOIN ""Scripts"" as sc Code = (int)format % 2 == 0 ? code : code.ToJson(), Storage = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, @@ -3321,9 +3321,9 @@ LEFT JOIN ""Scripts"" as sc Code = (int)format % 2 == 0 ? code : code.ToJson(), Storage = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, @@ -3400,9 +3400,9 @@ LEFT JOIN ""Scripts"" as sc Code = (int)format % 2 == 0 ? code : code.ToJson(), Storage = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, @@ -3759,9 +3759,9 @@ public async Task GetOriginations( foreach (var row in rows) result[j++][i] = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }; @@ -3979,11 +3979,11 @@ public async Task GetOriginations( foreach (var row in rows) { var code = row.ParameterSchema == null ? null : new MichelineArray - { - Micheline.FromBytes(row.ParameterSchema), - Micheline.FromBytes(row.StorageSchema), - Micheline.FromBytes(row.CodeSchema) - }; + { + Micheline.FromBytes(row.ParameterSchema), + Micheline.FromBytes(row.StorageSchema), + Micheline.FromBytes(row.CodeSchema) + }; result[j++] = (int)format % 2 == 0 ? code : code.ToJson(); } break; @@ -3991,9 +3991,9 @@ public async Task GetOriginations( foreach (var row in rows) result[j++] = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }; @@ -4089,18 +4089,18 @@ LEFT JOIN ""Storages"" as s Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } }, Storage = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, @@ -4152,18 +4152,18 @@ LEFT JOIN ""Storages"" as s Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } }, Storage = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, @@ -4215,18 +4215,18 @@ LEFT JOIN ""Storages"" as s Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } }, Storage = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, @@ -4274,9 +4274,9 @@ public async Task> GetTransactions(Block block Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } @@ -4392,9 +4392,9 @@ INNER JOIN ""Blocks"" as b Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } @@ -4683,9 +4683,9 @@ public async Task GetTransactions( Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } @@ -4695,9 +4695,9 @@ public async Task GetTransactions( foreach (var row in rows) result[j++][i] = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }; @@ -4925,9 +4925,9 @@ public async Task GetTransactions( Entrypoint = row.Entrypoint, Value = format switch { - MichelineFormat.Json => row.JsonParameters == null ? null : new JsonString(row.JsonParameters), + MichelineFormat.Json => row.JsonParameters == null ? null : new RawJson(row.JsonParameters), MichelineFormat.JsonString => row.JsonParameters, - MichelineFormat.Raw => row.RawParameters == null ? null : new JsonString(Micheline.ToJson(row.RawParameters)), + MichelineFormat.Raw => row.RawParameters == null ? null : new RawJson(Micheline.ToJson(row.RawParameters)), MichelineFormat.RawString => row.RawParameters == null ? null : Micheline.ToJson(row.RawParameters), _ => throw new Exception("Invalid MichelineFormat value") } @@ -4937,9 +4937,9 @@ public async Task GetTransactions( foreach (var row in rows) result[j++] = format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }; diff --git a/Tzkt.Api/Repositories/SoftwareRepository.cs b/Tzkt.Api/Repositories/SoftwareRepository.cs index bae64993f..473b9ae8d 100644 --- a/Tzkt.Api/Repositories/SoftwareRepository.cs +++ b/Tzkt.Api/Repositories/SoftwareRepository.cs @@ -47,7 +47,7 @@ public async Task> Get(SortParameter sort, OffsetParameter LastLevel = row.LastLevel, LastTime = Time[row.LastLevel], ShortHash = row.ShortHash, - Metadata = new JsonString(row.Metadata) + Metadata = new RawJson(row.Metadata) }); } @@ -117,7 +117,7 @@ public async Task Get(SortParameter sort, OffsetParameter offset, in break; case "metadata": foreach (var row in rows) - result[j++][i] = new JsonString(row.Metadata); + result[j++][i] = new RawJson(row.Metadata); break; } } @@ -185,7 +185,7 @@ public async Task Get(SortParameter sort, OffsetParameter offset, int break; case "metadata": foreach (var row in rows) - result[j++] = new JsonString(row.Metadata); + result[j++] = new RawJson(row.Metadata); break; } diff --git a/Tzkt.Api/Utils/Constants/BigMapActions.cs b/Tzkt.Api/Utils/Constants/BigMapActions.cs new file mode 100644 index 000000000..ac62a136b --- /dev/null +++ b/Tzkt.Api/Utils/Constants/BigMapActions.cs @@ -0,0 +1,15 @@ +namespace Tzkt.Api +{ + static class BigMapActions + { + public const string Allocate = "allocate"; + + public const string AddKey = "add_key"; + + public const string UpdateKey = "update_key"; + + public const string RemoveKey = "remove_key"; + + public const string Remove = "remove"; + } +} diff --git a/Tzkt.Api/Utils/JsonString.cs b/Tzkt.Api/Utils/JsonString.cs deleted file mode 100644 index ee7dfcac1..000000000 --- a/Tzkt.Api/Utils/JsonString.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Tzkt.Api -{ - [JsonConverter(typeof(JsonStringConverter))] - public class JsonString - { - public string Json { get; } - public JsonString(string json) => Json = json; - - public static implicit operator JsonString (string value) => new JsonString(value); - public static explicit operator string (JsonString value) => value.Json; - } - - class JsonStringConverter : JsonConverter - { - public override JsonString Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override void Write(Utf8JsonWriter writer, JsonString value, JsonSerializerOptions options) - { - using var doc = JsonDocument.Parse(value.Json, new JsonDocumentOptions { MaxDepth = 1024 }); - doc.WriteTo(writer); - } - } -} diff --git a/Tzkt.Api/Utils/RawJson.cs b/Tzkt.Api/Utils/RawJson.cs new file mode 100644 index 000000000..6bdf1499d --- /dev/null +++ b/Tzkt.Api/Utils/RawJson.cs @@ -0,0 +1,30 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Tzkt.Api +{ + [JsonConverter(typeof(JsonStringConverter))] + public class RawJson + { + public string Json { get; } + public RawJson(string json) => Json = json; + + public static implicit operator RawJson (string value) => new RawJson(value); + public static explicit operator string (RawJson value) => value.Json; + } + + class JsonStringConverter : JsonConverter + { + public override RawJson Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + throw new NotImplementedException(); + } + + public override void Write(Utf8JsonWriter writer, RawJson value, JsonSerializerOptions options) + { + using var doc = JsonDocument.Parse(value.Json, new JsonDocumentOptions { MaxDepth = 1024 }); + doc.WriteTo(writer); + } + } +} From f6e9da99490dd1c1287196d4592ae785c65eaa4f Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Fri, 2 Apr 2021 03:09:07 +0300 Subject: [PATCH 12/35] Add /historical_keys* API endpoints and minor refactoring --- Tzkt.Api/Controllers/BigMapsController.cs | 205 ++++++++---- Tzkt.Api/Models/BigMaps/BigMap.cs | 4 +- Tzkt.Api/Models/BigMaps/BigMapKey.cs | 9 +- Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs | 30 ++ Tzkt.Api/Models/BigMaps/BigMapUpdate.cs | 2 +- Tzkt.Api/Repositories/BigMapsRepository.cs | 372 +++++++++++++++++---- Tzkt.Api/Tzkt.Api.csproj | 4 +- Tzkt.Data/Tzkt.Data.csproj | 2 +- Tzkt.Sync/Tzkt.Sync.csproj | 2 +- 9 files changed, 480 insertions(+), 150 deletions(-) create mode 100644 Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 38431cb4e..3029271fa 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Netezos.Encoding; @@ -29,7 +30,7 @@ public BigMapsController(BigMapsRepository bigMaps) /// /// [HttpGet("count")] - public Task GetCount() + public Task GetBigMapsCount() { return BigMaps.GetCount(); } @@ -43,13 +44,13 @@ public Task GetCount() /// Filters bigmaps by smart contract address. /// Filters bigmaps by status: `true` - active, `false` - removed. /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. - /// Sorts bigmaps by specified field. Supported fields: `id` (default), `ptr`, `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. + /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. /// Specifies which or how many items should be skipped /// Maximum number of items to return - /// Format of the bigmap key and value type: `0` - JSON, `2` - raw micheline + /// Format of the bigmap key and value type: `0` - JSON, `2` - Micheline /// [HttpGet] - public async Task>> Get( + public async Task>> GetBigMaps( AccountParameter contract, bool? active, SelectParameter select, @@ -59,7 +60,7 @@ public async Task>> Get( MichelineFormat micheline = MichelineFormat.Json) { #region validate - if (sort != null && !sort.Validate("id", "ptr", "firstLevel", "lastLevel", "totalKeys", "activeKeys", "updates")) + if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "totalKeys", "activeKeys", "updates")) return new BadRequest($"{nameof(sort)}", "Sorting by the specified field is not allowed."); #endregion @@ -89,34 +90,34 @@ public async Task>> Get( } /// - /// Get bigmap by ptr + /// Get bigmap by Id /// /// - /// Returns a bigmap with the specified ptr. + /// Returns a bigmap with the specified Id. /// - /// Bigmap pointer - /// Format of the bigmap key and value type: `0` - JSON, `2` - raw micheline + /// Bigmap Id + /// Format of the bigmap key and value type: `0` - JSON, `2` - Micheline /// - [HttpGet("{ptr:int}")] - public Task GetByPtr( - [Min(0)] int ptr, + [HttpGet("{id:int}")] + public Task GetBigMapById( + [Min(0)] int id, MichelineFormat micheline = MichelineFormat.Json) { - return BigMaps.Get(ptr, micheline); + return BigMaps.Get(id, micheline); } /// /// Get bigmap type /// /// - /// Returns a type of the bigmap with the specified ptr in Micheline format (with annotations). + /// Returns a type of the bigmap with the specified Id in Micheline format (with annotations). /// - /// Bigmap pointer + /// Bigmap Id /// - [HttpGet("{ptr:int}/type")] - public Task GetTypeByPtr([Min(0)] int ptr) + [HttpGet("{id:int}/type")] + public Task GetBigMapType([Min(0)] int id) { - return BigMaps.GetMicheType(ptr); + return BigMaps.GetMicheType(id); } /// @@ -125,21 +126,21 @@ public Task GetTypeByPtr([Min(0)] int ptr) /// /// Returns a list of bigmap keys. /// - /// Bigmap pointer + /// Bigmap Id /// Filters keys by status: `true` - active, `false` - removed. /// Filters keys by JSON key. Note, this query parameter supports the following format: `?key{.path?}{.mode?}=...`, /// so you can specify a path to a particular field to filter by, for example: `?key.token_id=...`. /// Filters keys by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. - /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. + /// Sorts bigmap keys by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. /// Specifies which or how many items should be skipped /// Maximum number of items to return - /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// - [HttpGet("{ptr:int}/keys")] + [HttpGet("{id:int}/keys")] public async Task>> GetKeys( - [Min(0)] int ptr, + [Min(0)] int id, bool? active, JsonParameter key, JsonParameter value, @@ -155,25 +156,25 @@ public async Task>> GetKeys( #endregion if (select == null) - return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, micheline)); if (select.Values != null) { if (select.Values.Length == 1) - return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Values[0], micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Values[0], micheline)); else - return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Values, micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Values, micheline)); } else { if (select.Fields.Length == 1) - return Ok(await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Fields[0], micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Fields[0], micheline)); else { return Ok(new SelectionResponse { Cols = select.Fields, - Rows = await BigMaps.GetKeys(ptr, active, key, value, sort, offset, limit, select.Fields, micheline) + Rows = await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Fields, micheline) }); } } @@ -183,23 +184,26 @@ public async Task>> GetKeys( /// Get bigmap key /// /// - /// Returns a bigmap key with the specified key value. + /// Returns the specified bigmap key. /// - /// Bigmap pointer - /// Plain key, for example, `.../keys/abcde`. - /// If the key is complex (an object or an array), you can specify it as is, for example, `.../keys/{"address":"tz123","token":123}`. - /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// Bigmap Id + /// Either a key hash (`expr123...`) or a plain value (`abcde...`). + /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// - [HttpGet("{ptr:int}/keys/{key}")] + [HttpGet("{id:int}/keys/{key}")] public async Task> GetKey( - [Min(0)] int ptr, + [Min(0)] int id, string key, MichelineFormat micheline = MichelineFormat.Json) { try { + if (Regex.IsMatch(key, @"^expr[0-9A-z]{50}$")) + return Ok(await BigMaps.GetKeyByHash(id, key, micheline)); + using var doc = JsonDocument.Parse(WrapKey(key)); - return Ok(await BigMaps.GetKey(ptr, doc.RootElement.GetRawText(), micheline)); + return Ok(await BigMaps.GetKey(id, doc.RootElement.GetRawText(), micheline)); } catch (JsonException) { @@ -212,22 +216,22 @@ public async Task> GetKey( } /// - /// Get key history + /// Get bigmap key history /// /// /// Returns updates history for the specified bigmap key. /// - /// Bigmap pointer - /// Plain key, for example, `.../keys/abcde/history`. - /// If the key is complex (an object or an array), you can specify it as is, for example, `.../keys/{"address":"tz123","token":123}/history`. + /// Bigmap Id + /// Either a key hash (`expr123...`) or a plain value (`abcde...`). + /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. /// Sorts bigmaps by specified field. Supported fields: `id` (default). /// Specifies which or how many items should be skipped /// Maximum number of items to return - /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// - [HttpGet("{ptr:int}/keys/{key}/history")] + [HttpGet("{id:int}/keys/{key}/history")] public async Task>> GetKeyHistory( - [Min(0)] int ptr, + [Min(0)] int id, string key, SortParameter sort, OffsetParameter offset, @@ -241,8 +245,11 @@ public async Task>> GetKeyHistory( try { + if (Regex.IsMatch(key, @"^expr[0-9A-z]{50}$")) + return Ok(await BigMaps.GetKeyByHashUpdates(id, key, sort, offset, limit, micheline)); + using var doc = JsonDocument.Parse(WrapKey(key)); - return Ok(await BigMaps.GetKeyUpdates(ptr, doc.RootElement.GetRawText(), sort, offset, limit, micheline)); + return Ok(await BigMaps.GetKeyUpdates(id, doc.RootElement.GetRawText(), sort, offset, limit, micheline)); } catch (JsonException) { @@ -255,41 +262,32 @@ public async Task>> GetKeyHistory( } /// - /// Get bigmap key by hash - /// - /// - /// Returns a bigmap key with the specified key hash. - /// - /// Bigmap pointer - /// Key hash - /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string - /// - [HttpGet("{ptr:int}/keys/{hash:regex(^expr[[0-9A-z]]{{50}}$)}")] - public Task GetKeyByHash( - [Min(0)] int ptr, - string hash, - MichelineFormat micheline = MichelineFormat.Json) - { - return BigMaps.GetKeyByHash(ptr, hash, micheline); - } - - /// - /// Get key by hash history + /// Get historical keys /// /// - /// Returns updates history for the bigmap key with the specified key hash. + /// Returns a list of bigmap keys at the specific block. /// - /// Bigmap pointer - /// Key hash - /// Sorts bigmaps by specified field. Supported fields: `id` (default). + /// Bigmap Id + /// Level of the block at which you want to get bigmap keys + /// Filters keys by status: `true` - active, `false` - removed. + /// Filters keys by JSON key. Note, this query parameter supports the following format: `?key{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?key.token_id=...`. + /// Filters keys by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. + /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. + /// Sorts bigmap keys by specified field. Supported fields: `id` (default). /// Specifies which or how many items should be skipped /// Maximum number of items to return - /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - micheline, `3` - micheline string + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// - [HttpGet("{ptr:int}/keys/{hash:regex(^expr[[0-9A-z]]{{50}}$)}/history")] - public async Task>> GetKeyByHashHistory( - [Min(0)] int ptr, - string hash, + [HttpGet("{id:int}/historical_keys/{level:int}")] + public async Task>> GetHistoricalKeys( + [Min(0)] int id, + [Min(0)] int level, + bool? active, + JsonParameter key, + JsonParameter value, + SelectParameter select, SortParameter sort, OffsetParameter offset, [Range(0, 10000)] int limit = 100, @@ -300,7 +298,66 @@ public async Task>> GetKeyByHashHistory( return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); #endregion - return Ok(await BigMaps.GetKeyByHashUpdates(ptr, hash, sort, offset, limit, micheline)); + if (select == null) + return Ok(await BigMaps.GetHistoricalKeys(id, level, active, key, value, sort, offset, limit, micheline)); + + if (select.Values != null) + { + if (select.Values.Length == 1) + return Ok(await BigMaps.GetHistoricalKeys(id, level, active, key, value, sort, offset, limit, select.Values[0], micheline)); + else + return Ok(await BigMaps.GetHistoricalKeys(id, level, active, key, value, sort, offset, limit, select.Values, micheline)); + } + else + { + if (select.Fields.Length == 1) + return Ok(await BigMaps.GetHistoricalKeys(id, level, active, key, value, sort, offset, limit, select.Fields[0], micheline)); + else + { + return Ok(new SelectionResponse + { + Cols = select.Fields, + Rows = await BigMaps.GetHistoricalKeys(id, level, active, key, value, sort, offset, limit, select.Fields, micheline) + }); + } + } + } + + /// + /// Get historical key + /// + /// + /// Returns the specified bigmap key at the specified level. + /// + /// Bigmap Id + /// Level of the block at which you want to get bigmap key + /// Either a key hash (`expr123...`) or a plain value (`abcde...`). + /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{id:int}/historical_keys/{level:int}/{key}")] + public async Task> GetKey( + [Min(0)] int id, + [Min(0)] int level, + string key, + MichelineFormat micheline = MichelineFormat.Json) + { + try + { + if (Regex.IsMatch(key, @"^expr[0-9A-z]{50}$")) + return Ok(await BigMaps.GetHistoricalKeyByHash(id, level, key, micheline)); + + using var doc = JsonDocument.Parse(WrapKey(key)); + return Ok(await BigMaps.GetHistoricalKey(id, level, doc.RootElement.GetRawText(), micheline)); + } + catch (JsonException) + { + return new BadRequest(nameof(key), "invalid json value"); + } + catch + { + throw; + } } string WrapKey(string key) diff --git a/Tzkt.Api/Models/BigMaps/BigMap.cs b/Tzkt.Api/Models/BigMaps/BigMap.cs index ba3304ace..2fbd211b8 100644 --- a/Tzkt.Api/Models/BigMaps/BigMap.cs +++ b/Tzkt.Api/Models/BigMaps/BigMap.cs @@ -3,9 +3,9 @@ public class BigMap { /// - /// Bigmap pointer + /// Bigmap Id /// - public int Ptr { get; set; } + public int Id { get; set; } /// /// Smart contract in which's storage the bigmap is allocated diff --git a/Tzkt.Api/Models/BigMaps/BigMapKey.cs b/Tzkt.Api/Models/BigMaps/BigMapKey.cs index 0ab5fba81..ef730a802 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapKey.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKey.cs @@ -1,10 +1,10 @@ -using System.Text.Json.Serialization; - -namespace Tzkt.Api.Models +namespace Tzkt.Api.Models { public class BigMapKey { - [JsonIgnore] + /// + /// Internal Id, can be used for pagination + /// public int Id { get; set; } /// @@ -41,6 +41,5 @@ public class BigMapKey /// Total number of actions with the bigmap key /// public int Updates { get; set; } - } } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs b/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs new file mode 100644 index 000000000..ffef5f38a --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs @@ -0,0 +1,30 @@ +namespace Tzkt.Api.Models +{ + public class BigMapKeyShort + { + /// + /// Internal Id, can be used for pagination + /// + public int Id { get; set; } + + /// + /// Bigmap key status: `true` - active, `false` - removed + /// + public bool Active { get; set; } + + /// + /// Key hash + /// + public string Hash { get; set; } + + /// + /// Key in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Key { get; set; } + + /// + /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Value { get; set; } + } +} diff --git a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs index 6c5521221..852a8b95e 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs @@ -5,7 +5,7 @@ namespace Tzkt.Api.Models public class BigMapUpdate { /// - /// Internal ID that can be used for pagination + /// Internal Id, can be used for pagination /// public int Id { get; set; } diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 4405a7b0d..b2c3205de 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -80,7 +80,6 @@ public async Task> Get( .Filter("Active", active) .Take(sort, offset, limit, x => x switch { - "ptr" => ("Ptr", "Ptr"), "firstLevel" => ("Id", "FirstLevel"), "lastLevel" => ("LastLevel", "LastLevel"), "totalKeys" => ("TotalKeys", "TotalKeys"), @@ -109,7 +108,7 @@ public async Task Get( { switch (field) { - case "ptr": columns.Add(@"""Ptr"""); break; + case "id": columns.Add(@"""Ptr"""); break; case "contract": columns.Add(@"""ContractId"""); break; case "path": columns.Add(@"""StoragePath"""); break; case "active": columns.Add(@"""Active"""); break; @@ -131,7 +130,6 @@ public async Task Get( .Filter("Active", active) .Take(sort, offset, limit, x => x switch { - "ptr" => ("Ptr", "Ptr"), "firstLevel" => ("Id", "FirstLevel"), "lastLevel" => ("LastLevel", "LastLevel"), "totalKeys" => ("TotalKeys", "TotalKeys"), @@ -151,7 +149,7 @@ public async Task Get( { switch (fields[i]) { - case "ptr": + case "id": foreach (var row in rows) result[j++][i] = row.Ptr; break; @@ -217,7 +215,7 @@ public async Task Get( var columns = new HashSet(1); switch (field) { - case "ptr": columns.Add(@"""Ptr"""); break; + case "id": columns.Add(@"""Ptr"""); break; case "contract": columns.Add(@"""ContractId"""); break; case "path": columns.Add(@"""StoragePath"""); break; case "active": columns.Add(@"""Active"""); break; @@ -238,7 +236,6 @@ public async Task Get( .Filter("Active", active) .Take(sort, offset, limit, x => x switch { - "ptr" => ("Ptr", "Ptr"), "firstLevel" => ("Id", "FirstLevel"), "lastLevel" => ("LastLevel", "LastLevel"), "totalKeys" => ("TotalKeys", "TotalKeys"), @@ -256,7 +253,7 @@ public async Task Get( switch (field) { - case "ptr": + case "id": foreach (var row in rows) result[j++] = row.Ptr; break; @@ -457,25 +454,11 @@ public async Task GetKeys( break; case "key": foreach (var row in rows) - result[j++][i] = micheline switch - { - MichelineFormat.Json => new RawJson(row.JsonKey), - MichelineFormat.JsonString => row.JsonKey, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), - MichelineFormat.RawString => Micheline.ToJson(row.RawKey), - _ => null - }; + result[j++][i] = FormatKey(row, micheline); break; case "value": foreach (var row in rows) - result[j++][i] = micheline switch - { - MichelineFormat.Json => new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => Micheline.ToJson(row.RawValue), - _ => null - }; + result[j++][i] = FormatValue(row, micheline); break; } } @@ -557,25 +540,277 @@ public async Task GetKeys( break; case "key": foreach (var row in rows) - result[j++] = micheline switch - { - MichelineFormat.Json => new RawJson(row.JsonKey), - MichelineFormat.JsonString => row.JsonKey, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), - MichelineFormat.RawString => Micheline.ToJson(row.RawKey), - _ => null - }; + result[j++] = FormatKey(row, micheline); break; case "value": foreach (var row in rows) - result[j++] = micheline switch - { - MichelineFormat.Json => new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => Micheline.ToJson(row.RawValue), - _ => null - }; + result[j++] = FormatValue(row, micheline); + break; + } + + return result; + } + #endregion + + #region historical keys + async Task GetHistoricalKey( + BigMapKey key, + int level, + MichelineFormat micheline) + { + if (key == null || level < key.FirstLevel) + return null; + + if (level > key.LastLevel) + return new BigMapKeyShort + { + Id = key.Id, + Hash = key.Hash, + Key = key.Key, + Value = key.Value, + Active = key.Active + }; + + var valCol = (int)micheline < 2 ? "JsonValue" : "RawValue"; + + var sql = $@" + SELECT ""Action"", ""{valCol}"" + FROM ""BigMapUpdates"" + WHERE ""BigMapKeyId"" = {key.Id} + AND ""Level"" <= {level} + ORDER BY ""Level"" DESC + LIMIT 1"; + + using var db = GetConnection(); + var row = await db.QueryFirstOrDefaultAsync(sql); + if (row == null) return null; + + return new BigMapKeyShort + { + Id = key.Id, + Hash = key.Hash, + Key = key.Key, + Value = FormatValue(row, micheline), + Active = row.Action != (int)Data.Models.BigMapAction.RemoveKey + }; + } + + public async Task GetHistoricalKey( + int ptr, + int level, + string key, + MichelineFormat micheline) + { + return await GetHistoricalKey(await GetKey(ptr, key, micheline), level, micheline); + } + + public async Task GetHistoricalKeyByHash( + int ptr, + int level, + string hash, + MichelineFormat micheline) + { + return await GetHistoricalKey(await GetKeyByHash(ptr, hash, micheline), level, micheline); + } + + public async Task> GetHistoricalKeys( + int ptr, + int level, + bool? active, + JsonParameter key, + JsonParameter value, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + var (keyCol, valCol) = (int)micheline < 2 + ? ("JsonKey", "JsonValue") + : ("RawKey", "RawValue"); + + var subQuery = $@" + SELECT DISTINCT ON (""BigMapKeyId"") + k.""Id"", k.""KeyHash"", k.""{keyCol}"", + (u.""Action"" != {(int)Data.Models.BigMapAction.RemoveKey}) as ""Active"", u.""{valCol}"" + FROM ""BigMapUpdates"" as u + INNER JOIN ""BigMapKeys"" as k + ON k.""Id"" = u.""BigMapKeyId"" + WHERE u.""BigMapPtr"" = {ptr} + AND u.""Level"" <= {level} + ORDER BY ""BigMapKeyId"", ""Level"" DESC"; + + var sql = new SqlBuilder($"SELECT * from ({subQuery}) as updates") + .Filter("Active", active) + .Filter("JsonKey", key) + .Filter("JsonValue", value) + .Take(sort, offset, limit, x => ("Id", "Id")); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + return rows.Select(row => (BigMapKeyShort)ReadBigMapKeyShort(row, micheline)); + } + + public async Task GetHistoricalKeys( + int ptr, + int level, + bool? active, + JsonParameter key, + JsonParameter value, + SortParameter sort, + OffsetParameter offset, + int limit, + string[] fields, + MichelineFormat micheline) + { + var (keyCol, valCol) = (int)micheline < 2 + ? ("JsonKey", "JsonValue") + : ("RawKey", "RawValue"); + + var columns = new HashSet(fields.Length); + foreach (var field in fields) + { + switch (field) + { + case "id": columns.Add(@"""Id"""); break; + case "active": columns.Add(@"""Active"""); break; + case "hash": columns.Add(@"""KeyHash"""); break; + case "key": columns.Add($@"""{keyCol}"""); break; + case "value": columns.Add($@"""{valCol}"""); break; + } + } + + if (columns.Count == 0) + return Array.Empty(); + + var subQuery = $@" + SELECT DISTINCT ON (""BigMapKeyId"") + k.""Id"", k.""KeyHash"", k.""{keyCol}"", + (u.""Action"" != {(int)Data.Models.BigMapAction.RemoveKey}) as ""Active"", u.""{valCol}"" + FROM ""BigMapUpdates"" as u + INNER JOIN ""BigMapKeys"" as k + ON k.""Id"" = u.""BigMapKeyId"" + WHERE u.""BigMapPtr"" = {ptr} + AND u.""Level"" <= {level} + ORDER BY ""BigMapKeyId"", ""Level"" DESC"; + + var sql = new SqlBuilder($"SELECT {string.Join(',', columns)} from ({subQuery}) as updates") + .Filter("Active", active) + .Filter("JsonKey", key) + .Filter("JsonValue", value) + .Take(sort, offset, limit, x => ("Id", "Id")); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + var result = new object[rows.Count()][]; + for (int i = 0; i < result.Length; i++) + result[i] = new object[fields.Length]; + + for (int i = 0, j = 0; i < fields.Length; j = 0, i++) + { + switch (fields[i]) + { + case "id": + foreach (var row in rows) + result[j++][i] = row.Id; + break; + case "active": + foreach (var row in rows) + result[j++][i] = row.Active; + break; + case "hash": + foreach (var row in rows) + result[j++][i] = row.KeyHash; + break; + case "key": + foreach (var row in rows) + result[j++][i] = FormatKey(row, micheline); + break; + case "value": + foreach (var row in rows) + result[j++][i] = FormatValue(row, micheline); + break; + } + } + + return result; + } + + public async Task GetHistoricalKeys( + int ptr, + int level, + bool? active, + JsonParameter key, + JsonParameter value, + SortParameter sort, + OffsetParameter offset, + int limit, + string field, + MichelineFormat micheline) + { + var (keyCol, valCol) = (int)micheline < 2 + ? ("JsonKey", "JsonValue") + : ("RawKey", "RawValue"); + + var columns = new HashSet(1); + switch (field) + { + case "id": columns.Add(@"""Id"""); break; + case "active": columns.Add(@"""Active"""); break; + case "hash": columns.Add(@"""KeyHash"""); break; + case "key": columns.Add($@"""{keyCol}"""); break; + case "value": columns.Add($@"""{valCol}"""); break; + } + + if (columns.Count == 0) + return Array.Empty(); + + var subQuery = $@" + SELECT DISTINCT ON (""BigMapKeyId"") + k.""Id"", k.""KeyHash"", k.""{keyCol}"", + (u.""Action"" != {(int)Data.Models.BigMapAction.RemoveKey}) as ""Active"", u.""{valCol}"" + FROM ""BigMapUpdates"" as u + INNER JOIN ""BigMapKeys"" as k + ON k.""Id"" = u.""BigMapKeyId"" + WHERE u.""BigMapPtr"" = {ptr} + AND u.""Level"" <= {level} + ORDER BY ""BigMapKeyId"", ""Level"" DESC"; + + var sql = new SqlBuilder($"SELECT {string.Join(',', columns)} from ({subQuery}) as updates") + .Filter("Active", active) + .Filter("JsonKey", key) + .Filter("JsonValue", value) + .Take(sort, offset, limit, x => ("Id", "Id")); + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql.Query, sql.Params); + + //TODO: optimize memory allocation + var result = new object[rows.Count()]; + var j = 0; + + switch (field) + { + case "id": + foreach (var row in rows) + result[j++] = row.Id; + break; + case "active": + foreach (var row in rows) + result[j++] = row.Active; + break; + case "hash": + foreach (var row in rows) + result[j++] = row.KeyHash; + break; + case "key": + foreach (var row in rows) + result[j++] = FormatKey(row, micheline); + break; + case "value": + foreach (var row in rows) + result[j++] = FormatValue(row, micheline); break; } @@ -643,7 +878,7 @@ BigMap ReadBigMap(dynamic row, MichelineFormat format) { return new BigMap { - Ptr = row.Ptr, + Id = row.Ptr, Contract = Accounts.GetAlias(row.ContractId), Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), Active = row.Active, @@ -671,22 +906,20 @@ BigMapKey ReadBigMapKey(dynamic row, MichelineFormat format) LastLevel = row.LastLevel, Updates = row.Updates, Hash = row.KeyHash, - Key = format switch - { - MichelineFormat.Json => new RawJson(row.JsonKey), - MichelineFormat.JsonString => row.JsonKey, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), - MichelineFormat.RawString => Micheline.ToJson(row.RawKey), - _ => null - }, - Value = format switch - { - MichelineFormat.Json => new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => Micheline.ToJson(row.RawValue), - _ => null - } + Key = FormatKey(row, format), + Value = FormatValue(row, format) + }; + } + + BigMapKeyShort ReadBigMapKeyShort(dynamic row, MichelineFormat format) + { + return new BigMapKeyShort + { + Id = row.Id, + Active = row.Active, + Hash = row.KeyHash, + Key = FormatKey(row, format), + Value = FormatValue(row, format) }; } @@ -698,17 +931,28 @@ BigMapUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) Level = row.Level, Timestamp = Times[row.Level], Action = UpdateAction((int)row.Action), - Value = format switch - { - MichelineFormat.Json => new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => Micheline.ToJson(row.RawValue), - _ => null - } + Value = FormatValue(row, format) }; } + object FormatKey(dynamic row, MichelineFormat format) => format switch + { + MichelineFormat.Json => new RawJson(row.JsonKey), + MichelineFormat.JsonString => row.JsonKey, + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawKey)), + MichelineFormat.RawString => Micheline.ToJson(row.RawKey), + _ => null + }; + + object FormatValue(dynamic row, MichelineFormat format) => format switch + { + MichelineFormat.Json => new RawJson(row.JsonValue), + MichelineFormat.JsonString => row.JsonValue, + MichelineFormat.Raw => new RawJson(Micheline.ToJson(row.RawValue)), + MichelineFormat.RawString => Micheline.ToJson(row.RawValue), + _ => null + }; + string UpdateAction(int action) => action switch { 1 => BigMapActions.AddKey, diff --git a/Tzkt.Api/Tzkt.Api.csproj b/Tzkt.Api/Tzkt.Api.csproj index c12aa340e..e25c59164 100644 --- a/Tzkt.Api/Tzkt.Api.csproj +++ b/Tzkt.Api/Tzkt.Api.csproj @@ -37,8 +37,8 @@ - - + + diff --git a/Tzkt.Data/Tzkt.Data.csproj b/Tzkt.Data/Tzkt.Data.csproj index c0ee22487..5af14358d 100644 --- a/Tzkt.Data/Tzkt.Data.csproj +++ b/Tzkt.Data/Tzkt.Data.csproj @@ -6,7 +6,7 @@ - + diff --git a/Tzkt.Sync/Tzkt.Sync.csproj b/Tzkt.Sync/Tzkt.Sync.csproj index d04da81df..c5354b5ac 100644 --- a/Tzkt.Sync/Tzkt.Sync.csproj +++ b/Tzkt.Sync/Tzkt.Sync.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + From a983e5b0501cd240d69edbf10bed9631d1451355 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Fri, 2 Apr 2021 15:22:36 +0300 Subject: [PATCH 13/35] Add filtering of bigmaps and keys by last level --- Tzkt.Api/Controllers/BigMapsController.cs | 26 +++++++++++++--------- Tzkt.Api/Repositories/BigMapsRepository.cs | 12 ++++++++++ 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 3029271fa..31d3f5aca 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -43,6 +43,7 @@ public Task GetBigMapsCount() /// /// Filters bigmaps by smart contract address. /// Filters bigmaps by status: `true` - active, `false` - removed. + /// Filters bigmaps by the last update level. /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. /// Specifies which or how many items should be skipped @@ -53,6 +54,7 @@ public Task GetBigMapsCount() public async Task>> GetBigMaps( AccountParameter contract, bool? active, + Int32Parameter lastLevel, SelectParameter select, SortParameter sort, OffsetParameter offset, @@ -65,25 +67,25 @@ public async Task>> GetBigMaps( #endregion if (select == null) - return Ok(await BigMaps.Get(contract, active, sort, offset, limit, micheline)); + return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, micheline)); if (select.Values != null) { if (select.Values.Length == 1) - return Ok(await BigMaps.Get(contract, active, sort, offset, limit, select.Values[0], micheline)); + return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Values[0], micheline)); else - return Ok(await BigMaps.Get(contract, active, sort, offset, limit, select.Values, micheline)); + return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Values, micheline)); } else { if (select.Fields.Length == 1) - return Ok(await BigMaps.Get(contract, active, sort, offset, limit, select.Fields[0], micheline)); + return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Fields[0], micheline)); else { return Ok(new SelectionResponse { Cols = select.Fields, - Rows = await BigMaps.Get(contract, active, sort, offset, limit, select.Fields, micheline) + Rows = await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Fields, micheline) }); } } @@ -132,6 +134,7 @@ public Task GetBigMapType([Min(0)] int id) /// so you can specify a path to a particular field to filter by, for example: `?key.token_id=...`. /// Filters keys by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. + /// Filters bigmap keys by the last update level. /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. /// Sorts bigmap keys by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. /// Specifies which or how many items should be skipped @@ -144,6 +147,7 @@ public async Task>> GetKeys( bool? active, JsonParameter key, JsonParameter value, + Int32Parameter lastLevel, SelectParameter select, SortParameter sort, OffsetParameter offset, @@ -156,25 +160,25 @@ public async Task>> GetKeys( #endregion if (select == null) - return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, lastLevel, sort, offset, limit, micheline)); if (select.Values != null) { if (select.Values.Length == 1) - return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Values[0], micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, lastLevel, sort, offset, limit, select.Values[0], micheline)); else - return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Values, micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, lastLevel, sort, offset, limit, select.Values, micheline)); } else { if (select.Fields.Length == 1) - return Ok(await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Fields[0], micheline)); + return Ok(await BigMaps.GetKeys(id, active, key, value, lastLevel, sort, offset, limit, select.Fields[0], micheline)); else { return Ok(new SelectionResponse { Cols = select.Fields, - Rows = await BigMaps.GetKeys(id, active, key, value, sort, offset, limit, select.Fields, micheline) + Rows = await BigMaps.GetKeys(id, active, key, value, lastLevel, sort, offset, limit, select.Fields, micheline) }); } } @@ -327,7 +331,7 @@ public async Task>> GetHistoricalKeys( /// Get historical key /// /// - /// Returns the specified bigmap key at the specified level. + /// Returns the specified bigmap key at the specific block. /// /// Bigmap Id /// Level of the block at which you want to get bigmap key diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index b2c3205de..da26e051d 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -70,6 +70,7 @@ public async Task Get(int ptr, MichelineFormat micheline) public async Task> Get( AccountParameter contract, bool? active, + Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, @@ -78,6 +79,7 @@ public async Task> Get( var sql = new SqlBuilder(@"SELECT * FROM ""BigMaps""") .Filter("ContractId", contract) .Filter("Active", active) + .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { "firstLevel" => ("Id", "FirstLevel"), @@ -97,6 +99,7 @@ public async Task> Get( public async Task Get( AccountParameter contract, bool? active, + Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, @@ -128,6 +131,7 @@ public async Task Get( var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMaps""") .Filter("ContractId", contract) .Filter("Active", active) + .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { "firstLevel" => ("Id", "FirstLevel"), @@ -206,6 +210,7 @@ public async Task Get( public async Task Get( AccountParameter contract, bool? active, + Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, @@ -234,6 +239,7 @@ public async Task Get( var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMaps""") .Filter("ContractId", contract) .Filter("Active", active) + .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { "firstLevel" => ("Id", "FirstLevel"), @@ -351,6 +357,7 @@ public async Task> GetKeys( bool? active, JsonParameter key, JsonParameter value, + Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, @@ -361,6 +368,7 @@ public async Task> GetKeys( .Filter("Active", active) .Filter("JsonKey", key) .Filter("JsonValue", value) + .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { "firstLevel" => ("Id", "FirstLevel"), @@ -380,6 +388,7 @@ public async Task GetKeys( bool? active, JsonParameter key, JsonParameter value, + Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, @@ -413,6 +422,7 @@ public async Task GetKeys( .Filter("Active", active) .Filter("JsonKey", key) .Filter("JsonValue", value) + .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { "firstLevel" => ("Id", "FirstLevel"), @@ -471,6 +481,7 @@ public async Task GetKeys( bool? active, JsonParameter key, JsonParameter value, + Int32Parameter lastLevel, SortParameter sort, OffsetParameter offset, int limit, @@ -501,6 +512,7 @@ public async Task GetKeys( .Filter("Active", active) .Filter("JsonKey", key) .Filter("JsonValue", value) + .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { "firstLevel" => ("Id", "FirstLevel"), From bf2751e81cbee905905b82acfb187f6107cdd851 Mon Sep 17 00:00:00 2001 From: m-kus Date: Fri, 2 Apr 2021 19:11:25 +0300 Subject: [PATCH 14/35] Contract interface endpoint --- .dockerignore | 4 ++ Makefile | 6 ++ Tzkt.Api/Controllers/ContractsController.cs | 16 ++++- Tzkt.Api/Models/Accounts/ContractInterface.cs | 62 +++++++++++++++++++ Tzkt.Api/Repositories/AccountRepository.cs | 53 ++++++++++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Tzkt.Api/Models/Accounts/ContractInterface.cs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..077cc6e1f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +*.backup +bin/ +Makefile +docker-compose.yml \ No newline at end of file diff --git a/Makefile b/Makefile index de4126513..273f92f10 100644 --- a/Makefile +++ b/Makefile @@ -33,3 +33,9 @@ migration: sync: export $$(cat .env | xargs) && dotnet run -p Tzkt.Sync -v normal + +api-image: + docker build -t bakingbad/tzkt-api:latest -f ./Tzkt.Api/Dockerfile . + +sync-image: + docker build -t bakingbad/tzkt-sync:latest -f ./Tzkt.Sync/Dockerfile . \ No newline at end of file diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index 0fb10f14f..59e5b5df1 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -165,6 +165,20 @@ public async Task GetCode([Address] string address, [Range(0, 2)] int fo return await Accounts.GetByteCode(address); } + /// + /// Get JSON Schema [2020-12] interface for the contract + /// + /// + /// Returns standard JSON Schema for contract storage, entrypoints, and Big_map entries. + /// + /// Contract address + /// + [HttpGet("{address}/interface")] + public Task GetInterface([Address] string address) + { + return Accounts.GetContractInterface(address); + } + /// /// Get contract entrypoints /// @@ -350,8 +364,6 @@ public Task GetRawStorage([Address] string address, [Min(0)] int lev [HttpGet("{address}/storage/raw/schema")] public Task GetRawStorageSchema([Address] string address, [Min(0)] int level = 0) { - if (level == 0) - return Accounts.GetRawStorageSchema(address); return Accounts.GetRawStorageSchema(address, level); } diff --git a/Tzkt.Api/Models/Accounts/ContractInterface.cs b/Tzkt.Api/Models/Accounts/ContractInterface.cs new file mode 100644 index 000000000..e11980d81 --- /dev/null +++ b/Tzkt.Api/Models/Accounts/ContractInterface.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Tzkt.Api.Models +{ + public class BigMapInterface + { + /// + /// Full path to the Big_map in the contract storage + /// + public string Path { get; set; } + + /// + /// Big_map name, if exists (field annotation) + /// + public string Name { get; set; } + + /// + /// JSON Schema of the Big_map key in humanified format (as returned by API) + /// + public JsonString KeySchema { get; set; } + + /// + /// JSON Schema of the Big_map value in humanified format (as returned by API) + /// + public JsonString ValueSchema { get; set; } + } + + public class EntrypointInterface + { + /// + /// Entrypoint name + /// + public string Name { get; set; } + + /// + /// JSON Schema of the entrypoint parameter in humanified format (as returned by API) + /// + public JsonString ParameterSchema { get; set; } + } + + public class ContractInterface + { + /// + /// JSON Schema of the contract storage in humanified format (as returned by API) + /// + public JsonString StorageSchema { get; set; } + + /// + /// List of terminal entrypoints + /// + public List Entrypoints { get; set; } + + /// + /// List of currently available Big_maps + /// + public List BigMaps { get; set; } + } +} diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index 772035f37..9d899e955 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.IO; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Dapper; @@ -2264,6 +2266,57 @@ public async Task GetMichelsonCode(string address) return code.ToMichelson(); } + public async Task GetContractInterface(string address) + { + var rawAccount = await Accounts.GetAsync(address); + if (rawAccount is not RawContract contract) return null; + + ContractParameter param; + ContractStorage storage; + + if (contract.Kind == 0) + { + param = Data.Models.Script.ManagerTz.Parameter; + storage = Data.Models.Script.ManagerTz.Storage; + } + else + { + using var db = GetConnection(); + var script = await db.QueryFirstOrDefaultAsync($@" + SELECT ""StorageSchema"", ""ParameterSchema"" + FROM ""Scripts"" + WHERE ""ContractId"" = {contract.Id} AND ""Current"" = true + LIMIT 1" + ); + if (script == null) return null; + param = new ContractParameter(Micheline.FromBytes(script.ParameterSchema)); + storage = new ContractStorage(Micheline.FromBytes(script.StorageSchema)); + } + + var rawStorage = await GetRawStorageValue(address); + var storageTreeView = storage.Schema.ToTreeView(rawStorage); + + return new ContractInterface{ + StorageSchema = storage.GetJsonSchema(), + Entrypoints = param.Entrypoints + .Where(x => param.IsEntrypointUseful(x.Key)) + .Select(x => new EntrypointInterface{ + Name = x.Key, + ParameterSchema = x.Value.GetJsonSchema() + }) + .ToList(), + BigMaps = storageTreeView.Nodes() + .Where(x => x.Schema is BigMapSchema) + .Select(x => new BigMapInterface{ + Name = x.Name, + Path = x.Path, + KeySchema = (x.Schema as BigMapSchema).Key.GetJsonSchema(), + ValueSchema = (x.Schema as BigMapSchema).Value.GetJsonSchema() + }) + .ToList() + }; + } + public async Task BuildEntrypointParameters(string address, string name, object value) { var rawAccount = await Accounts.GetAsync(address); From 031cbf2cc577e4040cf58567bdbf399022a1db40 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Fri, 2 Apr 2021 19:57:31 +0300 Subject: [PATCH 15/35] Add /bigmaps API endpoints to /contracts --- Tzkt.Api/Controllers/ContractsController.cs | 64 ++++++++++++++++++- Tzkt.Api/Models/Accounts/ContractInterface.cs | 8 +-- Tzkt.Api/Repositories/AccountRepository.cs | 13 ++-- Tzkt.Api/Repositories/BigMapsRepository.cs | 38 ++++++++++- 4 files changed, 109 insertions(+), 14 deletions(-) diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index 59e5b5df1..7eefd39b4 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -17,9 +17,12 @@ namespace Tzkt.Api.Controllers public class ContractsController : ControllerBase { private readonly AccountRepository Accounts; - public ContractsController(AccountRepository accounts) + private readonly BigMapsRepository BigMaps; + + public ContractsController(AccountRepository accounts, BigMapsRepository bigMaps) { Accounts = accounts; + BigMaps = bigMaps; } /// @@ -364,6 +367,8 @@ public Task GetRawStorage([Address] string address, [Min(0)] int lev [HttpGet("{address}/storage/raw/schema")] public Task GetRawStorageSchema([Address] string address, [Min(0)] int level = 0) { + if (level == 0) + return Accounts.GetRawStorageSchema(address); return Accounts.GetRawStorageSchema(address, level); } @@ -382,5 +387,62 @@ public Task> GetRawStorageHistory([Address] string ad { return Accounts.GetRawStorageHistory(address, lastId, limit); } + + /// + /// Get contract bigmaps + /// + /// + /// Returns all active bigmaps allocated in contract's storage. + /// + /// Contract address + /// Sorts bigmap keys by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps")] + public async Task>> GetBigMaps( + [Address] string address, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + #region validate + if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "updates")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + var acc = await Accounts.Accounts.GetAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(Enumerable.Empty()); + + return Ok(await BigMaps.Get(new AccountParameter { Eq = contract.Id }, true, null, sort, offset, limit, micheline)); + } + + /// + /// Get bigmap by name + /// + /// + /// Returns contract bigmap with the specified name. + /// + /// Contract address + /// Bigmap name is the last part in bigmap's storage path. + /// If the path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `blabla.ledger`, you can specify the full path. + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps/{name}")] + public async Task> GetBigMapByName( + [Address] string address, + string name, + MichelineFormat micheline = MichelineFormat.Json) + { + var acc = await Accounts.Accounts.GetAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(Enumerable.Empty()); + + return Ok(await BigMaps.Get(contract.Id, name, micheline)); + } } } diff --git a/Tzkt.Api/Models/Accounts/ContractInterface.cs b/Tzkt.Api/Models/Accounts/ContractInterface.cs index e11980d81..73847f8ae 100644 --- a/Tzkt.Api/Models/Accounts/ContractInterface.cs +++ b/Tzkt.Api/Models/Accounts/ContractInterface.cs @@ -21,12 +21,12 @@ public class BigMapInterface /// /// JSON Schema of the Big_map key in humanified format (as returned by API) /// - public JsonString KeySchema { get; set; } + public RawJson KeySchema { get; set; } /// /// JSON Schema of the Big_map value in humanified format (as returned by API) /// - public JsonString ValueSchema { get; set; } + public RawJson ValueSchema { get; set; } } public class EntrypointInterface @@ -39,7 +39,7 @@ public class EntrypointInterface /// /// JSON Schema of the entrypoint parameter in humanified format (as returned by API) /// - public JsonString ParameterSchema { get; set; } + public RawJson ParameterSchema { get; set; } } public class ContractInterface @@ -47,7 +47,7 @@ public class ContractInterface /// /// JSON Schema of the contract storage in humanified format (as returned by API) /// - public JsonString StorageSchema { get; set; } + public RawJson StorageSchema { get; set; } /// /// List of terminal entrypoints diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index 9d899e955..2d01dfdcb 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.IO; -using System.Text.Json; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Dapper; @@ -17,7 +15,7 @@ namespace Tzkt.Api.Repositories { public class AccountRepository : DbConnection { - readonly AccountsCache Accounts; + public readonly AccountsCache Accounts; readonly StateCache State; readonly TimeCache Time; readonly OperationRepository Operations; @@ -2296,18 +2294,21 @@ LIMIT 1" var rawStorage = await GetRawStorageValue(address); var storageTreeView = storage.Schema.ToTreeView(rawStorage); - return new ContractInterface{ + return new ContractInterface + { StorageSchema = storage.GetJsonSchema(), Entrypoints = param.Entrypoints .Where(x => param.IsEntrypointUseful(x.Key)) - .Select(x => new EntrypointInterface{ + .Select(x => new EntrypointInterface + { Name = x.Key, ParameterSchema = x.Value.GetJsonSchema() }) .ToList(), BigMaps = storageTreeView.Nodes() .Where(x => x.Schema is BigMapSchema) - .Select(x => new BigMapInterface{ + .Select(x => new BigMapInterface + { Name = x.Name, Path = x.Path, KeySchema = (x.Schema as BigMapSchema).Key.GetJsonSchema(), diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index da26e051d..60b30c2a1 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -67,6 +67,38 @@ public async Task Get(int ptr, MichelineFormat micheline) return ReadBigMap(row, micheline); } + public async Task Get(int contractId, string path, MichelineFormat micheline) + { + var sql = @" + SELECT * + FROM ""BigMaps"" + WHERE ""ContractId"" = @id + AND ""StoragePath"" LIKE @path"; + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql, new { id = contractId, path = $"%{path}" }); + if (!rows.Any()) return null; + + var row = rows.FirstOrDefault(x => x.StoragePath == path); + return ReadBigMap(row ?? rows.FirstOrDefault(), micheline); + } + + public async Task GetId(int contractId, string path, MichelineFormat micheline) + { + var sql = @" + SELECT ""Ptr"", ""StoragePath"" + FROM ""BigMaps"" + WHERE ""ContractId"" = @id + AND ""StoragePath"" LIKE @path"; + + using var db = GetConnection(); + var rows = await db.QueryAsync(sql, new { id = contractId, path = $"%{path}" }); + if (!rows.Any()) return null; + + var row = rows.FirstOrDefault(x => x.StoragePath == path); + return (row ?? rows.FirstOrDefault())?.Ptr; + } + public async Task> Get( AccountParameter contract, bool? active, @@ -163,7 +195,7 @@ public async Task Get( break; case "path": foreach (var row in rows) - result[j++][i] = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'); + result[j++][i] = row.StoragePath; break; case "active": foreach (var row in rows) @@ -269,7 +301,7 @@ public async Task Get( break; case "path": foreach (var row in rows) - result[j++] = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'); + result[j++] = row.StoragePath; break; case "active": foreach (var row in rows) @@ -892,7 +924,7 @@ BigMap ReadBigMap(dynamic row, MichelineFormat format) { Id = row.Ptr, Contract = Accounts.GetAlias(row.ContractId), - Path = ((string)row.StoragePath).Replace(".", "..").Replace(',', '.'), + Path = row.StoragePath, Active = row.Active, FirstLevel = row.FirstLevel, LastLevel = row.LastLevel, From 85a1c8a3c9ebe4833a23dba44a8eff48544be1ff Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Sun, 4 Apr 2021 22:24:54 +0300 Subject: [PATCH 16/35] Add API endpoints for accessing bigmap keys, etc. by bigmap name --- Tzkt.Api/Controllers/BigMapsController.cs | 2 +- Tzkt.Api/Controllers/ContractsController.cs | 371 ++++++++++++++++++- Tzkt.Api/Repositories/AccountRepository.cs | 7 +- Tzkt.Api/Repositories/BigMapsRepository.cs | 4 +- Tzkt.Api/Repositories/OperationRepository.cs | 4 +- Tzkt.Data/Tzkt.Data.csproj | 2 +- Tzkt.Sync/Tzkt.Sync.csproj | 2 +- 7 files changed, 371 insertions(+), 21 deletions(-) diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 31d3f5aca..b9fdee7a7 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -228,7 +228,7 @@ public async Task> GetKey( /// Bigmap Id /// Either a key hash (`expr123...`) or a plain value (`abcde...`). /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. - /// Sorts bigmaps by specified field. Supported fields: `id` (default). + /// Sorts bigmap updates by specified field. Supported fields: `id` (default). /// Specifies which or how many items should be skipped /// Maximum number of items to return /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index 7eefd39b4..13ee24421 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -392,10 +392,12 @@ public Task> GetRawStorageHistory([Address] string ad /// Get contract bigmaps /// /// - /// Returns all active bigmaps allocated in contract's storage. + /// Returns all active bigmaps allocated in the contract storage. /// /// Contract address - /// Sorts bigmap keys by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. + /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. + /// If you select single field, response will be an array of values in both `.fields` and `.values` modes. + /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. /// Specifies which or how many items should be skipped /// Maximum number of items to return /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string @@ -403,33 +405,58 @@ public Task> GetRawStorageHistory([Address] string ad [HttpGet("{address}/bigmaps")] public async Task>> GetBigMaps( [Address] string address, + SelectParameter select, SortParameter sort, OffsetParameter offset, [Range(0, 10000)] int limit = 100, MichelineFormat micheline = MichelineFormat.Json) { #region validate - if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "updates")) - return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "totalKeys", "activeKeys", "updates")) + return new BadRequest($"{nameof(sort)}", "Sorting by the specified field is not allowed."); #endregion - var acc = await Accounts.Accounts.GetAsync(address); - if (acc is not Services.Cache.RawContract contract) + var acc = await Accounts.GetRawAsync(address); + if (acc is not Services.Cache.RawContract rawContract) return Ok(Enumerable.Empty()); - return Ok(await BigMaps.Get(new AccountParameter { Eq = contract.Id }, true, null, sort, offset, limit, micheline)); + var contract = new AccountParameter { Eq = rawContract.Id }; + + if (select == null) + return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, micheline)); + + if (select.Values != null) + { + if (select.Values.Length == 1) + return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, select.Values[0], micheline)); + else + return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, select.Values, micheline)); + } + else + { + if (select.Fields.Length == 1) + return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, select.Fields[0], micheline)); + else + { + return Ok(new SelectionResponse + { + Cols = select.Fields, + Rows = await BigMaps.Get(contract, true, null, sort, offset, limit, select.Fields, micheline) + }); + } + } } /// /// Get bigmap by name /// /// - /// Returns contract bigmap with the specified name. + /// Returns contract bigmap with the specified name or storage path. /// /// Contract address - /// Bigmap name is the last part in bigmap's storage path. - /// If the path is `ledger` or `assets.ledger`, then the name is `ledger`. - /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `blabla.ledger`, you can specify the full path. + /// Bigmap name is the last piece of the bigmap storage path. + /// For example, if the storage path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `tokens.ledger`, you can specify the full path. /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// [HttpGet("{address}/bigmaps/{name}")] @@ -438,11 +465,329 @@ public async Task> GetBigMapByName( string name, MichelineFormat micheline = MichelineFormat.Json) { - var acc = await Accounts.Accounts.GetAsync(address); + var acc = await Accounts.GetRawAsync(address); if (acc is not Services.Cache.RawContract contract) - return Ok(Enumerable.Empty()); + return Ok(null); return Ok(await BigMaps.Get(contract.Id, name, micheline)); } + + /// + /// Get bigmap keys + /// + /// + /// Returns keys of a contract bigmap with the specified name. + /// + /// Contract address + /// Bigmap name is the last piece of the bigmap storage path. + /// For example, if the storage path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `tokens.ledger`, you can specify the full path. + /// Filters keys by status: `true` - active, `false` - removed. + /// Filters keys by JSON key. Note, this query parameter supports the following format: `?key{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?key.token_id=...`. + /// Filters keys by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. + /// Filters bigmap keys by the last update level. + /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. + /// Sorts bigmap keys by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `updates`. + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps/{name}/keys")] + public async Task>> GetBigMapByNameKeys( + [Address] string address, + string name, + bool? active, + JsonParameter key, + JsonParameter value, + Int32Parameter lastLevel, + SelectParameter select, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + var acc = await Accounts.GetRawAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(Enumerable.Empty()); + + var ptr = await BigMaps.GetPtr(contract.Id, name); + if (ptr == null) + return Ok(Enumerable.Empty()); + + #region validate + if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "updates")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + if (select == null) + return Ok(await BigMaps.GetKeys((int)ptr, active, key, value, lastLevel, sort, offset, limit, micheline)); + + if (select.Values != null) + { + if (select.Values.Length == 1) + return Ok(await BigMaps.GetKeys((int)ptr, active, key, value, lastLevel, sort, offset, limit, select.Values[0], micheline)); + else + return Ok(await BigMaps.GetKeys((int)ptr, active, key, value, lastLevel, sort, offset, limit, select.Values, micheline)); + } + else + { + if (select.Fields.Length == 1) + return Ok(await BigMaps.GetKeys((int)ptr, active, key, value, lastLevel, sort, offset, limit, select.Fields[0], micheline)); + else + { + return Ok(new SelectionResponse + { + Cols = select.Fields, + Rows = await BigMaps.GetKeys((int)ptr, active, key, value, lastLevel, sort, offset, limit, select.Fields, micheline) + }); + } + } + } + + /// + /// Get bigmap key + /// + /// + /// Returns the specified bigmap key. + /// + /// Contract address + /// Bigmap name is the last piece of the bigmap storage path. + /// For example, if the storage path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `tokens.ledger`, you can specify the full path. + /// Either a key hash (`expr123...`) or a plain value (`abcde...`). + /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps/{name}/keys/{key}")] + public async Task> GetKey( + [Address] string address, + string name, + string key, + MichelineFormat micheline = MichelineFormat.Json) + { + var acc = await Accounts.GetRawAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(null); + + var ptr = await BigMaps.GetPtr(contract.Id, name); + if (ptr == null) + return Ok(null); + + try + { + if (Regex.IsMatch(key, @"^expr[0-9A-z]{50}$")) + return Ok(await BigMaps.GetKeyByHash((int)ptr, key, micheline)); + + using var doc = JsonDocument.Parse(WrapKey(key)); + return Ok(await BigMaps.GetKey((int)ptr, doc.RootElement.GetRawText(), micheline)); + } + catch (JsonException) + { + return new BadRequest(nameof(key), "invalid json value"); + } + catch + { + throw; + } + } + + /// + /// Get bigmap key history + /// + /// + /// Returns updates history for the specified bigmap key. + /// + /// Contract address + /// Bigmap name is the last piece of the bigmap storage path. + /// For example, if the storage path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `tokens.ledger`, you can specify the full path. + /// Either a key hash (`expr123...`) or a plain value (`abcde...`). + /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. + /// Sorts bigmap updates by specified field. Supported fields: `id` (default). + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps/{name}/keys/{key}/history")] + public async Task>> GetKeyHistory( + [Address] string address, + string name, + string key, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + var acc = await Accounts.GetRawAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(Enumerable.Empty()); + + var ptr = await BigMaps.GetPtr(contract.Id, name); + if (ptr == null) + return Ok(Enumerable.Empty()); + + #region validate + if (sort != null && !sort.Validate("id")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + try + { + if (Regex.IsMatch(key, @"^expr[0-9A-z]{50}$")) + return Ok(await BigMaps.GetKeyByHashUpdates((int)ptr, key, sort, offset, limit, micheline)); + + using var doc = JsonDocument.Parse(WrapKey(key)); + return Ok(await BigMaps.GetKeyUpdates((int)ptr, doc.RootElement.GetRawText(), sort, offset, limit, micheline)); + } + catch (JsonException) + { + return new BadRequest(nameof(key), "invalid json value"); + } + catch + { + throw; + } + } + + /// + /// Get historical keys + /// + /// + /// Returns a list of bigmap keys at the specific block. + /// + /// Contract address + /// Bigmap name is the last piece of the bigmap storage path. + /// For example, if the storage path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `tokens.ledger`, you can specify the full path. + /// Level of the block at which you want to get bigmap keys + /// Filters keys by status: `true` - active, `false` - removed. + /// Filters keys by JSON key. Note, this query parameter supports the following format: `?key{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?key.token_id=...`. + /// Filters keys by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. + /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. + /// Sorts bigmap keys by specified field. Supported fields: `id` (default). + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps/{name}/historical_keys/{level:int}")] + public async Task>> GetHistoricalKeys( + [Address] string address, + string name, + [Min(0)] int level, + bool? active, + JsonParameter key, + JsonParameter value, + SelectParameter select, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + var acc = await Accounts.GetRawAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(Enumerable.Empty()); + + var ptr = await BigMaps.GetPtr(contract.Id, name); + if (ptr == null) + return Ok(Enumerable.Empty()); + + #region validate + if (sort != null && !sort.Validate("id")) + return new BadRequest(nameof(sort), "Sorting by the specified field is not allowed."); + #endregion + + if (select == null) + return Ok(await BigMaps.GetHistoricalKeys((int)ptr, level, active, key, value, sort, offset, limit, micheline)); + + if (select.Values != null) + { + if (select.Values.Length == 1) + return Ok(await BigMaps.GetHistoricalKeys((int)ptr, level, active, key, value, sort, offset, limit, select.Values[0], micheline)); + else + return Ok(await BigMaps.GetHistoricalKeys((int)ptr, level, active, key, value, sort, offset, limit, select.Values, micheline)); + } + else + { + if (select.Fields.Length == 1) + return Ok(await BigMaps.GetHistoricalKeys((int)ptr, level, active, key, value, sort, offset, limit, select.Fields[0], micheline)); + else + { + return Ok(new SelectionResponse + { + Cols = select.Fields, + Rows = await BigMaps.GetHistoricalKeys((int)ptr, level, active, key, value, sort, offset, limit, select.Fields, micheline) + }); + } + } + } + + /// + /// Get historical key + /// + /// + /// Returns the specified bigmap key at the specific block. + /// + /// Contract address + /// Bigmap name is the last piece of the bigmap storage path. + /// For example, if the storage path is `ledger` or `assets.ledger`, then the name is `ledger`. + /// If there are multiple bigmaps with the same name, for example `assets.ledger` and `tokens.ledger`, you can specify the full path. + /// Level of the block at which you want to get bigmap key + /// Either a key hash (`expr123...`) or a plain value (`abcde...`). + /// Even if the key is complex (an object or an array), you can specify it as is, for example, `/keys/{"address":"tz123","token":123}`. + /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string + /// + [HttpGet("{address}/bigmaps/{name}/historical_keys/{level:int}/{key}")] + public async Task> GetKey( + [Address] string address, + string name, + [Min(0)] int level, + string key, + MichelineFormat micheline = MichelineFormat.Json) + { + var acc = await Accounts.GetRawAsync(address); + if (acc is not Services.Cache.RawContract contract) + return Ok(null); + + var ptr = await BigMaps.GetPtr(contract.Id, name); + if (ptr == null) + return Ok(null); + + try + { + if (Regex.IsMatch(key, @"^expr[0-9A-z]{50}$")) + return Ok(await BigMaps.GetHistoricalKeyByHash((int)ptr, level, key, micheline)); + + using var doc = JsonDocument.Parse(WrapKey(key)); + return Ok(await BigMaps.GetHistoricalKey((int)ptr, level, doc.RootElement.GetRawText(), micheline)); + } + catch (JsonException) + { + return new BadRequest(nameof(key), "invalid json value"); + } + catch + { + throw; + } + } + + string WrapKey(string key) + { + switch (key[0]) + { + case '{': + case '[': + case '"': + case 't' when key == "true": + case 'f' when key == "false": + case 'n' when key == "null": + return key; + default: + return $"\"{key}\""; + } + } } } diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index 2d01dfdcb..d5b1df33a 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -15,7 +15,7 @@ namespace Tzkt.Api.Repositories { public class AccountRepository : DbConnection { - public readonly AccountsCache Accounts; + readonly AccountsCache Accounts; readonly StateCache State; readonly TimeCache Time; readonly OperationRepository Operations; @@ -30,6 +30,11 @@ public AccountRepository(AccountsCache accounts, StateCache state, TimeCache tim Software = software; } + public Task GetRawAsync(string address) + { + return Accounts.GetAsync(address); + } + public async Task Get(string address, bool metadata) { var rawAccount = await Accounts.GetAsync(address); diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 60b30c2a1..db26a45a7 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -76,14 +76,14 @@ public async Task Get(int contractId, string path, MichelineFormat miche AND ""StoragePath"" LIKE @path"; using var db = GetConnection(); - var rows = await db.QueryAsync(sql, new { id = contractId, path = $"%{path}" }); + var rows = await db.QueryAsync(sql, new { id = contractId, path = $"%{path.Replace("_", "\\_")}" }); if (!rows.Any()) return null; var row = rows.FirstOrDefault(x => x.StoragePath == path); return ReadBigMap(row ?? rows.FirstOrDefault(), micheline); } - public async Task GetId(int contractId, string path, MichelineFormat micheline) + public async Task GetPtr(int contractId, string path) { var sql = @" SELECT ""Ptr"", ""StoragePath"" diff --git a/Tzkt.Api/Repositories/OperationRepository.cs b/Tzkt.Api/Repositories/OperationRepository.cs index a785a410b..c55f1d47d 100644 --- a/Tzkt.Api/Repositories/OperationRepository.cs +++ b/Tzkt.Api/Repositories/OperationRepository.cs @@ -4401,9 +4401,9 @@ INNER JOIN ""Blocks"" as b }, Storage = !includeStorage ? null : format switch { - MichelineFormat.Json => row.JsonValue == null ? null : new JsonString(row.JsonValue), + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new JsonString(Micheline.ToJson(row.RawValue)), + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, diff --git a/Tzkt.Data/Tzkt.Data.csproj b/Tzkt.Data/Tzkt.Data.csproj index 5af14358d..8d26cecd0 100644 --- a/Tzkt.Data/Tzkt.Data.csproj +++ b/Tzkt.Data/Tzkt.Data.csproj @@ -6,7 +6,7 @@ - + diff --git a/Tzkt.Sync/Tzkt.Sync.csproj b/Tzkt.Sync/Tzkt.Sync.csproj index c5354b5ac..5c0fd8d02 100644 --- a/Tzkt.Sync/Tzkt.Sync.csproj +++ b/Tzkt.Sync/Tzkt.Sync.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + From 79cbd36269a01bab4c90aa515c31d2555d8e5442 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Mon, 5 Apr 2021 00:22:47 +0300 Subject: [PATCH 17/35] Fix nulls in software metadata --- Tzkt.Api/Repositories/SoftwareRepository.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tzkt.Api/Repositories/SoftwareRepository.cs b/Tzkt.Api/Repositories/SoftwareRepository.cs index 473b9ae8d..fbef9a5e8 100644 --- a/Tzkt.Api/Repositories/SoftwareRepository.cs +++ b/Tzkt.Api/Repositories/SoftwareRepository.cs @@ -47,7 +47,7 @@ public async Task> Get(SortParameter sort, OffsetParameter LastLevel = row.LastLevel, LastTime = Time[row.LastLevel], ShortHash = row.ShortHash, - Metadata = new RawJson(row.Metadata) + Metadata = row.Metadata == null ? null : new RawJson(row.Metadata) }); } @@ -117,7 +117,7 @@ public async Task Get(SortParameter sort, OffsetParameter offset, in break; case "metadata": foreach (var row in rows) - result[j++][i] = new RawJson(row.Metadata); + result[j++][i] = row.Metadata == null ? null : new RawJson(row.Metadata); break; } } @@ -185,7 +185,7 @@ public async Task Get(SortParameter sort, OffsetParameter offset, int break; case "metadata": foreach (var row in rows) - result[j++] = new RawJson(row.Metadata); + result[j++] = row.Metadata == null ? null : new RawJson(row.Metadata); break; } From 9316f0ca6f656083a10319edc2ebba9d29332b38 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Wed, 7 Apr 2021 02:05:01 +0300 Subject: [PATCH 18/35] Add bigmap tags --- Tzkt.Api/Models/BigMaps/BigMap.cs | 21 ++++++++- Tzkt.Api/Repositories/BigMapsRepository.cs | 13 +++++- Tzkt.Api/Utils/Constants/BigMapTags.cs | 9 ++++ .../20210331210536_Bigmaps.Designer.cs | 3 ++ .../Migrations/20210331210536_Bigmaps.cs | 3 +- .../Migrations/TzktContextModelSnapshot.cs | 3 ++ Tzkt.Data/Models/Scripts/BigMap.cs | 15 ++++++- .../Handlers/Proto1/Commits/BigMapCommit.cs | 44 ++++++++++++++++++- 8 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 Tzkt.Api/Utils/Constants/BigMapTags.cs diff --git a/Tzkt.Api/Models/BigMaps/BigMap.cs b/Tzkt.Api/Models/BigMaps/BigMap.cs index 2fbd211b8..a95493cbf 100644 --- a/Tzkt.Api/Models/BigMaps/BigMap.cs +++ b/Tzkt.Api/Models/BigMaps/BigMap.cs @@ -1,4 +1,8 @@ -namespace Tzkt.Api.Models +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Tzkt.Data.Models; + +namespace Tzkt.Api.Models { public class BigMap { @@ -56,5 +60,20 @@ public class BigMap /// Bigmap value type as JSON schema or Micheline, depending on the `micheline` query parameter. /// public object ValueType { get; set; } + + /// + /// List of tags (`token_metadata` - tzip-12, `metadata` - tzip-16, `null` - no tags) + /// + public List Tags => GetTagsList(_Tags); + + [JsonIgnore] + public BigMapTag _Tags { get; set; } + + #region static + public static List GetTagsList(BigMapTag tags) => tags == BigMapTag.None ? null : new(1) + { + tags == BigMapTag.TokenMetadata ? BigMapTags.TokenMetadata : BigMapTags.Metadata + }; + #endregion } } diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index db26a45a7..33ef7733b 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -154,6 +154,7 @@ public async Task Get( case "updates": columns.Add(@"""Updates"""); break; case "keyType": columns.Add(@"""KeyType"""); break; case "valueType": columns.Add(@"""ValueType"""); break; + case "tags": columns.Add(@"""Tags"""); break; } } @@ -233,6 +234,10 @@ public async Task Get( ? new RawJson(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) : new RawJson(Micheline.ToJson(row.ValueType)); break; + case "tags": + foreach (var row in rows) + result[j++][i] = BigMap.GetTagsList((Data.Models.BigMapTag)row.Tags); + break; } } @@ -263,6 +268,7 @@ public async Task Get( case "updates": columns.Add(@"""Updates"""); break; case "keyType": columns.Add(@"""KeyType"""); break; case "valueType": columns.Add(@"""ValueType"""); break; + case "tags": columns.Add(@"""Tags"""); break; } if (columns.Count == 0) @@ -339,6 +345,10 @@ public async Task Get( ? new RawJson(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) : new RawJson(Micheline.ToJson(row.ValueType)); break; + case "tags": + foreach (var row in rows) + result[j++] = BigMap.GetTagsList((Data.Models.BigMapTag)row.Tags); + break; } return result; @@ -936,7 +946,8 @@ BigMap ReadBigMap(dynamic row, MichelineFormat format) : new RawJson(Micheline.ToJson(row.KeyType)), ValueType = (int)format < 2 ? new RawJson(Schema.Create(Micheline.FromBytes(row.ValueType) as MichelinePrim).Humanize()) - : new RawJson(Micheline.ToJson(row.ValueType)) + : new RawJson(Micheline.ToJson(row.ValueType)), + _Tags = (Data.Models.BigMapTag)row.Tags }; } diff --git a/Tzkt.Api/Utils/Constants/BigMapTags.cs b/Tzkt.Api/Utils/Constants/BigMapTags.cs new file mode 100644 index 000000000..6f5870c6c --- /dev/null +++ b/Tzkt.Api/Utils/Constants/BigMapTags.cs @@ -0,0 +1,9 @@ +namespace Tzkt.Api +{ + static class BigMapTags + { + public const string TokenMetadata = "token_metadata"; + + public const string Metadata = "metadata"; + } +} diff --git a/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs b/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs index 59727b2dd..7908781d3 100644 --- a/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs +++ b/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs @@ -630,6 +630,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("StoragePath") .HasColumnType("text"); + b.Property("Tags") + .HasColumnType("integer"); + b.Property("TotalKeys") .HasColumnType("integer"); diff --git a/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs b/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs index 640b12076..6697aa210 100644 --- a/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs +++ b/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs @@ -160,7 +160,8 @@ protected override void Up(MigrationBuilder migrationBuilder) LastLevel = table.Column(type: "integer", nullable: false), TotalKeys = table.Column(type: "integer", nullable: false), ActiveKeys = table.Column(type: "integer", nullable: false), - Updates = table.Column(type: "integer", nullable: false) + Updates = table.Column(type: "integer", nullable: false), + Tags = table.Column(type: "integer", nullable: false) }, constraints: table => { diff --git a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs index 6eb62075b..8682a2c5a 100644 --- a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs +++ b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs @@ -628,6 +628,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("StoragePath") .HasColumnType("text"); + b.Property("Tags") + .HasColumnType("integer"); + b.Property("TotalKeys") .HasColumnType("integer"); diff --git a/Tzkt.Data/Models/Scripts/BigMap.cs b/Tzkt.Data/Models/Scripts/BigMap.cs index e2d649d30..49b1787a3 100644 --- a/Tzkt.Data/Models/Scripts/BigMap.cs +++ b/Tzkt.Data/Models/Scripts/BigMap.cs @@ -1,7 +1,8 @@ -using Microsoft.EntityFrameworkCore; +using System; +using System.Collections.Generic; +using Microsoft.EntityFrameworkCore; using Netezos.Contracts; using Netezos.Encoding; -using System.Collections.Generic; namespace Tzkt.Data.Models { @@ -22,6 +23,8 @@ public class BigMap public int ActiveKeys { get; set; } public int Updates { get; set; } + public BigMapTag Tags { get; set; } + #region schema BigMapSchema _Schema = null; public BigMapSchema Schema @@ -43,6 +46,14 @@ public BigMapSchema Schema #endregion } + [Flags] + public enum BigMapTag + { + None = 0b_0000, + TokenMetadata = 0b_0001, // tzip-12 + Metadata = 0b_0010, // tzip-16 + } + public static class BigMapModel { public static void BuildBigMapModel(this ModelBuilder modelBuilder) diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs index db89eda9c..a70c947a0 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs @@ -132,7 +132,8 @@ await Cache.BigMapKeys.Prefetch(Diffs LastLevel = diff.op.Level, ActiveKeys = 0, TotalKeys = 0, - Updates = 1 + Updates = 1, + Tags = GetTags(bigMapNode) }; Db.BigMaps.Add(allocatedBigMap); Cache.BigMaps.Cache(allocatedBigMap); @@ -236,7 +237,8 @@ await Cache.BigMapKeys.Prefetch(Diffs LastLevel = diff.op.Level, ActiveKeys = keys.Count(), TotalKeys = keys.Count(), - Updates = keys.Count() + 1 + Updates = keys.Count() + 1, + Tags = GetTags(bigMapNode) }; Db.BigMaps.Add(copiedBigMap); @@ -432,6 +434,44 @@ int GetOrigin(CopyDiff copy) : copy.SourcePtr; } + BigMapTag GetTags(TreeView bigmap) + { + var tags = BigMapTag.None; + if (bigmap.Name == "token_metadata") + { + var schema = bigmap.Schema as BigMapSchema; + if (IsTopLevel(bigmap) && + schema.Key is NatSchema && + schema.Value is PairSchema pair && + pair.Left is NatSchema nat && nat.Field == "token_id" && + pair.Right is MapSchema map && map.Field == "token_info" && + map.Key is StringSchema && + map.Value is BytesSchema) + tags |= BigMapTag.TokenMetadata; + } + else if (bigmap.Name == "metadata") + { + var schema = bigmap.Schema as BigMapSchema; + if (IsTopLevel(bigmap) && + schema.Key is StringSchema && + schema.Value is BytesSchema) + tags |= BigMapTag.Metadata; + } + return tags; + } + + bool IsTopLevel(TreeView node) + { + var parent = node.Parent; + while (parent != null) + { + if (parent.Schema is not PairSchema) + return false; + parent = parent.Parent; + } + return true; + } + public virtual async Task Revert(Block block) { if (block.Events.HasFlag(BlockEvents.Bigmaps)) From 95e473cbd5a866c386d7fd84cc9a8a4c0ae6a3d6 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Wed, 7 Apr 2021 21:21:37 +0300 Subject: [PATCH 19/35] Add bigmap updates to transaction and origination API models --- Tzkt.Api/Controllers/OperationsController.cs | 2 +- Tzkt.Api/Models/BigMaps/BigMap.cs | 2 +- Tzkt.Api/Models/BigMaps/BigMapUpdate.cs | 3 +- Tzkt.Api/Models/BigMaps/OpBigMap.cs | 45 ++++++++ .../Models/Operations/OriginationOperation.cs | 5 + .../Models/Operations/TransactionOperation.cs | 5 + Tzkt.Api/Repositories/AccountRepository.cs | 6 +- Tzkt.Api/Repositories/BigMapsRepository.cs | 78 ++++++++++++- Tzkt.Api/Repositories/OperationRepository.cs | 108 +++++++++++++++++- .../Processors/OperationsProcessor.cs | 4 +- Tzkt.Data/Tzkt.Data.csproj | 2 +- Tzkt.Sync/Tzkt.Sync.csproj | 2 +- 12 files changed, 245 insertions(+), 17 deletions(-) create mode 100644 Tzkt.Api/Models/BigMaps/OpBigMap.cs diff --git a/Tzkt.Api/Controllers/OperationsController.cs b/Tzkt.Api/Controllers/OperationsController.cs index 4e08034c3..154dbbecf 100644 --- a/Tzkt.Api/Controllers/OperationsController.cs +++ b/Tzkt.Api/Controllers/OperationsController.cs @@ -1189,7 +1189,7 @@ public async Task>> GetOriginatio #endregion if (select == null) - return Ok(await Operations.GetOriginations(anyof, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, quote)); + return Ok(await Operations.GetOriginations(anyof, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, micheline, quote)); if (select.Values != null) { diff --git a/Tzkt.Api/Models/BigMaps/BigMap.cs b/Tzkt.Api/Models/BigMaps/BigMap.cs index a95493cbf..3083cec57 100644 --- a/Tzkt.Api/Models/BigMaps/BigMap.cs +++ b/Tzkt.Api/Models/BigMaps/BigMap.cs @@ -17,7 +17,7 @@ public class BigMap public Alias Contract { get; set; } /// - /// Path in the JSON storage to the bigmap + /// Path to the bigmap in the contract storage /// public string Path { get; set; } diff --git a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs index 852a8b95e..c396d115e 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs @@ -20,12 +20,13 @@ public class BigMapUpdate public DateTime Timestamp { get; set; } /// - /// Action with the key + /// Action with the key (`add_key`, `update_key`, `remove_key`) /// public string Action { get; set; } /// /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// If the action is `remove_key` it will contain be the last non-null value. /// public object Value { get; set; } } diff --git a/Tzkt.Api/Models/BigMaps/OpBigMap.cs b/Tzkt.Api/Models/BigMaps/OpBigMap.cs new file mode 100644 index 000000000..96bfc285c --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/OpBigMap.cs @@ -0,0 +1,45 @@ +namespace Tzkt.Api.Models +{ + public class OpBigMap + { + /// + /// Bigmap Id + /// + public int Id { get; set; } + + /// + /// Path to the bigmap in the contract storage + /// + public string Path { get; set; } + + /// + /// Action with the bigmap (`allocate`, `add_key`, `update_key`, `remove_key`, `remove`) + /// + public string Action { get; set; } + + /// + /// Affected key. + /// If the action is `remove_key` the key will contain the last non-null value. + /// If the action is `allocate` or `remove` the key will be `null`. + /// + public OpBigMapKey Key { get; set; } + } + + public class OpBigMapKey + { + /// + /// Key hash + /// + public string Hash { get; set; } + + /// + /// Key in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Key { get; set; } + + /// + /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Value { get; set; } + } +} diff --git a/Tzkt.Api/Models/Operations/OriginationOperation.cs b/Tzkt.Api/Models/Operations/OriginationOperation.cs index 7d566922f..7018ecbc9 100644 --- a/Tzkt.Api/Models/Operations/OriginationOperation.cs +++ b/Tzkt.Api/Models/Operations/OriginationOperation.cs @@ -118,6 +118,11 @@ public class OriginationOperation : Operation /// public object Storage { get; set; } + /// + /// List of bigmap updates (aka big_map_diffs) caused by the origination. + /// + public List Bigmaps { get; set; } + /// /// Operation status (`applied` - an operation applied by the node and successfully added to the blockchain, /// `failed` - an operation which failed with some particular error (not enough balance, gas limit, etc), diff --git a/Tzkt.Api/Models/Operations/TransactionOperation.cs b/Tzkt.Api/Models/Operations/TransactionOperation.cs index 3e69d5c31..2a3dc92a2 100644 --- a/Tzkt.Api/Models/Operations/TransactionOperation.cs +++ b/Tzkt.Api/Models/Operations/TransactionOperation.cs @@ -110,6 +110,11 @@ public class TransactionOperation : Operation /// public object Storage { get; set; } + /// + /// List of bigmap updates (aka big_map_diffs) caused by the transaction. + /// + public List Bigmaps { get; set; } + /// /// Operation status (`applied` - an operation applied by the node and successfully added to the blockchain, /// `failed` - an operation which failed with some particular error (not enough balance, gas limit, etc), diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index d5b1df33a..7d10c0b0c 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -2998,7 +2998,7 @@ public async Task> GetOperations( : Task.FromResult(Enumerable.Empty()); var originations = delegat.OriginationsCount > 0 && types.Contains(OpTypes.Origination) - ? Operations.GetOriginations(new AnyOfParameter { Fields = new[] { "initiator", "sender", "contractManager", "contractDelegate", "originatedContract" }, Value = delegat.Id }, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, quote) + ? Operations.GetOriginations(new AnyOfParameter { Fields = new[] { "initiator", "sender", "contractManager", "contractDelegate", "originatedContract" }, Value = delegat.Id }, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, format, quote) : Task.FromResult(Enumerable.Empty()); var transactions = delegat.TransactionsCount > 0 && types.Contains(OpTypes.Transaction) @@ -3065,7 +3065,7 @@ await Task.WhenAll( : Task.FromResult(Enumerable.Empty()); var userOriginations = user.OriginationsCount > 0 && types.Contains(OpTypes.Origination) - ? Operations.GetOriginations(new AnyOfParameter { Fields = new[] { "initiator", "sender", "contractManager", "contractDelegate", "originatedContract" }, Value = user.Id }, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, quote) + ? Operations.GetOriginations(new AnyOfParameter { Fields = new[] { "initiator", "sender", "contractManager", "contractDelegate", "originatedContract" }, Value = user.Id }, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, format, quote) : Task.FromResult(Enumerable.Empty()); var userTransactions = user.TransactionsCount > 0 && types.Contains(OpTypes.Transaction) @@ -3104,7 +3104,7 @@ await Task.WhenAll( : Task.FromResult(Enumerable.Empty()); var contractOriginations = contract.OriginationsCount > 0 && types.Contains(OpTypes.Origination) - ? Operations.GetOriginations(new AnyOfParameter { Fields = new[] { "initiator", "sender", "contractManager", "contractDelegate", "originatedContract" }, Value = contract.Id }, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, quote) + ? Operations.GetOriginations(new AnyOfParameter { Fields = new[] { "initiator", "sender", "contractManager", "contractDelegate", "originatedContract" }, Value = contract.Id }, initiator, sender, contractManager, contractDelegate, originatedContract, level, timestamp, status, sort, offset, limit, format, quote) : Task.FromResult(Enumerable.Empty()); var contractTransactions = contract.TransactionsCount > 0 && types.Contains(OpTypes.Transaction) diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 33ef7733b..fd6de6c46 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; @@ -928,6 +929,73 @@ public async Task> GetKeyByHashUpdates( } #endregion + #region diffs + public static async Task>> GetBigMapUpdates(IDbConnection db, List ops, bool isTxs, MichelineFormat format) + { + if (ops.Count == 0) return null; + + var opCol = isTxs ? "TransactionId" : "OriginationId"; + var fCol = (int)format < 2 ? "Json" : "Raw"; + + var rows = await db.QueryAsync($@" + SELECT ""BigMapPtr"", ""Action"", ""BigMapKeyId"", ""{fCol}Value"", ""{opCol}"" as ""OpId"" + FROM ""BigMapUpdates"" + WHERE ""{opCol}"" = ANY(@ops) + ORDER BY ""Id""", + new { ops }); + + if (!rows.Any()) return null; + + var ptrs = rows + .Select(x => (int)x.BigMapPtr) + .Distinct() + .ToList(); + + var bigmaps = (await db.QueryAsync($@" + SELECT ""Ptr"", ""StoragePath"" + FROM ""BigMaps"" + WHERE ""Ptr"" = ANY(@ptrs)", + new { ptrs })) + .ToDictionary(x => (int)x.Ptr); + + var keyIds = rows + .Where(x => x.BigMapKeyId != null) + .Select(x => (int)x.BigMapKeyId) + .Distinct() + .ToList(); + + var keys = keyIds.Count == 0 ? null : (await db.QueryAsync($@" + SELECT ""Id"", ""KeyHash"", ""{fCol}Key"" + FROM ""BigMapKeys"" + WHERE ""Id"" = ANY(@keyIds)", + new { keyIds })) + .ToDictionary(x => (int)x.Id); + + var res = new Dictionary>(rows.Count()); + foreach (var row in rows) + { + if (!res.TryGetValue((int)row.OpId, out var list)) + { + list = new List(); + res.Add((int)row.OpId, list); + } + list.Add(new OpBigMap + { + Id = row.BigMapPtr, + Path = bigmaps[row.BigMapPtr].StoragePath, + Action = BigMapAction(row.Action), + Key = row.BigMapKeyId == null ? null : new OpBigMapKey + { + Hash = keys[row.BigMapKeyId].KeyHash, + Key = FormatKey(keys[row.BigMapKeyId], format), + Value = FormatValue(row, format), + } + }); + } + return res; + } + #endregion + BigMap ReadBigMap(dynamic row, MichelineFormat format) { return new BigMap @@ -985,12 +1053,12 @@ BigMapUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) Id = row.Id, Level = row.Level, Timestamp = Times[row.Level], - Action = UpdateAction((int)row.Action), + Action = BigMapAction((int)row.Action), Value = FormatValue(row, format) }; } - object FormatKey(dynamic row, MichelineFormat format) => format switch + static object FormatKey(dynamic row, MichelineFormat format) => format switch { MichelineFormat.Json => new RawJson(row.JsonKey), MichelineFormat.JsonString => row.JsonKey, @@ -999,7 +1067,7 @@ BigMapUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) _ => null }; - object FormatValue(dynamic row, MichelineFormat format) => format switch + static object FormatValue(dynamic row, MichelineFormat format) => format switch { MichelineFormat.Json => new RawJson(row.JsonValue), MichelineFormat.JsonString => row.JsonValue, @@ -1008,11 +1076,13 @@ BigMapUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) _ => null }; - string UpdateAction(int action) => action switch + static string BigMapAction(int action) => action switch { + 0 => BigMapActions.Allocate, 1 => BigMapActions.AddKey, 2 => BigMapActions.UpdateKey, 3 => BigMapActions.RemoveKey, + 4 => BigMapActions.Remove, _ => "unknown" }; } diff --git a/Tzkt.Api/Repositories/OperationRepository.cs b/Tzkt.Api/Repositories/OperationRepository.cs index c55f1d47d..34bc1731e 100644 --- a/Tzkt.Api/Repositories/OperationRepository.cs +++ b/Tzkt.Api/Repositories/OperationRepository.cs @@ -3204,6 +3204,10 @@ LEFT JOIN ""Scripts"" as sc using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash }); + #region include bigmaps + var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); + #endregion + return rows.Select(row => { var contract = row.ContractId == null ? null @@ -3248,6 +3252,7 @@ LEFT JOIN ""Scripts"" as sc MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : new OriginatedContract @@ -3283,6 +3288,10 @@ LEFT JOIN ""Scripts"" as sc using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter }); + #region include bigmaps + var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); + #endregion + return rows.Select(row => { var contract = row.ContractId == null ? null @@ -3327,6 +3336,7 @@ LEFT JOIN ""Scripts"" as sc MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : new OriginatedContract @@ -3362,6 +3372,10 @@ LEFT JOIN ""Scripts"" as sc using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter, nonce }); + #region include bigmaps + var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); + #endregion + return rows.Select(row => { var contract = row.ContractId == null ? null @@ -3406,6 +3420,7 @@ LEFT JOIN ""Scripts"" as sc MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : new OriginatedContract @@ -3489,9 +3504,26 @@ public async Task> GetOriginations( SortParameter sort, OffsetParameter offset, int limit, - Symbols quote) + MichelineFormat format, + Symbols quote, + bool includeStorage = false, + bool includeBigmaps = false) { - var sql = new SqlBuilder(@"SELECT o.*, b.""Hash"" FROM ""OriginationOps"" AS o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level""") + var query = includeStorage + ? $@" + SELECT o.*, b.""Hash"", s.""{((int)format < 2 ? "JsonValue" : "RawValue")}"" + FROM ""OriginationOps"" AS o + INNER JOIN ""Blocks"" as b + ON b.""Level"" = o.""Level"" + LEFT JOIN ""Storages"" as s + ON s.""Id"" = o.""StorageId""" + : @" + SELECT o.*, b.""Hash"" + FROM ""OriginationOps"" AS o + INNER JOIN ""Blocks"" as b + ON b.""Level"" = o.""Level"""; + + var sql = new SqlBuilder(query) .Filter(anyof, x => x switch { "initiator" => "InitiatorId", @@ -3523,6 +3555,12 @@ public async Task> GetOriginations( using var db = GetConnection(); var rows = await db.QueryAsync(sql.Query, sql.Params); + #region include bigmaps + var updates = includeBigmaps + ? await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format) + : null; + #endregion + return rows.Select(row => { var contract = row.ContractId == null ? null @@ -3558,6 +3596,15 @@ public async Task> GetOriginations( Address = contract.Address, Kind = contract.KindString }, + Storage = format switch + { + MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), + MichelineFormat.JsonString => row.JsonValue, + MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), + MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), + _ => throw new Exception("Invalid MichelineFormat value") + }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), ContractManager = row.ManagerId != null ? Accounts.GetAlias(row.ManagerId) : null, Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, Quote = Quotes.Get(quote, row.Level) @@ -3624,6 +3671,7 @@ public async Task GetOriginations( columns.Add((int)format < 2 ? @"st.""JsonValue""" : @"st.""RawValue"""); joins.Add(@"LEFT JOIN ""Storages"" as st ON st.""Id"" = o.""StorageId"""); break; + case "bigmaps": columns.Add(@"o.""Id"""); break; case "quote": columns.Add(@"o.""Level"""); break; } } @@ -3766,6 +3814,13 @@ public async Task GetOriginations( _ => throw new Exception("Invalid MichelineFormat value") }; break; + case "bigmaps": + var ids = rows.Select(x => (int)x.Id).ToList(); + var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, false, format); + if (updates != null) + foreach (var row in rows) + result[j++][i] = updates.GetValueOrDefault((int)row.Id); + break; case "status": foreach (var row in rows) result[j++][i] = StatusToString(row.Status); @@ -3859,6 +3914,7 @@ public async Task GetOriginations( columns.Add((int)format < 2 ? @"st.""JsonValue""" : @"st.""RawValue"""); joins.Add(@"LEFT JOIN ""Storages"" as st ON st.""Id"" = o.""StorageId"""); break; + case "bigmaps": columns.Add(@"o.""Id"""); break; case "quote": columns.Add(@"o.""Level"""); break; } @@ -3998,6 +4054,13 @@ public async Task GetOriginations( _ => throw new Exception("Invalid MichelineFormat value") }; break; + case "bigmaps": + var ids = rows.Select(x => (int)x.Id).ToList(); + var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, false, format); + if (updates != null) + foreach (var row in rows) + result[j++] = updates.GetValueOrDefault((int)row.Id); + break; case "status": foreach (var row in rows) result[j++] = StatusToString(row.Status); @@ -4064,6 +4127,10 @@ LEFT JOIN ""Storages"" as s using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash }); + #region include bigmaps + var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); + #endregion + return rows.Select(row => new TransactionOperation { Id = row.Id, @@ -4104,6 +4171,7 @@ LEFT JOIN ""Storages"" as s MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4127,6 +4195,10 @@ LEFT JOIN ""Storages"" as s using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter }); + #region include bigmaps + var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); + #endregion + return rows.Select(row => new TransactionOperation { Id = row.Id, @@ -4167,6 +4239,7 @@ LEFT JOIN ""Storages"" as s MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4190,6 +4263,10 @@ LEFT JOIN ""Storages"" as s using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter, nonce }); + #region include bigmaps + var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); + #endregion + return rows.Select(row => new TransactionOperation { Id = row.Id, @@ -4230,6 +4307,7 @@ LEFT JOIN ""Storages"" as s MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4307,7 +4385,8 @@ public async Task> GetTransactions( int limit, MichelineFormat format, Symbols quote, - bool includeStorage = false) + bool includeStorage = false, + bool includeBigmaps = false) { #region backward compatibility // TODO: remove it asap @@ -4367,6 +4446,12 @@ INNER JOIN ""Blocks"" as b using var db = GetConnection(); var rows = await db.QueryAsync(sql.Query, sql.Params); + #region include bigmaps + var updates = includeBigmaps + ? await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format) + : null; + #endregion + var res = rows.Select(row => new TransactionOperation { Id = row.Id, @@ -4407,6 +4492,7 @@ INNER JOIN ""Blocks"" as b MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), _ => throw new Exception("Invalid MichelineFormat value") }, + Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4547,6 +4633,7 @@ public async Task GetTransactions( }); joins.Add(@"LEFT JOIN ""Storages"" as s ON s.""Id"" = o.""StorageId"""); break; + case "bigmaps": columns.Add(@"o.""Id"""); break; case "status": columns.Add(@"o.""Status"""); break; case "errors": columns.Add(@"o.""Errors"""); break; case "hasInternals": columns.Add(@"o.""InternalOperations"""); break; @@ -4702,6 +4789,13 @@ public async Task GetTransactions( _ => throw new Exception("Invalid MichelineFormat value") }; break; + case "bigmaps": + var ids = rows.Select(x => (int)x.Id).ToList(); + var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, true, format); + if (updates != null) + foreach (var row in rows) + result[j++][i] = updates.GetValueOrDefault((int)row.Id); + break; case "status": foreach (var row in rows) result[j++][i] = StatusToString(row.Status); @@ -4792,6 +4886,7 @@ public async Task GetTransactions( }); joins.Add(@"LEFT JOIN ""Storages"" as s ON s.""Id"" = o.""StorageId"""); break; + case "bigmaps": columns.Add(@"o.""Id"""); break; case "status": columns.Add(@"o.""Status"""); break; case "errors": columns.Add(@"o.""Errors"""); break; case "hasInternals": columns.Add(@"o.""InternalOperations"""); break; @@ -4944,6 +5039,13 @@ public async Task GetTransactions( _ => throw new Exception("Invalid MichelineFormat value") }; break; + case "bigmaps": + var ids = rows.Select(x => (int)x.Id).ToList(); + var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, true, format); + if (updates != null) + foreach (var row in rows) + result[j++] = updates.GetValueOrDefault((int)row.Id); + break; case "status": foreach (var row in rows) result[j++] = StatusToString(row.Status); diff --git a/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs b/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs index 6465f7485..31bef9e67 100644 --- a/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs +++ b/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs @@ -109,11 +109,11 @@ public async Task OnStateChanged() : Task.FromResult(Enumerable.Empty()); var originations = ActiveOps.HasFlag(Operations.Originations) - ? Repo.GetOriginations(null, null, null, null, null, null, level, null, null, null, null, limit, symbols) + ? Repo.GetOriginations(null, null, null, null, null, null, level, null, null, null, null, limit, MichelineFormat.Json, symbols, true, true) : Task.FromResult(Enumerable.Empty()); var transactions = ActiveOps.HasFlag(Operations.Transactions) - ? Repo.GetTransactions(null, null, null, null, null, level, null, null, null, null, null, null, null, null, limit, MichelineFormat.Json, symbols, true) + ? Repo.GetTransactions(null, null, null, null, null, level, null, null, null, null, null, null, null, null, limit, MichelineFormat.Json, symbols, true, true) : Task.FromResult(Enumerable.Empty()); var reveals = ActiveOps.HasFlag(Operations.Reveals) diff --git a/Tzkt.Data/Tzkt.Data.csproj b/Tzkt.Data/Tzkt.Data.csproj index 8d26cecd0..7feff3f82 100644 --- a/Tzkt.Data/Tzkt.Data.csproj +++ b/Tzkt.Data/Tzkt.Data.csproj @@ -6,7 +6,7 @@ - + diff --git a/Tzkt.Sync/Tzkt.Sync.csproj b/Tzkt.Sync/Tzkt.Sync.csproj index 5c0fd8d02..97914dedb 100644 --- a/Tzkt.Sync/Tzkt.Sync.csproj +++ b/Tzkt.Sync/Tzkt.Sync.csproj @@ -11,7 +11,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + From 3ff3431f183096d2fa7c87841be99f03078b36db Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Wed, 7 Apr 2021 23:04:33 +0300 Subject: [PATCH 20/35] Optimize storage including --- Tzkt.Api/Repositories/AccountRepository.cs | 23 ++ Tzkt.Api/Repositories/OperationRepository.cs | 324 ++++++++----------- 2 files changed, 166 insertions(+), 181 deletions(-) diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index 7d10c0b0c..1c19ea4e0 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Data; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Configuration; @@ -2806,6 +2807,28 @@ LEFT JOIN ""OriginationOps"" as o_op }; }); } + + public static async Task> GetStorages(IDbConnection db, List ids, MichelineFormat format) + { + if (ids.Count == 0) return null; + + var rows = await db.QueryAsync($@" + SELECT ""Id"", ""{((int)format < 2 ? "Json" : "Raw")}Value"" + FROM ""Storages"" + WHERE ""Id"" = ANY(@ids)", + new { ids }); + + return rows.Any() + ? rows.ToDictionary(x => (int)x.Id, x => format switch + { + MichelineFormat.Json => new RawJson(x.JsonValue), + MichelineFormat.JsonString => x.JsonValue, + MichelineFormat.Raw => new RawJson(Micheline.ToJson(x.RawValue)), + MichelineFormat.RawString => Micheline.ToJson(x.RawValue), + _ => throw new Exception("Invalid MichelineFormat value") + }) + : null; + } #endregion public async Task> GetRelatedContracts( diff --git a/Tzkt.Api/Repositories/OperationRepository.cs b/Tzkt.Api/Repositories/OperationRepository.cs index 34bc1731e..c88febf54 100644 --- a/Tzkt.Api/Repositories/OperationRepository.cs +++ b/Tzkt.Api/Repositories/OperationRepository.cs @@ -3189,13 +3189,11 @@ public async Task> GetOriginations(string hash var sql = $@" SELECT o.""Id"", o.""Level"", o.""Timestamp"", o.""SenderId"", o.""InitiatorId"", o.""Counter"", o.""BakerFee"", o.""StorageFee"", o.""AllocationFee"", o.""GasLimit"", o.""GasUsed"", o.""StorageLimit"", o.""StorageUsed"", - o.""Status"", o.""Nonce"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", - b.""Hash"", st.""{((int)format < 2 ? "JsonValue" : "RawValue")}"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" + o.""Status"", o.""Nonce"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", o.""StorageId"", + b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" FROM ""OriginationOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as st - ON st.""Id"" = o.""StorageId"" LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"" WHERE o.""OpHash"" = @hash::character(51) @@ -3204,6 +3202,15 @@ LEFT JOIN ""Scripts"" as sc using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash }); + #region include storage + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + #endregion + #region include bigmaps var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); #endregion @@ -3244,14 +3251,7 @@ LEFT JOIN ""Scripts"" as sc ContractDelegate = row.DelegateId != null ? Accounts.GetAlias(row.DelegateId) : null, ContractBalance = row.Balance, Code = (int)format % 2 == 0 ? code : code.ToJson(), - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : @@ -3273,13 +3273,11 @@ public async Task> GetOriginations(string hash var sql = $@" SELECT o.""Id"", o.""Level"", o.""Timestamp"", o.""SenderId"", o.""InitiatorId"", o.""BakerFee"", o.""StorageFee"", o.""AllocationFee"", o.""GasLimit"", o.""GasUsed"", o.""StorageLimit"", o.""StorageUsed"", - o.""Status"", o.""Nonce"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", - b.""Hash"", st.""{((int)format < 2 ? "JsonValue" : "RawValue")}"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" + o.""Status"", o.""Nonce"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", o.""StorageId"", + b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" FROM ""OriginationOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as st - ON st.""Id"" = o.""StorageId"" LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"" WHERE o.""OpHash"" = @hash::character(51) AND o.""Counter"" = @counter @@ -3288,6 +3286,15 @@ LEFT JOIN ""Scripts"" as sc using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter }); + #region include storage + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + #endregion + #region include bigmaps var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); #endregion @@ -3328,14 +3335,7 @@ LEFT JOIN ""Scripts"" as sc ContractDelegate = row.DelegateId != null ? Accounts.GetAlias(row.DelegateId) : null, ContractBalance = row.Balance, Code = (int)format % 2 == 0 ? code : code.ToJson(), - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : @@ -3357,13 +3357,11 @@ public async Task> GetOriginations(string hash var sql = $@" SELECT o.""Id"", o.""Level"", o.""Timestamp"", o.""SenderId"", o.""InitiatorId"", o.""BakerFee"", o.""StorageFee"", o.""AllocationFee"", o.""GasLimit"", o.""GasUsed"", o.""StorageLimit"", o.""StorageUsed"", - o.""Status"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", - b.""Hash"", st.""{((int)format < 2 ? "JsonValue" : "RawValue")}"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" + o.""Status"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", o.""StorageId"", + b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" FROM ""OriginationOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as st - ON st.""Id"" = o.""StorageId"" LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"" WHERE o.""OpHash"" = @hash::character(51) AND o.""Counter"" = @counter AND o.""Nonce"" = @nonce @@ -3372,6 +3370,15 @@ LEFT JOIN ""Scripts"" as sc using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter, nonce }); + #region include storage + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + #endregion + #region include bigmaps var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); #endregion @@ -3412,14 +3419,7 @@ LEFT JOIN ""Scripts"" as sc ContractDelegate = row.DelegateId != null ? Accounts.GetAlias(row.DelegateId) : null, ContractBalance = row.Balance, Code = (int)format % 2 == 0 ? code : code.ToJson(), - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : @@ -3509,21 +3509,11 @@ public async Task> GetOriginations( bool includeStorage = false, bool includeBigmaps = false) { - var query = includeStorage - ? $@" - SELECT o.*, b.""Hash"", s.""{((int)format < 2 ? "JsonValue" : "RawValue")}"" - FROM ""OriginationOps"" AS o - INNER JOIN ""Blocks"" as b - ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as s - ON s.""Id"" = o.""StorageId""" - : @" - SELECT o.*, b.""Hash"" - FROM ""OriginationOps"" AS o - INNER JOIN ""Blocks"" as b - ON b.""Level"" = o.""Level"""; - - var sql = new SqlBuilder(query) + var sql = new SqlBuilder(@" + SELECT o.*, b.""Hash"" + FROM ""OriginationOps"" AS o + INNER JOIN ""Blocks"" as b + ON b.""Level"" = o.""Level""") .Filter(anyof, x => x switch { "initiator" => "InitiatorId", @@ -3555,6 +3545,17 @@ INNER JOIN ""Blocks"" as b using var db = GetConnection(); var rows = await db.QueryAsync(sql.Query, sql.Params); + #region include storage + var storages = includeStorage + ? await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format) + : null; + #endregion + #region include bigmaps var updates = includeBigmaps ? await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format) @@ -3596,14 +3597,7 @@ INNER JOIN ""Blocks"" as b Address = contract.Address, Kind = contract.KindString }, - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), ContractManager = row.ManagerId != null ? Accounts.GetAlias(row.ManagerId) : null, Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, @@ -3667,10 +3661,7 @@ public async Task GetOriginations( columns.Add(@"sc.""CodeSchema"""); joins.Add(@"LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"""); break; - case "storage": - columns.Add((int)format < 2 ? @"st.""JsonValue""" : @"st.""RawValue"""); - joins.Add(@"LEFT JOIN ""Storages"" as st ON st.""Id"" = o.""StorageId"""); - break; + case "storage": columns.Add(@"o.""StorageId"""); break; case "bigmaps": columns.Add(@"o.""Id"""); break; case "quote": columns.Add(@"o.""Level"""); break; } @@ -3804,15 +3795,15 @@ public async Task GetOriginations( } break; case "storage": - foreach (var row in rows) - result[j++][i] = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }; + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + if (storages != null) + foreach (var row in rows) + result[j++][i] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": var ids = rows.Select(x => (int)x.Id).ToList(); @@ -3910,10 +3901,7 @@ public async Task GetOriginations( columns.Add(@"sc.""CodeSchema"""); joins.Add(@"LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"""); break; - case "storage": - columns.Add((int)format < 2 ? @"st.""JsonValue""" : @"st.""RawValue"""); - joins.Add(@"LEFT JOIN ""Storages"" as st ON st.""Id"" = o.""StorageId"""); - break; + case "storage": columns.Add(@"o.""StorageId"""); break; case "bigmaps": columns.Add(@"o.""Id"""); break; case "quote": columns.Add(@"o.""Level"""); break; } @@ -4044,15 +4032,15 @@ public async Task GetOriginations( } break; case "storage": - foreach (var row in rows) - result[j++] = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }; + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + if (storages != null) + foreach (var row in rows) + result[j++] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": var ids = rows.Select(x => (int)x.Id).ToList(); @@ -4115,18 +4103,25 @@ public async Task GetTransactionsCount( public async Task> GetTransactions(string hash, MichelineFormat format, Symbols quote) { var sql = $@" - SELECT o.*, b.""Hash"", s.""{((int)format < 2 ? "JsonValue" : "RawValue")}"" + SELECT o.*, b.""Hash"" FROM ""TransactionOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as s - ON s.""Id"" = o.""StorageId"" WHERE o.""OpHash"" = @hash::character(51) ORDER BY o.""Id"""; using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash }); + #region include storage + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + #endregion + #region include bigmaps var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); #endregion @@ -4163,14 +4158,7 @@ LEFT JOIN ""Storages"" as s _ => throw new Exception("Invalid MichelineFormat value") } }, - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, @@ -4183,18 +4171,25 @@ LEFT JOIN ""Storages"" as s public async Task> GetTransactions(string hash, int counter, MichelineFormat format, Symbols quote) { var sql = $@" - SELECT o.*, b.""Hash"", s.""{((int)format < 2 ? "JsonValue" : "RawValue")}"" + SELECT o.*, b.""Hash"" FROM ""TransactionOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as s - ON s.""Id"" = o.""StorageId"" WHERE o.""OpHash"" = @hash::character(51) AND o.""Counter"" = @counter ORDER BY o.""Id"""; using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter }); + #region include storage + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + #endregion + #region include bigmaps var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); #endregion @@ -4231,14 +4226,7 @@ LEFT JOIN ""Storages"" as s _ => throw new Exception("Invalid MichelineFormat value") } }, - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, @@ -4251,18 +4239,25 @@ LEFT JOIN ""Storages"" as s public async Task> GetTransactions(string hash, int counter, int nonce, MichelineFormat format, Symbols quote) { var sql = $@" - SELECT o.*, b.""Hash"", s.""{((int)format < 2 ? "JsonValue" : "RawValue")}"" + SELECT o.*, b.""Hash"" FROM ""TransactionOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as s - ON s.""Id"" = o.""StorageId"" WHERE o.""OpHash"" = @hash::character(51) AND o.""Counter"" = @counter AND o.""Nonce"" = @nonce LIMIT 1"; using var db = GetConnection(); var rows = await db.QueryAsync(sql, new { hash, counter, nonce }); + #region include storage + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + #endregion + #region include bigmaps var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); #endregion @@ -4299,14 +4294,7 @@ LEFT JOIN ""Storages"" as s _ => throw new Exception("Invalid MichelineFormat value") } }, - Storage = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, @@ -4401,21 +4389,11 @@ public async Task> GetTransactions( } #endregion - var query = includeStorage - ? $@" - SELECT o.*, b.""Hash"", s.""{((int)format < 2 ? "JsonValue" : "RawValue")}"" - FROM ""TransactionOps"" AS o - INNER JOIN ""Blocks"" as b - ON b.""Level"" = o.""Level"" - LEFT JOIN ""Storages"" as s - ON s.""Id"" = o.""StorageId""" - : @" - SELECT o.*, b.""Hash"" - FROM ""TransactionOps"" AS o - INNER JOIN ""Blocks"" as b - ON b.""Level"" = o.""Level"""; - - var sql = new SqlBuilder(query) + var sql = new SqlBuilder(@" + SELECT o.*, b.""Hash"" + FROM ""TransactionOps"" AS o + INNER JOIN ""Blocks"" as b + ON b.""Level"" = o.""Level""") .Filter(anyof, x => x == "sender" ? "SenderId" : x == "target" ? "TargetId" : "InitiatorId") .Filter("InitiatorId", initiator, x => "TargetId") .Filter("SenderId", sender, x => "TargetId") @@ -4446,6 +4424,17 @@ INNER JOIN ""Blocks"" as b using var db = GetConnection(); var rows = await db.QueryAsync(sql.Query, sql.Params); + #region include storage + var storages = includeStorage + ? await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format) + : null; + #endregion + #region include bigmaps var updates = includeBigmaps ? await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format) @@ -4484,14 +4473,7 @@ INNER JOIN ""Blocks"" as b _ => throw new Exception("Invalid MichelineFormat value") } }, - Storage = !includeStorage ? null : format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }, + Storage = row.StorageId == null ? null : storages?[row.StorageId], Bigmaps = updates?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, @@ -4622,17 +4604,7 @@ public async Task GetTransactions( _ => throw new Exception("Invalid MichelineFormat value") }); break; - case "storage": - columns.Add(format switch - { - MichelineFormat.Json => $@"s.""JsonValue""", - MichelineFormat.JsonString => $@"s.""JsonValue""", - MichelineFormat.Raw => $@"s.""RawValue""", - MichelineFormat.RawString => $@"s.""RawValue""", - _ => throw new Exception("Invalid MichelineFormat value") - }); - joins.Add(@"LEFT JOIN ""Storages"" as s ON s.""Id"" = o.""StorageId"""); - break; + case "storage": columns.Add(@"o.""StorageId"""); break; case "bigmaps": columns.Add(@"o.""Id"""); break; case "status": columns.Add(@"o.""Status"""); break; case "errors": columns.Add(@"o.""Errors"""); break; @@ -4779,15 +4751,15 @@ public async Task GetTransactions( }; break; case "storage": - foreach (var row in rows) - result[j++][i] = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }; + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + if (storages != null) + foreach (var row in rows) + result[j++][i] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": var ids = rows.Select(x => (int)x.Id).ToList(); @@ -4875,17 +4847,7 @@ public async Task GetTransactions( _ => throw new Exception("Invalid MichelineFormat value") }); break; - case "storage": - columns.Add(format switch - { - MichelineFormat.Json => $@"s.""JsonValue""", - MichelineFormat.JsonString => $@"s.""JsonValue""", - MichelineFormat.Raw => $@"s.""RawValue""", - MichelineFormat.RawString => $@"s.""RawValue""", - _ => throw new Exception("Invalid MichelineFormat value") - }); - joins.Add(@"LEFT JOIN ""Storages"" as s ON s.""Id"" = o.""StorageId"""); - break; + case "storage": columns.Add(@"o.""StorageId"""); break; case "bigmaps": columns.Add(@"o.""Id"""); break; case "status": columns.Add(@"o.""Status"""); break; case "errors": columns.Add(@"o.""Errors"""); break; @@ -5029,15 +4991,15 @@ public async Task GetTransactions( }; break; case "storage": - foreach (var row in rows) - result[j++] = format switch - { - MichelineFormat.Json => row.JsonValue == null ? null : new RawJson(row.JsonValue), - MichelineFormat.JsonString => row.JsonValue, - MichelineFormat.Raw => row.RawValue == null ? null : new RawJson(Micheline.ToJson(row.RawValue)), - MichelineFormat.RawString => row.RawValue == null ? null : Micheline.ToJson(row.RawValue), - _ => throw new Exception("Invalid MichelineFormat value") - }; + var storages = await AccountRepository.GetStorages(db, + rows.Where(x => x.StorageId != null) + .Select(x => (int)x.StorageId) + .Distinct() + .ToList(), + format); + if (storages != null) + foreach (var row in rows) + result[j++] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": var ids = rows.Select(x => (int)x.Id).ToList(); From 68139ebb6cead387592d0f01988922b3b2ee9662 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 8 Apr 2021 01:45:09 +0300 Subject: [PATCH 21/35] Optimize bigmap diffs including --- Tzkt.Api/Repositories/OperationRepository.cs | 123 +++++++++++++----- .../20210331210536_Bigmaps.Designer.cs | 6 + .../Migrations/20210331210536_Bigmaps.cs | 20 +++ .../Migrations/TzktContextModelSnapshot.cs | 6 + .../Operations/Base/ContractOperation.cs | 8 ++ .../Operations/Base/ManagerOperation.cs | 3 +- .../Models/Operations/OriginationOperation.cs | 3 +- .../Models/Operations/TransactionOperation.cs | 4 +- .../Handlers/Proto1/Commits/BigMapCommit.cs | 17 ++- 9 files changed, 146 insertions(+), 44 deletions(-) create mode 100644 Tzkt.Data/Models/Operations/Base/ContractOperation.cs diff --git a/Tzkt.Api/Repositories/OperationRepository.cs b/Tzkt.Api/Repositories/OperationRepository.cs index c88febf54..3d69eb66e 100644 --- a/Tzkt.Api/Repositories/OperationRepository.cs +++ b/Tzkt.Api/Repositories/OperationRepository.cs @@ -3187,10 +3187,7 @@ public async Task GetOriginationsCount( public async Task> GetOriginations(string hash, MichelineFormat format, Symbols quote) { var sql = $@" - SELECT o.""Id"", o.""Level"", o.""Timestamp"", o.""SenderId"", o.""InitiatorId"", o.""Counter"", - o.""BakerFee"", o.""StorageFee"", o.""AllocationFee"", o.""GasLimit"", o.""GasUsed"", o.""StorageLimit"", o.""StorageUsed"", - o.""Status"", o.""Nonce"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", o.""StorageId"", - b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" + SELECT o.*, b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" FROM ""OriginationOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" @@ -3212,7 +3209,12 @@ LEFT JOIN ""Scripts"" as sc #endregion #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + false, + format); #endregion return rows.Select(row => @@ -3271,10 +3273,7 @@ LEFT JOIN ""Scripts"" as sc public async Task> GetOriginations(string hash, int counter, MichelineFormat format, Symbols quote) { var sql = $@" - SELECT o.""Id"", o.""Level"", o.""Timestamp"", o.""SenderId"", o.""InitiatorId"", - o.""BakerFee"", o.""StorageFee"", o.""AllocationFee"", o.""GasLimit"", o.""GasUsed"", o.""StorageLimit"", o.""StorageUsed"", - o.""Status"", o.""Nonce"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", o.""StorageId"", - b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" + SELECT o.*, b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" FROM ""OriginationOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" @@ -3296,7 +3295,12 @@ LEFT JOIN ""Scripts"" as sc #endregion #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + false, + format); #endregion return rows.Select(row => @@ -3355,10 +3359,7 @@ LEFT JOIN ""Scripts"" as sc public async Task> GetOriginations(string hash, int counter, int nonce, MichelineFormat format, Symbols quote) { var sql = $@" - SELECT o.""Id"", o.""Level"", o.""Timestamp"", o.""SenderId"", o.""InitiatorId"", - o.""BakerFee"", o.""StorageFee"", o.""AllocationFee"", o.""GasLimit"", o.""GasUsed"", o.""StorageLimit"", o.""StorageUsed"", - o.""Status"", o.""ContractId"", o.""DelegateId"", o.""Balance"", o.""ManagerId"", o.""Errors"", o.""StorageId"", - b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" + SELECT o.*, b.""Hash"", sc.""ParameterSchema"", sc.""StorageSchema"", sc.""CodeSchema"" FROM ""OriginationOps"" as o INNER JOIN ""Blocks"" as b ON b.""Level"" = o.""Level"" @@ -3380,7 +3381,12 @@ LEFT JOIN ""Scripts"" as sc #endregion #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + false, + format); #endregion return rows.Select(row => @@ -3558,7 +3564,12 @@ INNER JOIN ""Blocks"" as b #region include bigmaps var updates = includeBigmaps - ? await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), false, format) + ? await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + false, + format) : null; #endregion @@ -3662,7 +3673,10 @@ public async Task GetOriginations( joins.Add(@"LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"""); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": columns.Add(@"o.""Id"""); break; + case "bigmaps": + columns.Add(@"o.""Id"""); + columns.Add(@"o.""BigMapUpdates"""); + break; case "quote": columns.Add(@"o.""Level"""); break; } } @@ -3806,8 +3820,12 @@ public async Task GetOriginations( result[j++][i] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": - var ids = rows.Select(x => (int)x.Id).ToList(); - var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, false, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + false, + format); if (updates != null) foreach (var row in rows) result[j++][i] = updates.GetValueOrDefault((int)row.Id); @@ -3902,7 +3920,10 @@ public async Task GetOriginations( joins.Add(@"LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"""); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": columns.Add(@"o.""Id"""); break; + case "bigmaps": + columns.Add(@"o.""Id"""); + columns.Add(@"o.""BigMapUpdates"""); + break; case "quote": columns.Add(@"o.""Level"""); break; } @@ -4043,8 +4064,12 @@ public async Task GetOriginations( result[j++] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": - var ids = rows.Select(x => (int)x.Id).ToList(); - var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, false, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + false, + format); if (updates != null) foreach (var row in rows) result[j++] = updates.GetValueOrDefault((int)row.Id); @@ -4123,7 +4148,12 @@ INNER JOIN ""Blocks"" as b #endregion #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + true, + format); #endregion return rows.Select(row => new TransactionOperation @@ -4191,7 +4221,12 @@ INNER JOIN ""Blocks"" as b #endregion #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + true, + format); #endregion return rows.Select(row => new TransactionOperation @@ -4259,7 +4294,12 @@ INNER JOIN ""Blocks"" as b #endregion #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + true, + format); #endregion return rows.Select(row => new TransactionOperation @@ -4437,7 +4477,12 @@ INNER JOIN ""Blocks"" as b #region include bigmaps var updates = includeBigmaps - ? await BigMapsRepository.GetBigMapUpdates(db, rows.Select(x => (int)x.Id).ToList(), true, format) + ? await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + true, + format) : null; #endregion @@ -4605,7 +4650,10 @@ public async Task GetTransactions( }); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": columns.Add(@"o.""Id"""); break; + case "bigmaps": + columns.Add(@"o.""Id"""); + columns.Add(@"o.""BigMapUpdates"""); + break; case "status": columns.Add(@"o.""Status"""); break; case "errors": columns.Add(@"o.""Errors"""); break; case "hasInternals": columns.Add(@"o.""InternalOperations"""); break; @@ -4762,8 +4810,12 @@ public async Task GetTransactions( result[j++][i] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": - var ids = rows.Select(x => (int)x.Id).ToList(); - var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, true, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + true, + format); if (updates != null) foreach (var row in rows) result[j++][i] = updates.GetValueOrDefault((int)row.Id); @@ -4848,7 +4900,10 @@ public async Task GetTransactions( }); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": columns.Add(@"o.""Id"""); break; + case "bigmaps": + columns.Add(@"o.""Id"""); + columns.Add(@"o.""BigMapUpdates"""); + break; case "status": columns.Add(@"o.""Status"""); break; case "errors": columns.Add(@"o.""Errors"""); break; case "hasInternals": columns.Add(@"o.""InternalOperations"""); break; @@ -5002,8 +5057,12 @@ public async Task GetTransactions( result[j++] = row.StorageId == null ? null : storages[row.StorageId]; break; case "bigmaps": - var ids = rows.Select(x => (int)x.Id).ToList(); - var updates = await BigMapsRepository.GetBigMapUpdates(db, ids, true, format); + var updates = await BigMapsRepository.GetBigMapUpdates(db, + rows.Where(x => x.BigMapUpdates != null) + .Select(x => (int)x.Id) + .ToList(), + true, + format); if (updates != null) foreach (var row in rows) result[j++] = updates.GetValueOrDefault((int)row.Id); diff --git a/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs b/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs index 7908781d3..eb3694e63 100644 --- a/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs +++ b/Tzkt.Data/Migrations/20210331210536_Bigmaps.Designer.cs @@ -1304,6 +1304,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("Balance") .HasColumnType("bigint"); + b.Property("BigMapUpdates") + .HasColumnType("integer"); + b.Property("ContractId") .HasColumnType("integer"); @@ -1940,6 +1943,9 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("BakerFee") .HasColumnType("bigint"); + b.Property("BigMapUpdates") + .HasColumnType("integer"); + b.Property("Counter") .HasColumnType("integer"); diff --git a/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs b/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs index 6697aa210..fef9ddf56 100644 --- a/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs +++ b/Tzkt.Data/Migrations/20210331210536_Bigmaps.cs @@ -24,6 +24,12 @@ protected override void Up(MigrationBuilder migrationBuilder) name: "Version", table: "Software"); + migrationBuilder.AddColumn( + name: "BigMapUpdates", + table: "TransactionOps", + type: "integer", + nullable: true); + migrationBuilder.AddColumn( name: "InternalDelegations", table: "TransactionOps", @@ -67,6 +73,12 @@ protected override void Up(MigrationBuilder migrationBuilder) type: "jsonb", nullable: true); + migrationBuilder.AddColumn( + name: "BigMapUpdates", + table: "OriginationOps", + type: "integer", + nullable: true); + migrationBuilder.AddColumn( name: "FirstLevel", table: "Cycles", @@ -297,6 +309,10 @@ protected override void Down(MigrationBuilder migrationBuilder) name: "IX_Accounts_Metadata", table: "Accounts"); + migrationBuilder.DropColumn( + name: "BigMapUpdates", + table: "TransactionOps"); + migrationBuilder.DropColumn( name: "InternalDelegations", table: "TransactionOps"); @@ -325,6 +341,10 @@ protected override void Down(MigrationBuilder migrationBuilder) name: "Metadata", table: "Proposals"); + migrationBuilder.DropColumn( + name: "BigMapUpdates", + table: "OriginationOps"); + migrationBuilder.DropColumn( name: "FirstLevel", table: "Cycles"); diff --git a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs index 8682a2c5a..a86ea7b88 100644 --- a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs +++ b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs @@ -1302,6 +1302,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("Balance") .HasColumnType("bigint"); + b.Property("BigMapUpdates") + .HasColumnType("integer"); + b.Property("ContractId") .HasColumnType("integer"); @@ -1938,6 +1941,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("BakerFee") .HasColumnType("bigint"); + b.Property("BigMapUpdates") + .HasColumnType("integer"); + b.Property("Counter") .HasColumnType("integer"); diff --git a/Tzkt.Data/Models/Operations/Base/ContractOperation.cs b/Tzkt.Data/Models/Operations/Base/ContractOperation.cs new file mode 100644 index 000000000..18722f946 --- /dev/null +++ b/Tzkt.Data/Models/Operations/Base/ContractOperation.cs @@ -0,0 +1,8 @@ +namespace Tzkt.Data.Models.Base +{ + public class ContractOperation : InternalOperation + { + public int? StorageId { get; set; } + public int? BigMapUpdates { get; set; } + } +} diff --git a/Tzkt.Data/Models/Operations/Base/ManagerOperation.cs b/Tzkt.Data/Models/Operations/Base/ManagerOperation.cs index f4c081736..cb9c74d89 100644 --- a/Tzkt.Data/Models/Operations/Base/ManagerOperation.cs +++ b/Tzkt.Data/Models/Operations/Base/ManagerOperation.cs @@ -1,5 +1,4 @@ -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; +using System.ComponentModel.DataAnnotations.Schema; namespace Tzkt.Data.Models.Base { diff --git a/Tzkt.Data/Models/Operations/OriginationOperation.cs b/Tzkt.Data/Models/Operations/OriginationOperation.cs index 904efc5f0..e37dafa50 100644 --- a/Tzkt.Data/Models/Operations/OriginationOperation.cs +++ b/Tzkt.Data/Models/Operations/OriginationOperation.cs @@ -4,13 +4,12 @@ namespace Tzkt.Data.Models { - public class OriginationOperation : InternalOperation + public class OriginationOperation : ContractOperation { public int? ManagerId { get; set; } public int? DelegateId { get; set; } public int? ContractId { get; set; } public int? ScriptId { get; set; } - public int? StorageId { get; set; } public long Balance { get; set; } diff --git a/Tzkt.Data/Models/Operations/TransactionOperation.cs b/Tzkt.Data/Models/Operations/TransactionOperation.cs index 5ac0bd3b2..29a26471a 100644 --- a/Tzkt.Data/Models/Operations/TransactionOperation.cs +++ b/Tzkt.Data/Models/Operations/TransactionOperation.cs @@ -4,7 +4,7 @@ namespace Tzkt.Data.Models { - public class TransactionOperation : InternalOperation + public class TransactionOperation : ContractOperation { public int? TargetId { get; set; } public int? ResetDeactivation { get; set; } @@ -15,8 +15,6 @@ public class TransactionOperation : InternalOperation public byte[] RawParameters { get; set; } public string JsonParameters { get; set; } - public int? StorageId { get; set; } - public short? InternalOperations { get; set; } public short? InternalDelegations { get; set; } public short? InternalOriginations { get; set; } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs index a70c947a0..1889e1610 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto1/Commits/BigMapCommit.cs @@ -13,13 +13,13 @@ namespace Tzkt.Sync.Protocols.Proto1 { class BigMapCommit : ProtocolCommit { - readonly List<(BaseOperation op, Contract contract, BigMapDiff diff)> Diffs = new(); + readonly List<(ContractOperation op, Contract contract, BigMapDiff diff)> Diffs = new(); readonly Dictionary TempPtrs = new(7); int TempPtr = 0; public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } - public virtual void Append(BaseOperation op, Contract contract, IEnumerable diffs) + public virtual void Append(ContractOperation op, Contract contract, IEnumerable diffs) { foreach (var diff in diffs) { @@ -119,6 +119,8 @@ await Cache.BigMapKeys.Prefetch(Diffs TransactionId = (diff.op as TransactionOperation)?.Id, OriginationId = (diff.op as OriginationOperation)?.Id }); + diff.op.BigMapUpdates = (diff.op.BigMapUpdates ?? 0) + 1; + var allocatedBigMap = new BigMap { Id = Cache.AppState.NextBigMapId(), @@ -223,6 +225,7 @@ await Cache.BigMapKeys.Prefetch(Diffs TransactionId = (diff.op as TransactionOperation)?.Id, OriginationId = (diff.op as OriginationOperation)?.Id })); + diff.op.BigMapUpdates = (diff.op.BigMapUpdates ?? 0) + keys.Count + 1; var copiedBigMap = new BigMap { @@ -235,9 +238,9 @@ await Cache.BigMapKeys.Prefetch(Diffs Active = true, FirstLevel = diff.op.Level, LastLevel = diff.op.Level, - ActiveKeys = keys.Count(), - TotalKeys = keys.Count(), - Updates = keys.Count() + 1, + ActiveKeys = keys.Count, + TotalKeys = keys.Count, + Updates = keys.Count + 1, Tags = GetTags(bigMapNode) }; @@ -295,6 +298,7 @@ await Cache.BigMapKeys.Prefetch(Diffs TransactionId = (diff.op as TransactionOperation)?.Id, OriginationId = (diff.op as OriginationOperation)?.Id }); + diff.op.BigMapUpdates = (diff.op.BigMapUpdates ?? 0) + 1; #endregion } else if (key.Active) // WTF: edo2net:76611 - key was removed twice @@ -322,6 +326,7 @@ await Cache.BigMapKeys.Prefetch(Diffs TransactionId = (diff.op as TransactionOperation)?.Id, OriginationId = (diff.op as OriginationOperation)?.Id }); + diff.op.BigMapUpdates = (diff.op.BigMapUpdates ?? 0) + 1; #endregion } } @@ -364,6 +369,7 @@ await Cache.BigMapKeys.Prefetch(Diffs TransactionId = (diff.op as TransactionOperation)?.Id, OriginationId = (diff.op as OriginationOperation)?.Id }); + diff.op.BigMapUpdates = (diff.op.BigMapUpdates ?? 0) + 1; #endregion } } @@ -408,6 +414,7 @@ await Cache.BigMapKeys.Prefetch(Diffs TransactionId = (diff.op as TransactionOperation)?.Id, OriginationId = (diff.op as OriginationOperation)?.Id }); + diff.op.BigMapUpdates = (diff.op.BigMapUpdates ?? 0) + 1; var removed = Cache.BigMaps.Get(remove.Ptr); Db.TryAttach(removed); From 9ec2221a78e25917d4a961cb10e4927f96290366 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 8 Apr 2021 02:09:12 +0300 Subject: [PATCH 22/35] Minor refactoring --- Tzkt.Api/Controllers/BigMapsController.cs | 6 ++--- Tzkt.Api/Controllers/ContractsController.cs | 10 ++++---- .../BigMaps/{OpBigMap.cs => BigMapDiff.cs} | 10 ++++---- Tzkt.Api/Models/BigMaps/BigMapKey.cs | 1 + Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs | 1 + .../{BigMapUpdate.cs => BigMapKeyUpdate.cs} | 4 ++-- .../Models/Operations/OriginationOperation.cs | 2 +- .../Models/Operations/TransactionOperation.cs | 2 +- Tzkt.Api/Repositories/BigMapsRepository.cs | 24 +++++++++---------- 9 files changed, 31 insertions(+), 29 deletions(-) rename Tzkt.Api/Models/BigMaps/{OpBigMap.cs => BigMapDiff.cs} (81%) rename Tzkt.Api/Models/BigMaps/{BigMapUpdate.cs => BigMapKeyUpdate.cs} (87%) diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index b9fdee7a7..2dbb8f0c7 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -220,7 +220,7 @@ public async Task> GetKey( } /// - /// Get bigmap key history + /// Get bigmap key updates /// /// /// Returns updates history for the specified bigmap key. @@ -233,8 +233,8 @@ public async Task> GetKey( /// Maximum number of items to return /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// - [HttpGet("{id:int}/keys/{key}/history")] - public async Task>> GetKeyHistory( + [HttpGet("{id:int}/keys/{key}/updates")] + public async Task>> GetKeyUpdates( [Min(0)] int id, string key, SortParameter sort, diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index 13ee24421..39508f6a0 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -594,7 +594,7 @@ public async Task> GetKey( } /// - /// Get bigmap key history + /// Get bigmap key updates /// /// /// Returns updates history for the specified bigmap key. @@ -610,8 +610,8 @@ public async Task> GetKey( /// Maximum number of items to return /// Format of the key value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// - [HttpGet("{address}/bigmaps/{name}/keys/{key}/history")] - public async Task>> GetKeyHistory( + [HttpGet("{address}/bigmaps/{name}/keys/{key}/updates")] + public async Task>> GetKeyUpdates( [Address] string address, string name, string key, @@ -622,11 +622,11 @@ public async Task>> GetKeyHistory( { var acc = await Accounts.GetRawAsync(address); if (acc is not Services.Cache.RawContract contract) - return Ok(Enumerable.Empty()); + return Ok(Enumerable.Empty()); var ptr = await BigMaps.GetPtr(contract.Id, name); if (ptr == null) - return Ok(Enumerable.Empty()); + return Ok(Enumerable.Empty()); #region validate if (sort != null && !sort.Validate("id")) diff --git a/Tzkt.Api/Models/BigMaps/OpBigMap.cs b/Tzkt.Api/Models/BigMaps/BigMapDiff.cs similarity index 81% rename from Tzkt.Api/Models/BigMaps/OpBigMap.cs rename to Tzkt.Api/Models/BigMaps/BigMapDiff.cs index 96bfc285c..465e4eb84 100644 --- a/Tzkt.Api/Models/BigMaps/OpBigMap.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapDiff.cs @@ -1,11 +1,11 @@ namespace Tzkt.Api.Models { - public class OpBigMap + public class BigMapDiff { /// /// Bigmap Id /// - public int Id { get; set; } + public int Bigmap { get; set; } /// /// Path to the bigmap in the contract storage @@ -19,13 +19,12 @@ public class OpBigMap /// /// Affected key. - /// If the action is `remove_key` the key will contain the last non-null value. /// If the action is `allocate` or `remove` the key will be `null`. /// - public OpBigMapKey Key { get; set; } + public BigMapDiffKey Key { get; set; } } - public class OpBigMapKey + public class BigMapDiffKey { /// /// Key hash @@ -39,6 +38,7 @@ public class OpBigMapKey /// /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// Note, if the action is `remove_key` it will contain the last non-null value. /// public object Value { get; set; } } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKey.cs b/Tzkt.Api/Models/BigMaps/BigMapKey.cs index ef730a802..e0c3d2adb 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapKey.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKey.cs @@ -24,6 +24,7 @@ public class BigMapKey /// /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// Note, if the key is inactive (removed) it will contain the last non-null value. /// public object Value { get; set; } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs b/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs index ffef5f38a..d9871feb9 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs @@ -24,6 +24,7 @@ public class BigMapKeyShort /// /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// Note, if the key is inactive (removed) it will contain the last non-null value. /// public object Value { get; set; } } diff --git a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs b/Tzkt.Api/Models/BigMaps/BigMapKeyUpdate.cs similarity index 87% rename from Tzkt.Api/Models/BigMaps/BigMapUpdate.cs rename to Tzkt.Api/Models/BigMaps/BigMapKeyUpdate.cs index c396d115e..d57c26372 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKeyUpdate.cs @@ -2,7 +2,7 @@ namespace Tzkt.Api.Models { - public class BigMapUpdate + public class BigMapKeyUpdate { /// /// Internal Id, can be used for pagination @@ -26,7 +26,7 @@ public class BigMapUpdate /// /// Value in JSON or Micheline format, depending on the `micheline` query parameter. - /// If the action is `remove_key` it will contain be the last non-null value. + /// Note, if the action is `remove_key` it will contain the last non-null value. /// public object Value { get; set; } } diff --git a/Tzkt.Api/Models/Operations/OriginationOperation.cs b/Tzkt.Api/Models/Operations/OriginationOperation.cs index 7018ecbc9..56930bd7e 100644 --- a/Tzkt.Api/Models/Operations/OriginationOperation.cs +++ b/Tzkt.Api/Models/Operations/OriginationOperation.cs @@ -121,7 +121,7 @@ public class OriginationOperation : Operation /// /// List of bigmap updates (aka big_map_diffs) caused by the origination. /// - public List Bigmaps { get; set; } + public List Bigmaps { get; set; } /// /// Operation status (`applied` - an operation applied by the node and successfully added to the blockchain, diff --git a/Tzkt.Api/Models/Operations/TransactionOperation.cs b/Tzkt.Api/Models/Operations/TransactionOperation.cs index 2a3dc92a2..694b315e1 100644 --- a/Tzkt.Api/Models/Operations/TransactionOperation.cs +++ b/Tzkt.Api/Models/Operations/TransactionOperation.cs @@ -113,7 +113,7 @@ public class TransactionOperation : Operation /// /// List of bigmap updates (aka big_map_diffs) caused by the transaction. /// - public List Bigmaps { get; set; } + public List Bigmaps { get; set; } /// /// Operation status (`applied` - an operation applied by the node and successfully added to the blockchain, diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index fd6de6c46..19d9f72c6 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -874,7 +874,7 @@ INNER JOIN ""BigMapKeys"" as k #endregion #region bigmap key updates - public async Task> GetKeyUpdates( + public async Task> GetKeyUpdates( int ptr, string key, SortParameter sort, @@ -898,10 +898,10 @@ public async Task> GetKeyUpdates( .Take(sort, offset, limit, x => ("Id", "Id")); var rows = await db.QueryAsync(sql.Query, sql.Params); - return rows.Select(row => (BigMapUpdate)ReadBigMapUpdate(row, micheline)); + return rows.Select(row => (BigMapKeyUpdate)ReadBigMapUpdate(row, micheline)); } - public async Task> GetKeyByHashUpdates( + public async Task> GetKeyByHashUpdates( int ptr, string hash, SortParameter sort, @@ -925,12 +925,12 @@ public async Task> GetKeyByHashUpdates( .Take(sort, offset, limit, x => ("Id", "Id")); var rows = await db.QueryAsync(sql.Query, sql.Params); - return rows.Select(row => (BigMapUpdate)ReadBigMapUpdate(row, micheline)); + return rows.Select(row => (BigMapKeyUpdate)ReadBigMapUpdate(row, micheline)); } #endregion #region diffs - public static async Task>> GetBigMapUpdates(IDbConnection db, List ops, bool isTxs, MichelineFormat format) + public static async Task>> GetBigMapUpdates(IDbConnection db, List ops, bool isTxs, MichelineFormat format) { if (ops.Count == 0) return null; @@ -971,20 +971,20 @@ public static async Task>> GetBigMapUpdates(IDbCo new { keyIds })) .ToDictionary(x => (int)x.Id); - var res = new Dictionary>(rows.Count()); + var res = new Dictionary>(rows.Count()); foreach (var row in rows) { if (!res.TryGetValue((int)row.OpId, out var list)) { - list = new List(); + list = new List(); res.Add((int)row.OpId, list); } - list.Add(new OpBigMap + list.Add(new BigMapDiff { - Id = row.BigMapPtr, + Bigmap = row.BigMapPtr, Path = bigmaps[row.BigMapPtr].StoragePath, Action = BigMapAction(row.Action), - Key = row.BigMapKeyId == null ? null : new OpBigMapKey + Key = row.BigMapKeyId == null ? null : new BigMapDiffKey { Hash = keys[row.BigMapKeyId].KeyHash, Key = FormatKey(keys[row.BigMapKeyId], format), @@ -1046,9 +1046,9 @@ BigMapKeyShort ReadBigMapKeyShort(dynamic row, MichelineFormat format) }; } - BigMapUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) + BigMapKeyUpdate ReadBigMapUpdate(dynamic row, MichelineFormat format) { - return new BigMapUpdate + return new BigMapKeyUpdate { Id = row.Id, Level = row.Level, From 20edaaa036f66709e05339502e563f1507f32ef3 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Mon, 12 Apr 2021 01:34:46 +0300 Subject: [PATCH 23/35] Add bigmap updates API endpoint and minor refactoring --- Tzkt.Api/Controllers/BigMapsController.cs | 63 +++++- Tzkt.Api/Controllers/ContractsController.cs | 20 +- .../ModelBindingContextExtension.cs | 149 +++++++++++++ Tzkt.Api/Models/BigMaps/BigMap.cs | 18 +- Tzkt.Api/Models/BigMaps/BigMapDiff.cs | 21 +- Tzkt.Api/Models/BigMaps/BigMapKey.cs | 2 +- .../Models/BigMaps/BigMapKeyHistorical.cs | 31 +++ Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs | 12 +- Tzkt.Api/Models/BigMaps/BigMapUpdate.cs | 53 +++++ .../Models/Operations/OriginationOperation.cs | 2 +- .../Models/Operations/TransactionOperation.cs | 2 +- Tzkt.Api/Parameters/BigMapActionParameter.cs | 46 ++++ Tzkt.Api/Parameters/BigMapTagsParameter.cs | 37 ++++ .../Parameters/Binders/BigMapActionBinder.cs | 45 ++++ .../Parameters/Binders/BigMapTagsBinder.cs | 41 ++++ Tzkt.Api/Repositories/BigMapsRepository.cs | 204 ++++++++++++++++-- Tzkt.Api/Repositories/OperationRepository.cs | 92 ++++---- Tzkt.Api/Utils/SqlBuilder.cs | 41 ++++ 18 files changed, 756 insertions(+), 123 deletions(-) create mode 100644 Tzkt.Api/Models/BigMaps/BigMapKeyHistorical.cs create mode 100644 Tzkt.Api/Models/BigMaps/BigMapUpdate.cs create mode 100644 Tzkt.Api/Parameters/BigMapActionParameter.cs create mode 100644 Tzkt.Api/Parameters/BigMapTagsParameter.cs create mode 100644 Tzkt.Api/Parameters/Binders/BigMapActionBinder.cs create mode 100644 Tzkt.Api/Parameters/Binders/BigMapTagsBinder.cs diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 2dbb8f0c7..fa56076d5 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -42,10 +42,12 @@ public Task GetBigMapsCount() /// Returns a list of bigmaps. /// /// Filters bigmaps by smart contract address. + /// Filters bigmaps path in the contract storage. + /// Filters bigmaps tags (`token_metadata` - tzip-12, `metadata` - tzip-16). /// Filters bigmaps by status: `true` - active, `false` - removed. /// Filters bigmaps by the last update level. /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. - /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. + /// Sorts bigmaps by specified field. Supported fields: `id` (default), `ptr`, `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. /// Specifies which or how many items should be skipped /// Maximum number of items to return /// Format of the bigmap key and value type: `0` - JSON, `2` - Micheline @@ -53,6 +55,8 @@ public Task GetBigMapsCount() [HttpGet] public async Task>> GetBigMaps( AccountParameter contract, + StringParameter path, + BigMapTagsParameter tags, bool? active, Int32Parameter lastLevel, SelectParameter select, @@ -62,35 +66,76 @@ public async Task>> GetBigMaps( MichelineFormat micheline = MichelineFormat.Json) { #region validate - if (sort != null && !sort.Validate("id", "firstLevel", "lastLevel", "totalKeys", "activeKeys", "updates")) + if (sort != null && !sort.Validate("id", "ptr", "firstLevel", "lastLevel", "totalKeys", "activeKeys", "updates")) return new BadRequest($"{nameof(sort)}", "Sorting by the specified field is not allowed."); #endregion if (select == null) - return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, micheline)); + return Ok(await BigMaps.Get(contract, path, tags, active, lastLevel, sort, offset, limit, micheline)); if (select.Values != null) { if (select.Values.Length == 1) - return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Values[0], micheline)); + return Ok(await BigMaps.Get(contract, path, tags, active, lastLevel, sort, offset, limit, select.Values[0], micheline)); else - return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Values, micheline)); + return Ok(await BigMaps.Get(contract, path, tags, active, lastLevel, sort, offset, limit, select.Values, micheline)); } else { if (select.Fields.Length == 1) - return Ok(await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Fields[0], micheline)); + return Ok(await BigMaps.Get(contract, path, tags, active, lastLevel, sort, offset, limit, select.Fields[0], micheline)); else { return Ok(new SelectionResponse { Cols = select.Fields, - Rows = await BigMaps.Get(contract, active, lastLevel, sort, offset, limit, select.Fields, micheline) + Rows = await BigMaps.Get(contract, path, tags, active, lastLevel, sort, offset, limit, select.Fields, micheline) }); } } } + /// + /// Get bigmap updates + /// + /// + /// Returns a list of all bigmap updates. + /// + /// Filters updates by bigmap ptr + /// Filters updates by bigmap path + /// Filters updates by bigmap contract + /// Filters updates by bigmap tags + /// Filters updates by action + /// Filters updates by level + /// Sorts bigmaps by specified field. Supported fields: `id` (default), `ptr`, `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. + /// Specifies which or how many items should be skipped + /// Maximum number of items to return + /// Format of the bigmap key and value type: `0` - JSON, `2` - Micheline + /// + [HttpGet("updates")] + public async Task>> GetBigMapUpdates( + Int32Parameter bigmap, + StringParameter path, + AccountParameter contract, + BigMapTagsParameter tags, + BigMapActionParameter action, + Int32Parameter level, + SortParameter sort, + OffsetParameter offset, + [Range(0, 10000)] int limit = 100, + MichelineFormat micheline = MichelineFormat.Json) + { + #region validate + if (sort != null && !sort.Validate("id", "level")) + return new BadRequest($"{nameof(sort)}", "Sorting by the specified field is not allowed."); + #endregion + + if (path == null && contract == null && tags == null) + return Ok(await BigMaps.GetUpdates(bigmap, action, level, sort, offset, limit, micheline)); + + return Ok(await BigMaps.GetUpdates(bigmap, path, contract, action, tags, level, sort, offset, limit, micheline)); + } + /// /// Get bigmap by Id /// @@ -285,7 +330,7 @@ public async Task>> GetKeyUpdates( /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// [HttpGet("{id:int}/historical_keys/{level:int}")] - public async Task>> GetHistoricalKeys( + public async Task>> GetHistoricalKeys( [Min(0)] int id, [Min(0)] int level, bool? active, @@ -340,7 +385,7 @@ public async Task>> GetHistoricalKeys( /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// [HttpGet("{id:int}/historical_keys/{level:int}/{key}")] - public async Task> GetKey( + public async Task> GetKey( [Min(0)] int id, [Min(0)] int level, string key, diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index 39508f6a0..a88b9d779 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -395,6 +395,7 @@ public Task> GetRawStorageHistory([Address] string ad /// Returns all active bigmaps allocated in the contract storage. /// /// Contract address + /// Filters bigmaps tags (`token_metadata` - tzip-12, `metadata` - tzip-16). /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. /// If you select single field, response will be an array of values in both `.fields` and `.values` modes. /// Sorts bigmaps by specified field. Supported fields: `id` (default), `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. @@ -405,6 +406,7 @@ public Task> GetRawStorageHistory([Address] string ad [HttpGet("{address}/bigmaps")] public async Task>> GetBigMaps( [Address] string address, + BigMapTagsParameter tags, SelectParameter select, SortParameter sort, OffsetParameter offset, @@ -423,25 +425,25 @@ public async Task>> GetBigMaps( var contract = new AccountParameter { Eq = rawContract.Id }; if (select == null) - return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, micheline)); + return Ok(await BigMaps.Get(contract, null, tags, true, null, sort, offset, limit, micheline)); if (select.Values != null) { if (select.Values.Length == 1) - return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, select.Values[0], micheline)); + return Ok(await BigMaps.Get(contract, null, tags, true, null, sort, offset, limit, select.Values[0], micheline)); else - return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, select.Values, micheline)); + return Ok(await BigMaps.Get(contract, null, tags, true, null, sort, offset, limit, select.Values, micheline)); } else { if (select.Fields.Length == 1) - return Ok(await BigMaps.Get(contract, true, null, sort, offset, limit, select.Fields[0], micheline)); + return Ok(await BigMaps.Get(contract, null, tags, true, null, sort, offset, limit, select.Fields[0], micheline)); else { return Ok(new SelectionResponse { Cols = select.Fields, - Rows = await BigMaps.Get(contract, true, null, sort, offset, limit, select.Fields, micheline) + Rows = await BigMaps.Get(contract, null, tags, true, null, sort, offset, limit, select.Fields, micheline) }); } } @@ -674,7 +676,7 @@ public async Task>> GetKeyUpdates( /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// [HttpGet("{address}/bigmaps/{name}/historical_keys/{level:int}")] - public async Task>> GetHistoricalKeys( + public async Task>> GetHistoricalKeys( [Address] string address, string name, [Min(0)] int level, @@ -689,11 +691,11 @@ public async Task>> GetHistoricalKeys( { var acc = await Accounts.GetRawAsync(address); if (acc is not Services.Cache.RawContract contract) - return Ok(Enumerable.Empty()); + return Ok(Enumerable.Empty()); var ptr = await BigMaps.GetPtr(contract.Id, name); if (ptr == null) - return Ok(Enumerable.Empty()); + return Ok(Enumerable.Empty()); #region validate if (sort != null && !sort.Validate("id")) @@ -741,7 +743,7 @@ public async Task>> GetHistoricalKeys( /// Format of the bigmap key and value: `0` - JSON, `1` - JSON string, `2` - Micheline, `3` - Micheline string /// [HttpGet("{address}/bigmaps/{name}/historical_keys/{level:int}/{key}")] - public async Task> GetKey( + public async Task> GetKey( [Address] string address, string name, [Min(0)] int level, diff --git a/Tzkt.Api/Extensions/ModelBindingContextExtension.cs b/Tzkt.Api/Extensions/ModelBindingContextExtension.cs index 89ab15e2d..781aca9ac 100644 --- a/Tzkt.Api/Extensions/ModelBindingContextExtension.cs +++ b/Tzkt.Api/Extensions/ModelBindingContextExtension.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Collections.Generic; using System.Text.RegularExpressions; using Microsoft.AspNetCore.Mvc.ModelBinding; @@ -503,6 +504,154 @@ public static bool TryGetContractKindList(this ModelBindingContext bindingContex return true; } + public static bool TryGetBigMapAction(this ModelBindingContext bindingContext, string name, ref bool hasValue, out int? result) + { + result = null; + var valueObject = (bindingContext.ValueProvider as CompositeValueProvider)? + .FirstOrDefault(x => x is QueryStringValueProvider)? + .GetValue(name) ?? ValueProviderResult.None; + + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + switch (valueObject.FirstValue) + { + case BigMapActions.Allocate: + hasValue = true; + result = (int)Data.Models.BigMapAction.Allocate; + break; + case BigMapActions.AddKey: + hasValue = true; + result = (int)Data.Models.BigMapAction.AddKey; + break; + case BigMapActions.UpdateKey: + hasValue = true; + result = (int)Data.Models.BigMapAction.UpdateKey; + break; + case BigMapActions.RemoveKey: + hasValue = true; + result = (int)Data.Models.BigMapAction.RemoveKey; + break; + case BigMapActions.Remove: + hasValue = true; + result = (int)Data.Models.BigMapAction.Remove; + break; + default: + bindingContext.ModelState.TryAddModelError(name, "Invalid bigmap action."); + return false; + } + } + } + + return true; + } + + public static bool TryGetBigMapActionList(this ModelBindingContext bindingContext, string name, ref bool hasValue, out List result) + { + result = null; + var valueObject = (bindingContext.ValueProvider as CompositeValueProvider)? + .FirstOrDefault(x => x is QueryStringValueProvider)? + .GetValue(name) ?? ValueProviderResult.None; + + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + var rawValues = valueObject.FirstValue.Split(',', StringSplitOptions.RemoveEmptyEntries); + + if (rawValues.Length == 0) + { + bindingContext.ModelState.TryAddModelError(name, "List should contain at least one item."); + return false; + } + + hasValue = true; + result = new List(rawValues.Length); + + foreach (var rawValue in rawValues) + { + switch (rawValue) + { + case BigMapActions.Allocate: + hasValue = true; + result.Add((int)Data.Models.BigMapAction.Allocate); + break; + case BigMapActions.AddKey: + hasValue = true; + result.Add((int)Data.Models.BigMapAction.AddKey); + break; + case BigMapActions.UpdateKey: + hasValue = true; + result.Add((int)Data.Models.BigMapAction.UpdateKey); + break; + case BigMapActions.RemoveKey: + hasValue = true; + result.Add((int)Data.Models.BigMapAction.RemoveKey); + break; + case BigMapActions.Remove: + hasValue = true; + result.Add((int)Data.Models.BigMapAction.Remove); + break; + default: + bindingContext.ModelState.TryAddModelError(name, "List contains invalid bigmap action."); + return false; + } + } + } + } + + return true; + } + + public static bool TryGetBigMapTags(this ModelBindingContext bindingContext, string name, ref bool hasValue, out int? result) + { + result = null; + var valueObject = bindingContext.ValueProvider.GetValue(name); + + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + var rawValues = valueObject.FirstValue.Split(',', StringSplitOptions.RemoveEmptyEntries); + + if (rawValues.Length == 0) + { + bindingContext.ModelState.TryAddModelError(name, "List should contain at least one item."); + return false; + } + + hasValue = true; + result = (int)Data.Models.BigMapTag.None; + + foreach (var rawValue in rawValues) + { + switch (rawValue) + { + case BigMapTags.Metadata: + hasValue = true; + result |= (int)Data.Models.BigMapTag.Metadata; + break; + case BigMapTags.TokenMetadata: + hasValue = true; + result |= (int)Data.Models.BigMapTag.TokenMetadata; + break; + default: + bindingContext.ModelState.TryAddModelError(name, "Invalid bigmap tags."); + return false; + } + } + + + } + } + + return true; + } + public static bool TryGetVoterStatus(this ModelBindingContext bindingContext, string name, ref bool hasValue, out int? result) { result = null; diff --git a/Tzkt.Api/Models/BigMaps/BigMap.cs b/Tzkt.Api/Models/BigMaps/BigMap.cs index 3083cec57..d37b1f61a 100644 --- a/Tzkt.Api/Models/BigMaps/BigMap.cs +++ b/Tzkt.Api/Models/BigMaps/BigMap.cs @@ -7,9 +7,9 @@ namespace Tzkt.Api.Models public class BigMap { /// - /// Bigmap Id + /// Bigmap pointer /// - public int Id { get; set; } + public int Ptr { get; set; } /// /// Smart contract in which's storage the bigmap is allocated @@ -22,7 +22,12 @@ public class BigMap public string Path { get; set; } /// - /// Bigmap status: `true` - active, `false` - removed + /// List of tags (`token_metadata` - tzip-12, `metadata` - tzip-16, `null` - no tags) + /// + public List Tags => GetTagsList(_Tags); + + /// + /// Bigmap status (`true` - active, `false` - removed) /// public bool Active { get; set; } @@ -42,7 +47,7 @@ public class BigMap public int TotalKeys { get; set; } /// - /// Total number of active (current) keys + /// Total number of currently active keys /// public int ActiveKeys { get; set; } @@ -61,11 +66,6 @@ public class BigMap /// public object ValueType { get; set; } - /// - /// List of tags (`token_metadata` - tzip-12, `metadata` - tzip-16, `null` - no tags) - /// - public List Tags => GetTagsList(_Tags); - [JsonIgnore] public BigMapTag _Tags { get; set; } diff --git a/Tzkt.Api/Models/BigMaps/BigMapDiff.cs b/Tzkt.Api/Models/BigMaps/BigMapDiff.cs index 465e4eb84..255b0f749 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapDiff.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapDiff.cs @@ -21,25 +21,6 @@ public class BigMapDiff /// Affected key. /// If the action is `allocate` or `remove` the key will be `null`. /// - public BigMapDiffKey Key { get; set; } - } - - public class BigMapDiffKey - { - /// - /// Key hash - /// - public string Hash { get; set; } - - /// - /// Key in JSON or Micheline format, depending on the `micheline` query parameter. - /// - public object Key { get; set; } - - /// - /// Value in JSON or Micheline format, depending on the `micheline` query parameter. - /// Note, if the action is `remove_key` it will contain the last non-null value. - /// - public object Value { get; set; } + public BigMapKeyShort Content { get; set; } } } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKey.cs b/Tzkt.Api/Models/BigMaps/BigMapKey.cs index e0c3d2adb..23bb80410 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapKey.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKey.cs @@ -8,7 +8,7 @@ public class BigMapKey public int Id { get; set; } /// - /// Bigmap key status: `true` - active, `false` - removed + /// Bigmap key status (`true` - active, `false` - removed) /// public bool Active { get; set; } diff --git a/Tzkt.Api/Models/BigMaps/BigMapKeyHistorical.cs b/Tzkt.Api/Models/BigMaps/BigMapKeyHistorical.cs new file mode 100644 index 000000000..e3bc8b446 --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/BigMapKeyHistorical.cs @@ -0,0 +1,31 @@ +namespace Tzkt.Api.Models +{ + public class BigMapKeyHistorical + { + /// + /// Internal Id, can be used for pagination + /// + public int Id { get; set; } + + /// + /// Bigmap key status (`true` - active, `false` - removed) + /// + public bool Active { get; set; } + + /// + /// Key hash + /// + public string Hash { get; set; } + + /// + /// Key in JSON or Micheline format, depending on the `micheline` query parameter. + /// + public object Key { get; set; } + + /// + /// Value in JSON or Micheline format, depending on the `micheline` query parameter. + /// Note, if the key is inactive (removed) it will contain the last non-null value. + /// + public object Value { get; set; } + } +} diff --git a/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs b/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs index d9871feb9..9f3af3ad5 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapKeyShort.cs @@ -2,16 +2,6 @@ { public class BigMapKeyShort { - /// - /// Internal Id, can be used for pagination - /// - public int Id { get; set; } - - /// - /// Bigmap key status: `true` - active, `false` - removed - /// - public bool Active { get; set; } - /// /// Key hash /// @@ -24,7 +14,7 @@ public class BigMapKeyShort /// /// Value in JSON or Micheline format, depending on the `micheline` query parameter. - /// Note, if the key is inactive (removed) it will contain the last non-null value. + /// Note, if the action is `remove_key` it will contain the last non-null value. /// public object Value { get; set; } } diff --git a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs new file mode 100644 index 000000000..15a0655f1 --- /dev/null +++ b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs @@ -0,0 +1,53 @@ +using System; +using System.Text.Json.Serialization; +using Tzkt.Data.Models; + +namespace Tzkt.Api.Models +{ + public class BigMapUpdate + { + /// + /// Internal Id, can be used for pagination + /// + public int Id { get; set; } + + /// + /// Level of the block where the bigmap was updated + /// + public int Level { get; set; } + + /// + /// Timestamp of the block where the bigmap was updated + /// + public DateTime Timestamp { get; set; } + + /// + /// Bigmap ptr + /// + public int Bigmap { get; set; } + + /// + /// Smart contract in which's storage the bigmap is allocated + /// + public Alias Contract { get; set; } + + /// + /// Path to the bigmap in the contract storage + /// + public string Path { get; set; } + + /// + /// Action with the bigmap (`allocate`, `add_key`, `update_key`, `remove_key`, `remove`) + /// + public string Action { get; set; } + + /// + /// Updated key. + /// If the action is `allocate` or `remove` the content will be `null`. + /// + public BigMapKeyShort Content { get; set; } + + [JsonIgnore] + public BigMapTag _Tags { get; set; } + } +} diff --git a/Tzkt.Api/Models/Operations/OriginationOperation.cs b/Tzkt.Api/Models/Operations/OriginationOperation.cs index 56930bd7e..9d9353c1f 100644 --- a/Tzkt.Api/Models/Operations/OriginationOperation.cs +++ b/Tzkt.Api/Models/Operations/OriginationOperation.cs @@ -121,7 +121,7 @@ public class OriginationOperation : Operation /// /// List of bigmap updates (aka big_map_diffs) caused by the origination. /// - public List Bigmaps { get; set; } + public List Diffs { get; set; } /// /// Operation status (`applied` - an operation applied by the node and successfully added to the blockchain, diff --git a/Tzkt.Api/Models/Operations/TransactionOperation.cs b/Tzkt.Api/Models/Operations/TransactionOperation.cs index 694b315e1..41c8d04fc 100644 --- a/Tzkt.Api/Models/Operations/TransactionOperation.cs +++ b/Tzkt.Api/Models/Operations/TransactionOperation.cs @@ -113,7 +113,7 @@ public class TransactionOperation : Operation /// /// List of bigmap updates (aka big_map_diffs) caused by the transaction. /// - public List Bigmaps { get; set; } + public List Diffs { get; set; } /// /// Operation status (`applied` - an operation applied by the node and successfully added to the blockchain, diff --git a/Tzkt.Api/Parameters/BigMapActionParameter.cs b/Tzkt.Api/Parameters/BigMapActionParameter.cs new file mode 100644 index 000000000..a50736791 --- /dev/null +++ b/Tzkt.Api/Parameters/BigMapActionParameter.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using Microsoft.AspNetCore.Mvc; +using NJsonSchema.Annotations; + +namespace Tzkt.Api +{ + [ModelBinder(BinderType = typeof(BigMapActionBinder))] + public class BigMapActionParameter + { + /// + /// **Equal** filter mode (optional, i.e. `param.eq=123` is the same as `param=123`). \ + /// Specify a contract kind to get items where the specified field is equal to the specified value. + /// + /// Example: `?kind=smart_contract`. + /// + [JsonSchemaType(typeof(string))] + public int? Eq { get; set; } + + /// + /// **Not equal** filter mode. \ + /// Specify a contract kind to get items where the specified field is not equal to the specified value. + /// + /// Example: `?kind.ne=delegator_contract`. + /// + [JsonSchemaType(typeof(string))] + public int? Ne { get; set; } + + /// + /// **In list** (any of) filter mode. \ + /// Specify a comma-separated list of contract kinds to get items where the specified field is equal to one of the specified values. + /// + /// Example: `?kind.in=smart_contract,asset`. + /// + [JsonSchemaType(typeof(List))] + public List In { get; set; } + + /// + /// **Not in list** (none of) filter mode. \ + /// Specify a comma-separated list of contract kinds to get items where the specified field is not equal to all the specified values. + /// + /// Example: `?kind.ni=smart_contract,asset`. + /// + [JsonSchemaType(typeof(List))] + public List Ni { get; set; } + } +} diff --git a/Tzkt.Api/Parameters/BigMapTagsParameter.cs b/Tzkt.Api/Parameters/BigMapTagsParameter.cs new file mode 100644 index 000000000..aae437414 --- /dev/null +++ b/Tzkt.Api/Parameters/BigMapTagsParameter.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using Microsoft.AspNetCore.Mvc; +using NJsonSchema.Annotations; + +namespace Tzkt.Api +{ + [ModelBinder(BinderType = typeof(BigMapTagsBinder))] + public class BigMapTagsParameter + { + /// + /// **Equal** filter mode (optional, i.e. `param.eq=123` is the same as `param=123`). \ + /// Specify a contract kind to get items where the specified field is equal to the specified value. + /// + /// Example: `?kind=smart_contract`. + /// + [JsonSchemaType(typeof(List))] + public int? Eq { get; set; } + + /// + /// **Has any** filter mode. \ + /// Specify a comma-separated list of contract kinds to get items where the specified field is equal to one of the specified values. + /// + /// Example: `?kind.in=smart_contract,asset`. + /// + [JsonSchemaType(typeof(List))] + public int? Any { get; set; } + + /// + /// **Has all** filter mode. \ + /// Specify a comma-separated list of contract kinds to get items where the specified field is not equal to all the specified values. + /// + /// Example: `?kind.ni=smart_contract,asset`. + /// + [JsonSchemaType(typeof(List))] + public int? All { get; set; } + } +} diff --git a/Tzkt.Api/Parameters/Binders/BigMapActionBinder.cs b/Tzkt.Api/Parameters/Binders/BigMapActionBinder.cs new file mode 100644 index 000000000..aae39e535 --- /dev/null +++ b/Tzkt.Api/Parameters/Binders/BigMapActionBinder.cs @@ -0,0 +1,45 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc.ModelBinding; + +namespace Tzkt.Api +{ + public class BigMapActionBinder : IModelBinder + { + public Task BindModelAsync(ModelBindingContext bindingContext) + { + var model = bindingContext.ModelName; + var hasValue = false; + + if (!bindingContext.TryGetBigMapAction($"{model}", ref hasValue, out var value)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapAction($"{model}.eq", ref hasValue, out var eq)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapAction($"{model}.ne", ref hasValue, out var ne)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapActionList($"{model}.in", ref hasValue, out var @in)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapActionList($"{model}.ni", ref hasValue, out var ni)) + return Task.CompletedTask; + + if (!hasValue) + { + bindingContext.Result = ModelBindingResult.Success(null); + return Task.CompletedTask; + } + + bindingContext.Result = ModelBindingResult.Success(new BigMapActionParameter + { + Eq = value ?? eq, + Ne = ne, + In = @in, + Ni = ni + }); + + return Task.CompletedTask; + } + } +} diff --git a/Tzkt.Api/Parameters/Binders/BigMapTagsBinder.cs b/Tzkt.Api/Parameters/Binders/BigMapTagsBinder.cs new file mode 100644 index 000000000..e17bee554 --- /dev/null +++ b/Tzkt.Api/Parameters/Binders/BigMapTagsBinder.cs @@ -0,0 +1,41 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc.ModelBinding; + +namespace Tzkt.Api +{ + public class BigMapTagsBinder : IModelBinder + { + public Task BindModelAsync(ModelBindingContext bindingContext) + { + var model = bindingContext.ModelName; + var hasValue = false; + + if (!bindingContext.TryGetBigMapTags($"{model}", ref hasValue, out var value)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapTags($"{model}.eq", ref hasValue, out var eq)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapTags($"{model}.any", ref hasValue, out var any)) + return Task.CompletedTask; + + if (!bindingContext.TryGetBigMapTags($"{model}.all", ref hasValue, out var all)) + return Task.CompletedTask; + + if (!hasValue) + { + bindingContext.Result = ModelBindingResult.Success(null); + return Task.CompletedTask; + } + + bindingContext.Result = ModelBindingResult.Success(new BigMapTagsParameter + { + Eq = value ?? eq, + Any = any, + All = all + }); + + return Task.CompletedTask; + } + } +} diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 19d9f72c6..4e5aca628 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -102,6 +102,8 @@ public async Task Get(int contractId, string path, MichelineFormat miche public async Task> Get( AccountParameter contract, + StringParameter path, + BigMapTagsParameter tags, bool? active, Int32Parameter lastLevel, SortParameter sort, @@ -111,10 +113,13 @@ public async Task> Get( { var sql = new SqlBuilder(@"SELECT * FROM ""BigMaps""") .Filter("ContractId", contract) + .Filter("StoragePath", path) + .Filter("Tags", tags) .Filter("Active", active) .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { + "ptr" => ("Ptr", "Ptr"), "firstLevel" => ("Id", "FirstLevel"), "lastLevel" => ("LastLevel", "LastLevel"), "totalKeys" => ("TotalKeys", "TotalKeys"), @@ -131,6 +136,8 @@ public async Task> Get( public async Task Get( AccountParameter contract, + StringParameter path, + BigMapTagsParameter tags, bool? active, Int32Parameter lastLevel, SortParameter sort, @@ -144,7 +151,7 @@ public async Task Get( { switch (field) { - case "id": columns.Add(@"""Ptr"""); break; + case "ptr": columns.Add(@"""Ptr"""); break; case "contract": columns.Add(@"""ContractId"""); break; case "path": columns.Add(@"""StoragePath"""); break; case "active": columns.Add(@"""Active"""); break; @@ -164,10 +171,13 @@ public async Task Get( var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMaps""") .Filter("ContractId", contract) + .Filter("StoragePath", path) + .Filter("Tags", tags) .Filter("Active", active) .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { + "ptr" => ("Ptr", "Ptr"), "firstLevel" => ("Id", "FirstLevel"), "lastLevel" => ("LastLevel", "LastLevel"), "totalKeys" => ("TotalKeys", "TotalKeys"), @@ -187,7 +197,7 @@ public async Task Get( { switch (fields[i]) { - case "id": + case "ptr": foreach (var row in rows) result[j++][i] = row.Ptr; break; @@ -247,6 +257,8 @@ public async Task Get( public async Task Get( AccountParameter contract, + StringParameter path, + BigMapTagsParameter tags, bool? active, Int32Parameter lastLevel, SortParameter sort, @@ -258,7 +270,7 @@ public async Task Get( var columns = new HashSet(1); switch (field) { - case "id": columns.Add(@"""Ptr"""); break; + case "ptr": columns.Add(@"""Ptr"""); break; case "contract": columns.Add(@"""ContractId"""); break; case "path": columns.Add(@"""StoragePath"""); break; case "active": columns.Add(@"""Active"""); break; @@ -277,10 +289,13 @@ public async Task Get( var sql = new SqlBuilder($@"SELECT {string.Join(',', columns)} FROM ""BigMaps""") .Filter("ContractId", contract) + .Filter("StoragePath", path) + .Filter("Tags", tags) .Filter("Active", active) .Filter("LastLevel", lastLevel) .Take(sort, offset, limit, x => x switch { + "ptr" => ("Ptr", "Ptr"), "firstLevel" => ("Id", "FirstLevel"), "lastLevel" => ("LastLevel", "LastLevel"), "totalKeys" => ("TotalKeys", "TotalKeys"), @@ -298,7 +313,7 @@ public async Task Get( switch (field) { - case "id": + case "ptr": foreach (var row in rows) result[j++] = row.Ptr; break; @@ -608,7 +623,7 @@ public async Task GetKeys( #endregion #region historical keys - async Task GetHistoricalKey( + async Task GetHistoricalKey( BigMapKey key, int level, MichelineFormat micheline) @@ -617,7 +632,7 @@ async Task GetHistoricalKey( return null; if (level > key.LastLevel) - return new BigMapKeyShort + return new BigMapKeyHistorical { Id = key.Id, Hash = key.Hash, @@ -640,7 +655,7 @@ ORDER BY ""Level"" DESC var row = await db.QueryFirstOrDefaultAsync(sql); if (row == null) return null; - return new BigMapKeyShort + return new BigMapKeyHistorical { Id = key.Id, Hash = key.Hash, @@ -650,7 +665,7 @@ ORDER BY ""Level"" DESC }; } - public async Task GetHistoricalKey( + public async Task GetHistoricalKey( int ptr, int level, string key, @@ -659,7 +674,7 @@ public async Task GetHistoricalKey( return await GetHistoricalKey(await GetKey(ptr, key, micheline), level, micheline); } - public async Task GetHistoricalKeyByHash( + public async Task GetHistoricalKeyByHash( int ptr, int level, string hash, @@ -668,7 +683,7 @@ public async Task GetHistoricalKeyByHash( return await GetHistoricalKey(await GetKeyByHash(ptr, hash, micheline), level, micheline); } - public async Task> GetHistoricalKeys( + public async Task> GetHistoricalKeys( int ptr, int level, bool? active, @@ -703,7 +718,7 @@ INNER JOIN ""BigMapKeys"" as k using var db = GetConnection(); var rows = await db.QueryAsync(sql.Query, sql.Params); - return rows.Select(row => (BigMapKeyShort)ReadBigMapKeyShort(row, micheline)); + return rows.Select(row => (BigMapKeyHistorical)ReadBigMapKeyShort(row, micheline)); } public async Task GetHistoricalKeys( @@ -929,8 +944,165 @@ public async Task> GetKeyByHashUpdates( } #endregion + #region bigmap updates + public async Task> GetUpdates( + Int32Parameter ptr, + BigMapActionParameter action, + Int32Parameter level, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + var fCol = (int)micheline < 2 ? "Json" : "Raw"; + + var sql = new SqlBuilder($@"SELECT ""Id"", ""BigMapPtr"", ""Action"", ""Level"", ""BigMapKeyId"", ""{fCol}Value"" FROM ""BigMapUpdates""") + .Filter("BigMapPtr", ptr) + .Filter("Action", action) + .Filter("Level", level) + .Take(sort, offset, limit, x => x switch + { + "level" => ("Id", "Level"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var updateRows = await db.QueryAsync(sql.Query, sql.Params); + if (!updateRows.Any()) + return Enumerable.Empty(); + + #region fetch keys + var keyIds = updateRows + .Where(x => x.BigMapKeyId != null) + .Select(x => (int)x.BigMapKeyId) + .Distinct() + .ToList(); + + var keyRows = keyIds.Any() + ? (await db.QueryAsync($@" + SELECT ""Id"", ""KeyHash"", ""{fCol}Key"" FROM ""BigMapKeys"" + WHERE ""Id"" = ANY(@keyIds)", + new { keyIds })).ToDictionary(x => (int)x.Id) + : null; + #endregion + + #region fetch bigmaps + var bigmapPtrs = updateRows + .Select(x => (int)x.BigMapPtr) + .Distinct() + .ToList(); + + var bigmapRows = (await db.QueryAsync($@" + SELECT ""Ptr"", ""ContractId"", ""StoragePath"", ""Tags"" FROM ""BigMaps"" + WHERE ""Ptr"" = ANY(@bigmapPtrs)", + new { bigmapPtrs })).ToDictionary(x => (int)x.Ptr); + #endregion + + return updateRows.Select(row => + { + var bigmap = bigmapRows[(int)row.BigMapPtr]; + var key = row.BigMapKeyId == null ? null : keyRows[(int)row.BigMapKeyId]; + + return new BigMapUpdate + { + Id = row.Id, + Action = BigMapAction((int)row.Action), + Bigmap = row.BigMapPtr, + Level = row.Level, + Timestamp = Times[row.Level], + Contract = Accounts.GetAlias(bigmap.ContractId), + Path = bigmap.StoragePath, + _Tags = (Data.Models.BigMapTag)bigmap.Tags, + Content = key == null ? null : new BigMapKeyShort + { + Hash = key.KeyHash, + Key = FormatKey(key, micheline), + Value = FormatValue(row, micheline) + } + + }; + }); + } + + public async Task> GetUpdates( + Int32Parameter ptr, + StringParameter path, + AccountParameter contract, + BigMapActionParameter action, + BigMapTagsParameter tags, + Int32Parameter level, + SortParameter sort, + OffsetParameter offset, + int limit, + MichelineFormat micheline) + { + var fCol = (int)micheline < 2 ? "Json" : "Raw"; + + var sql = new SqlBuilder($@" + SELECT uu.""Id"", uu.""BigMapPtr"", uu.""Action"", uu.""Level"", uu.""BigMapKeyId"", uu.""{fCol}Value"", + bb.""ContractId"", bb.""StoragePath"", bb.""Tags"" + FROM ""BigMapUpdates"" as uu + LEFT JOIN ""BigMaps"" as bb on bb.""Ptr"" = uu.""BigMapPtr""") + .Filter("BigMapPtr", ptr) + .Filter("StoragePath", path) + .Filter("Action", action) + .Filter("Tags", tags) + .Filter("ContractId", contract) + .Filter("Level", level) + .Take(sort, offset, limit, x => x switch + { + "level" => ("Id", "Level"), + _ => ("Id", "Id") + }); + + using var db = GetConnection(); + var updateRows = await db.QueryAsync(sql.Query, sql.Params); + if (!updateRows.Any()) + return Enumerable.Empty(); + + #region fetch keys + var keyIds = updateRows + .Where(x => x.BigMapKeyId != null) + .Select(x => (int)x.BigMapKeyId) + .Distinct() + .ToList(); + + var keyRows = keyIds.Any() + ? (await db.QueryAsync($@" + SELECT ""Id"", ""KeyHash"", ""{fCol}Key"" FROM ""BigMapKeys"" + WHERE ""Id"" = ANY(@keyIds)", + new { keyIds })).ToDictionary(x => (int)x.Id) + : null; + #endregion + + return updateRows.Select(row => + { + var key = row.BigMapKeyId == null ? null : keyRows[(int)row.BigMapKeyId]; + + return new BigMapUpdate + { + Id = row.Id, + Action = BigMapAction((int)row.Action), + Bigmap = row.BigMapPtr, + Level = row.Level, + Timestamp = Times[row.Level], + Contract = Accounts.GetAlias(row.ContractId), + Path = row.StoragePath, + _Tags = (Data.Models.BigMapTag)row.Tags, + Content = key == null ? null : new BigMapKeyShort + { + Hash = key.KeyHash, + Key = FormatKey(key, micheline), + Value = FormatValue(row, micheline) + } + + }; + }); + } + #endregion + #region diffs - public static async Task>> GetBigMapUpdates(IDbConnection db, List ops, bool isTxs, MichelineFormat format) + public static async Task>> GetBigMapDiffs(IDbConnection db, List ops, bool isTxs, MichelineFormat format) { if (ops.Count == 0) return null; @@ -984,7 +1156,7 @@ public static async Task>> GetBigMapUpdates(IDb Bigmap = row.BigMapPtr, Path = bigmaps[row.BigMapPtr].StoragePath, Action = BigMapAction(row.Action), - Key = row.BigMapKeyId == null ? null : new BigMapDiffKey + Content = row.BigMapKeyId == null ? null : new BigMapKeyShort { Hash = keys[row.BigMapKeyId].KeyHash, Key = FormatKey(keys[row.BigMapKeyId], format), @@ -1000,7 +1172,7 @@ BigMap ReadBigMap(dynamic row, MichelineFormat format) { return new BigMap { - Id = row.Ptr, + Ptr = row.Ptr, Contract = Accounts.GetAlias(row.ContractId), Path = row.StoragePath, Active = row.Active, @@ -1034,9 +1206,9 @@ BigMapKey ReadBigMapKey(dynamic row, MichelineFormat format) }; } - BigMapKeyShort ReadBigMapKeyShort(dynamic row, MichelineFormat format) + BigMapKeyHistorical ReadBigMapKeyShort(dynamic row, MichelineFormat format) { - return new BigMapKeyShort + return new BigMapKeyHistorical { Id = row.Id, Active = row.Active, diff --git a/Tzkt.Api/Repositories/OperationRepository.cs b/Tzkt.Api/Repositories/OperationRepository.cs index 3d69eb66e..f893e58a8 100644 --- a/Tzkt.Api/Repositories/OperationRepository.cs +++ b/Tzkt.Api/Repositories/OperationRepository.cs @@ -3208,8 +3208,8 @@ LEFT JOIN ""Scripts"" as sc format); #endregion - #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -3254,7 +3254,7 @@ LEFT JOIN ""Scripts"" as sc ContractBalance = row.Balance, Code = (int)format % 2 == 0 ? code : code.ToJson(), Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : new OriginatedContract @@ -3294,8 +3294,8 @@ LEFT JOIN ""Scripts"" as sc format); #endregion - #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -3340,7 +3340,7 @@ LEFT JOIN ""Scripts"" as sc ContractBalance = row.Balance, Code = (int)format % 2 == 0 ? code : code.ToJson(), Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : new OriginatedContract @@ -3380,8 +3380,8 @@ LEFT JOIN ""Scripts"" as sc format); #endregion - #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -3426,7 +3426,7 @@ LEFT JOIN ""Scripts"" as sc ContractBalance = row.Balance, Code = (int)format % 2 == 0 ? code : code.ToJson(), Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), OriginatedContract = contract == null ? null : new OriginatedContract @@ -3562,9 +3562,9 @@ INNER JOIN ""Blocks"" as b : null; #endregion - #region include bigmaps - var updates = includeBigmaps - ? await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = includeBigmaps + ? await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -3609,7 +3609,7 @@ INNER JOIN ""Blocks"" as b Kind = contract.KindString }, Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), ContractManager = row.ManagerId != null ? Accounts.GetAlias(row.ManagerId) : null, Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, Quote = Quotes.Get(quote, row.Level) @@ -3673,7 +3673,7 @@ public async Task GetOriginations( joins.Add(@"LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"""); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": + case "diffs": columns.Add(@"o.""Id"""); columns.Add(@"o.""BigMapUpdates"""); break; @@ -3819,16 +3819,16 @@ public async Task GetOriginations( foreach (var row in rows) result[j++][i] = row.StorageId == null ? null : storages[row.StorageId]; break; - case "bigmaps": - var updates = await BigMapsRepository.GetBigMapUpdates(db, + case "diffs": + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), false, format); - if (updates != null) + if (diffs != null) foreach (var row in rows) - result[j++][i] = updates.GetValueOrDefault((int)row.Id); + result[j++][i] = diffs.GetValueOrDefault((int)row.Id); break; case "status": foreach (var row in rows) @@ -3920,7 +3920,7 @@ public async Task GetOriginations( joins.Add(@"LEFT JOIN ""Scripts"" as sc ON sc.""Id"" = o.""ScriptId"""); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": + case "diffs": columns.Add(@"o.""Id"""); columns.Add(@"o.""BigMapUpdates"""); break; @@ -4063,16 +4063,16 @@ public async Task GetOriginations( foreach (var row in rows) result[j++] = row.StorageId == null ? null : storages[row.StorageId]; break; - case "bigmaps": - var updates = await BigMapsRepository.GetBigMapUpdates(db, + case "diffs": + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), false, format); - if (updates != null) + if (diffs != null) foreach (var row in rows) - result[j++] = updates.GetValueOrDefault((int)row.Id); + result[j++] = diffs.GetValueOrDefault((int)row.Id); break; case "status": foreach (var row in rows) @@ -4147,8 +4147,8 @@ INNER JOIN ""Blocks"" as b format); #endregion - #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -4189,7 +4189,7 @@ INNER JOIN ""Blocks"" as b } }, Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4220,8 +4220,8 @@ INNER JOIN ""Blocks"" as b format); #endregion - #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -4262,7 +4262,7 @@ INNER JOIN ""Blocks"" as b } }, Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4293,8 +4293,8 @@ INNER JOIN ""Blocks"" as b format); #endregion - #region include bigmaps - var updates = await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -4335,7 +4335,7 @@ INNER JOIN ""Blocks"" as b } }, Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4475,9 +4475,9 @@ INNER JOIN ""Blocks"" as b : null; #endregion - #region include bigmaps - var updates = includeBigmaps - ? await BigMapsRepository.GetBigMapUpdates(db, + #region include diffs + var diffs = includeBigmaps + ? await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), @@ -4519,7 +4519,7 @@ INNER JOIN ""Blocks"" as b } }, Storage = row.StorageId == null ? null : storages?[row.StorageId], - Bigmaps = updates?.GetValueOrDefault((int)row.Id), + Diffs = diffs?.GetValueOrDefault((int)row.Id), Status = StatusToString(row.Status), Errors = row.Errors != null ? OperationErrorSerializer.Deserialize(row.Errors) : null, HasInternals = row.InternalOperations > 0, @@ -4650,7 +4650,7 @@ public async Task GetTransactions( }); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": + case "diffs": columns.Add(@"o.""Id"""); columns.Add(@"o.""BigMapUpdates"""); break; @@ -4809,16 +4809,16 @@ public async Task GetTransactions( foreach (var row in rows) result[j++][i] = row.StorageId == null ? null : storages[row.StorageId]; break; - case "bigmaps": - var updates = await BigMapsRepository.GetBigMapUpdates(db, + case "diffs": + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), true, format); - if (updates != null) + if (diffs != null) foreach (var row in rows) - result[j++][i] = updates.GetValueOrDefault((int)row.Id); + result[j++][i] = diffs.GetValueOrDefault((int)row.Id); break; case "status": foreach (var row in rows) @@ -4900,7 +4900,7 @@ public async Task GetTransactions( }); break; case "storage": columns.Add(@"o.""StorageId"""); break; - case "bigmaps": + case "diffs": columns.Add(@"o.""Id"""); columns.Add(@"o.""BigMapUpdates"""); break; @@ -5056,16 +5056,16 @@ public async Task GetTransactions( foreach (var row in rows) result[j++] = row.StorageId == null ? null : storages[row.StorageId]; break; - case "bigmaps": - var updates = await BigMapsRepository.GetBigMapUpdates(db, + case "diffs": + var diffs = await BigMapsRepository.GetBigMapDiffs(db, rows.Where(x => x.BigMapUpdates != null) .Select(x => (int)x.Id) .ToList(), true, format); - if (updates != null) + if (diffs != null) foreach (var row in rows) - result[j++] = updates.GetValueOrDefault((int)row.Id); + result[j++] = diffs.GetValueOrDefault((int)row.Id); break; case "status": foreach (var row in rows) diff --git a/Tzkt.Api/Utils/SqlBuilder.cs b/Tzkt.Api/Utils/SqlBuilder.cs index 25ff0b83f..e90604cf7 100644 --- a/Tzkt.Api/Utils/SqlBuilder.cs +++ b/Tzkt.Api/Utils/SqlBuilder.cs @@ -129,6 +129,47 @@ public SqlBuilder Filter(string column, ContractKindParameter kind) return this; } + public SqlBuilder Filter(string column, BigMapActionParameter action) + { + if (action == null) return this; + + if (action.Eq != null) + AppendFilter($@"""{column}"" = {action.Eq}"); + + if (action.Ne != null) + AppendFilter($@"""{column}"" != {action.Ne}"); + + if (action.In != null) + { + AppendFilter($@"""{column}"" = ANY (@p{Counter})"); + Params.Add($"p{Counter++}", action.In); + } + + if (action.Ni != null && action.Ni.Count > 0) + { + AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); + Params.Add($"p{Counter++}", action.Ni); + } + + return this; + } + + public SqlBuilder Filter(string column, BigMapTagsParameter tags) + { + if (tags == null) return this; + + if (tags.Eq != null) + AppendFilter($@"""{column}"" = {tags.Eq}"); + + if (tags.Any != null) + AppendFilter($@"""{column}"" & {tags.Any} > 0"); + + if (tags.All != null) + AppendFilter($@"""{column}"" & {tags.All} = {tags.All}"); + + return this; + } + public SqlBuilder Filter(string column, MigrationKindParameter kind) { if (kind == null) return this; From 158c6f7d5d66f4a42bfcece8e956f58901e7aada Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Wed, 14 Apr 2021 23:56:13 +0300 Subject: [PATCH 24/35] Add jsonb indexes, rework building sql from json parameters --- .../ModelBindingContextExtension.cs | 126 +- Tzkt.Api/Parameters/Binders/JsonBinder.cs | 195 +- Tzkt.Api/Parameters/Binders/SelectBinder.cs | 6 +- Tzkt.Api/Parameters/Binders/StringBinder.cs | 4 +- Tzkt.Api/Parameters/JsonParameter.cs | 40 +- Tzkt.Api/Utils/JsonPath.cs | 76 + Tzkt.Api/Utils/SqlBuilder.cs | 98 +- .../20210412164322_JsonIndexes.Designer.cs | 2799 +++++++++++++++++ .../Migrations/20210412164322_JsonIndexes.cs | 69 + .../Migrations/TzktContextModelSnapshot.cs | 14 +- Tzkt.Data/Models/Accounts/Account.cs | 2 +- .../Models/Operations/TransactionOperation.cs | 5 + Tzkt.Data/Models/Scripts/BigMapKey.cs | 16 +- 13 files changed, 3230 insertions(+), 220 deletions(-) create mode 100644 Tzkt.Api/Utils/JsonPath.cs create mode 100644 Tzkt.Data/Migrations/20210412164322_JsonIndexes.Designer.cs create mode 100644 Tzkt.Data/Migrations/20210412164322_JsonIndexes.cs diff --git a/Tzkt.Api/Extensions/ModelBindingContextExtension.cs b/Tzkt.Api/Extensions/ModelBindingContextExtension.cs index 781aca9ac..3596c1de2 100644 --- a/Tzkt.Api/Extensions/ModelBindingContextExtension.cs +++ b/Tzkt.Api/Extensions/ModelBindingContextExtension.cs @@ -1,6 +1,7 @@ using System; using System.Linq; using System.Collections.Generic; +using System.Text.Json; using System.Text.RegularExpressions; using Microsoft.AspNetCore.Mvc.ModelBinding; @@ -889,6 +890,18 @@ public static bool TryGetOperationStatus(this ModelBindingContext bindingContext return true; } + public static bool TryGetBool(this ModelBindingContext bindingContext, string name, out bool? result) + { + result = null; + var valueObject = bindingContext.ValueProvider.GetValue(name); + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + result = !(valueObject.FirstValue == "false" || valueObject.FirstValue == "0"); + } + return true; + } + public static bool TryGetBool(this ModelBindingContext bindingContext, string name, ref bool hasValue, out bool? result) { result = null; @@ -904,56 +917,120 @@ public static bool TryGetBool(this ModelBindingContext bindingContext, string na return true; } - public static bool TryGetString(this ModelBindingContext bindingContext, string name, ref bool hasValue, out string result) + public static bool TryGetString(this ModelBindingContext bindingContext, string name, out string result) { result = null; var valueObject = bindingContext.ValueProvider.GetValue(name); - if (valueObject != ValueProviderResult.None) { bindingContext.ModelState.SetModelValue(name, valueObject); if (!string.IsNullOrEmpty(valueObject.FirstValue)) { - hasValue = true; result = valueObject.FirstValue; + return true; } } - - return true; + bindingContext.ModelState.TryAddModelError(name, "Invalid value."); + return false; } - public static bool TryGetStringList(this ModelBindingContext bindingContext, string name, ref bool hasValue, out List result) + public static bool TryGetJson(this ModelBindingContext bindingContext, string name, out string result) { result = null; var valueObject = bindingContext.ValueProvider.GetValue(name); - if (valueObject != ValueProviderResult.None) { bindingContext.ModelState.SetModelValue(name, valueObject); if (!string.IsNullOrEmpty(valueObject.FirstValue)) { - var rawValues = valueObject.FirstValue - .Replace("\\,", "ъуъ") - .Split(',', StringSplitOptions.RemoveEmptyEntries); + try + { + var json = NormalizeJson(valueObject.FirstValue); + using var doc = JsonDocument.Parse(json); + result = json; + return true; + } + catch (JsonException) { } + } + } + bindingContext.ModelState.TryAddModelError(name, "Invalid JSON value."); + return false; + } - if (rawValues.Length == 0) + public static bool TryGetJsonArray(this ModelBindingContext bindingContext, string name, out string[] result) + { + result = null; + var valueObject = bindingContext.ValueProvider.GetValue(name); + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + try { - bindingContext.ModelState.TryAddModelError(name, "List should contain at least one item."); - return false; + if (Regex.IsMatch(valueObject.FirstValue, @"^[\w,]+$")) + { + result = valueObject.FirstValue.Split(',').Select(x => NormalizeJson(x)).ToArray(); + } + else + { + using var doc = JsonDocument.Parse(valueObject.FirstValue); + if (doc.RootElement.ValueKind != JsonValueKind.Array) + { + bindingContext.ModelState.TryAddModelError(name, "Invalid JSON array."); + return false; + } + result = doc.RootElement.EnumerateArray().Select(x => NormalizeJson(x.GetRawText())).ToArray(); + } + if (result.Length < 2) + { + bindingContext.ModelState.TryAddModelError(name, "JSON array must contain at least two items."); + return false; + } + return true; } + catch (JsonException) { } + } + } + bindingContext.ModelState.TryAddModelError(name, "Invalid JSON array."); + return false; + } - hasValue = true; - result = new List(rawValues.Length); + static string NormalizeJson(string value) + { + switch (value[0]) + { + case '{': + case '[': + case '"': + case 't' when value == "true": + case 'f' when value == "false": + case 'n' when value == "null": + return value; + default: + return $"\"{value}\""; + } + } - foreach (var rawValue in rawValues) - result.Add(rawValue.Replace("ъуъ", ",")); + public static bool TryGetString(this ModelBindingContext bindingContext, string name, ref bool hasValue, out string result) + { + result = null; + var valueObject = bindingContext.ValueProvider.GetValue(name); + + if (valueObject != ValueProviderResult.None) + { + bindingContext.ModelState.SetModelValue(name, valueObject); + if (!string.IsNullOrEmpty(valueObject.FirstValue)) + { + hasValue = true; + result = valueObject.FirstValue; } } return true; } - public static bool TryGetStringListSimple(this ModelBindingContext bindingContext, string name, ref bool hasValue, out List result) + public static bool TryGetStringList(this ModelBindingContext bindingContext, string name, ref bool hasValue, out string[] result) { result = null; var valueObject = bindingContext.ValueProvider.GetValue(name); @@ -972,14 +1049,14 @@ public static bool TryGetStringListSimple(this ModelBindingContext bindingContex } hasValue = true; - result = new List(rawValues); + result = rawValues; } } return true; } - public static bool TryGetStringArray(this ModelBindingContext bindingContext, string name, ref bool hasValue, out string[] result) + public static bool TryGetStringListEscaped(this ModelBindingContext bindingContext, string name, ref bool hasValue, out List result) { result = null; var valueObject = bindingContext.ValueProvider.GetValue(name); @@ -989,7 +1066,9 @@ public static bool TryGetStringArray(this ModelBindingContext bindingContext, st bindingContext.ModelState.SetModelValue(name, valueObject); if (!string.IsNullOrEmpty(valueObject.FirstValue)) { - var rawValues = valueObject.FirstValue.Split(',', StringSplitOptions.RemoveEmptyEntries); + var rawValues = valueObject.FirstValue + .Replace("\\,", "ъуъ") + .Split(',', StringSplitOptions.RemoveEmptyEntries); if (rawValues.Length == 0) { @@ -998,7 +1077,10 @@ public static bool TryGetStringArray(this ModelBindingContext bindingContext, st } hasValue = true; - result = rawValues; + result = new List(rawValues.Length); + + foreach (var rawValue in rawValues) + result.Add(rawValue.Replace("ъуъ", ",")); } } diff --git a/Tzkt.Api/Parameters/Binders/JsonBinder.cs b/Tzkt.Api/Parameters/Binders/JsonBinder.cs index 91cf68968..f4c60ce51 100644 --- a/Tzkt.Api/Parameters/Binders/JsonBinder.cs +++ b/Tzkt.Api/Parameters/Binders/JsonBinder.cs @@ -1,181 +1,130 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; +using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc.ModelBinding; +using Tzkt.Api.Utils; namespace Tzkt.Api { public class JsonBinder : IModelBinder { - public Task BindModelAsync(ModelBindingContext bindingContext) + public Task BindModelAsync(ModelBindingContext ctx) { - var model = bindingContext.ModelName; + var model = ctx.ModelName; JsonParameter res = null; - foreach (var key in bindingContext.HttpContext.Request.Query.Keys.Where(x => x == model || x.StartsWith($"{model}."))) + foreach (var key in ctx.HttpContext.Request.Query.Keys.Where(x => x == model || x.StartsWith($"{model}."))) { - var sKey = key.Replace("..", "*"); - var arr = sKey.Split(".", StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < arr.Length; i++) + if (!JsonPath.TryParse(key, out var path)) { - arr[i] = arr[i].Replace("*", "."); - if (!Regex.IsMatch(arr[i], "^[0-9A-z_.%@]+$")) - { - bindingContext.ModelState.AddModelError(key, $"Invalid path value '{arr[i]}'"); - return Task.CompletedTask; - } + ctx.ModelState.AddModelError(key, + $"Path contains invalid item: {path.First(x => x.Type == JsonPathType.None).Value}"); + return Task.CompletedTask; + } + if (path.Any(x => x.Type == JsonPathType.Any) && path.Any(x => x.Type == JsonPathType.Index)) + { + ctx.ModelState.AddModelError(key, + $"Mixed array access is not allowed: [{path.First(x => x.Type == JsonPathType.Index).Value}] and [*]"); + return Task.CompletedTask; } - var hasValue = false; - switch (arr[^1]) + res ??= new JsonParameter(); + switch (path[^1].Value) { case "eq": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var eq)) + if (!ctx.TryGetJson(key, out var eq)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Eq ??= new List<(string, string)>(); - res.Eq.Add((string.Join(',', arr[1..^1]), eq)); - } + res.Eq ??= new(); + res.Eq.Add((path[1..^1], eq)); break; case "ne": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var ne)) + if (!ctx.TryGetJson(key, out var ne)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Ne ??= new List<(string, string)>(); - res.Ne.Add((string.Join(',', arr[1..^1]), ne)); - } + res.Ne ??= new(); + res.Ne.Add((path[1..^1], ne)); break; case "gt": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var gt)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetString(key, out var gt)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Gt ??= new List<(string, string)>(); - res.Gt.Add((string.Join(',', arr[1..^1]), gt)); - } + res.Gt ??= new(); + res.Gt.Add((path[1..^1], gt)); break; case "ge": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var ge)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetString(key, out var ge)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Ge ??= new List<(string, string)>(); - res.Ge.Add((string.Join(',', arr[1..^1]), ge)); - } + res.Ge ??= new(); + res.Ge.Add((path[1..^1], ge)); break; case "lt": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var lt)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetString(key, out var lt)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Lt ??= new List<(string, string)>(); - res.Lt.Add((string.Join(',', arr[1..^1]), lt)); - } + res.Lt ??= new(); + res.Lt.Add((path[1..^1], lt)); break; case "le": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var le)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetString(key, out var le)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Le ??= new List<(string, string)>(); - res.Le.Add((string.Join(',', arr[1..^1]), le)); - } + res.Le ??= new(); + res.Le.Add((path[1..^1], le)); break; case "as": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var @as)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetString(key, out var @as)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.As ??= new List<(string, string)>(); - res.As.Add((string.Join(',', arr[1..^1]), @as - .Replace("%", "\\%") - .Replace("\\*", "ъуъ") - .Replace("*", "%") - .Replace("ъуъ", "*"))); - } + res.As ??= new(); + res.As.Add((path[1..^1], @as + .Replace("%", "\\%") + .Replace("\\*", "ъуъ") + .Replace("*", "%") + .Replace("ъуъ", "*"))); break; case "un": - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var un)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetString(key, out var un)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Un ??= new List<(string, string)>(); - res.Un.Add((string.Join(',', arr[1..^1]), un - .Replace("%", "\\%") - .Replace("\\*", "ъуъ") - .Replace("*", "%") - .Replace("ъуъ", "*"))); - } + res.Un ??= new(); + res.Un.Add((path[1..^1], un + .Replace("%", "\\%") + .Replace("\\*", "ъуъ") + .Replace("*", "%") + .Replace("ъуъ", "*"))); break; case "in": - hasValue = false; - if (!bindingContext.TryGetStringList(key, ref hasValue, out var @in)) + if (!ctx.TryGetJsonArray(key, out var @in)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.In ??= new List<(string, List)>(); - res.In.Add((string.Join(',', arr[1..^1]), @in)); - } + res.In ??= new(); + res.In.Add((path[1..^1], @in)); break; case "ni": - hasValue = false; - if (!bindingContext.TryGetStringList(key, ref hasValue, out var ni)) + if (!ctx.TryGetJsonArray(key, out var ni)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Ni ??= new List<(string, List)>(); - res.Ni.Add((string.Join(',', arr[1..^1]), ni)); - } + res.Ni ??= new(); + res.Ni.Add((path[1..^1], ni)); break; case "null": - hasValue = false; - if (!bindingContext.TryGetBool(key, ref hasValue, out var isNull)) + if (HasWildcard(ctx, key, path) || !ctx.TryGetBool(key, out var isNull)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Null ??= new List<(string, bool)>(); - res.Null.Add((string.Join(',', arr[1..^1]), (bool)isNull)); - } + res.Null ??= new(); + res.Null.Add((path[1..^1], (bool)isNull)); break; default: - hasValue = false; - if (!bindingContext.TryGetString(key, ref hasValue, out var value)) + if (!ctx.TryGetJson(key, out var val)) return Task.CompletedTask; - if (hasValue) - { - res ??= new JsonParameter(); - res.Eq ??= new List<(string, string)>(); - res.Eq.Add((string.Join(',', arr[1..]), value)); - } + res.Eq ??= new(); + res.Eq.Add((path[1..], val)); break; } } - bindingContext.Result = ModelBindingResult.Success(res); + ctx.Result = ModelBindingResult.Success(res); return Task.CompletedTask; } + + static bool HasWildcard(ModelBindingContext ctx, string key, JsonPath[] path) + { + if (path.Any(x => x.Type == JsonPathType.Any)) + { + ctx.ModelState.AddModelError(key, $"Path contains invalid item: [*] is not allowed in this mode"); + return true; + } + return false; + } } } diff --git a/Tzkt.Api/Parameters/Binders/SelectBinder.cs b/Tzkt.Api/Parameters/Binders/SelectBinder.cs index c4b67a364..66c7346ad 100644 --- a/Tzkt.Api/Parameters/Binders/SelectBinder.cs +++ b/Tzkt.Api/Parameters/Binders/SelectBinder.cs @@ -13,13 +13,13 @@ public Task BindModelAsync(ModelBindingContext bindingContext) var model = bindingContext.ModelName; var hasValue = false; - if (!bindingContext.TryGetStringArray($"{model}", ref hasValue, out var value)) + if (!bindingContext.TryGetStringList($"{model}", ref hasValue, out var value)) return Task.CompletedTask; - if (!bindingContext.TryGetStringArray($"{model}.fields", ref hasValue, out var rec)) + if (!bindingContext.TryGetStringList($"{model}.fields", ref hasValue, out var rec)) return Task.CompletedTask; - if (!bindingContext.TryGetStringArray($"{model}.values", ref hasValue, out var tup)) + if (!bindingContext.TryGetStringList($"{model}.values", ref hasValue, out var tup)) return Task.CompletedTask; if (!hasValue) diff --git a/Tzkt.Api/Parameters/Binders/StringBinder.cs b/Tzkt.Api/Parameters/Binders/StringBinder.cs index caf41d359..301f2463c 100644 --- a/Tzkt.Api/Parameters/Binders/StringBinder.cs +++ b/Tzkt.Api/Parameters/Binders/StringBinder.cs @@ -28,10 +28,10 @@ public Task BindModelAsync(ModelBindingContext bindingContext) if (!bindingContext.TryGetString($"{model}.un", ref hasValue, out var un)) return Task.CompletedTask; - if (!bindingContext.TryGetStringList($"{model}.in", ref hasValue, out var @in)) + if (!bindingContext.TryGetStringListEscaped($"{model}.in", ref hasValue, out var @in)) return Task.CompletedTask; - if (!bindingContext.TryGetStringList($"{model}.ni", ref hasValue, out var ni)) + if (!bindingContext.TryGetStringListEscaped($"{model}.ni", ref hasValue, out var ni)) return Task.CompletedTask; if (!bindingContext.TryGetBool($"{model}.null", ref hasValue, out var isNull)) diff --git a/Tzkt.Api/Parameters/JsonParameter.cs b/Tzkt.Api/Parameters/JsonParameter.cs index 7a4f9f08c..a7f9a9564 100644 --- a/Tzkt.Api/Parameters/JsonParameter.cs +++ b/Tzkt.Api/Parameters/JsonParameter.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using Microsoft.AspNetCore.Mvc; using NJsonSchema.Annotations; +using Tzkt.Api.Utils; namespace Tzkt.Api { @@ -9,21 +10,21 @@ public class JsonParameter { /// /// **Equal** filter mode (optional, i.e. `param.eq=123` is the same as `param=123`). \ - /// Specify a string to get items where the specified field is equal to the specified value. + /// Specify a JSON value to get items where the specified field is equal to the specified value. /// - /// Example: `?parameter={}` or `?parameter.to=tz1...`. + /// Example: `?parameter.from=tz1...` or `?parameter.signatures.[3].[0]=null` or `?parameter.sigs.[*]=null`. /// [JsonSchemaType(typeof(string))] - public List<(string, string)> Eq { get; set; } + public List<(JsonPath[], string)> Eq { get; set; } /// /// **Not equal** filter mode. \ - /// Specify a string to get items where the specified field is not equal to the specified value. + /// Specify a JSON value to get items where the specified field is not equal to the specified value. /// - /// Example: `?parameter.ne={}` or `?parameter.amount.ne=0`. + /// Example: `?parameter.ne=true` or `?parameter.amount.ne=0`. /// [JsonSchemaType(typeof(string))] - public List<(string, string)> Ne { get; set; } + public List<(JsonPath[], string)> Ne { get; set; } /// /// **Greater than** filter mode. \ @@ -34,7 +35,7 @@ public class JsonParameter /// /// Example: `?parameter.balance.gt=1234` or `?parameter.time.gt=2021-02-01`. /// - public List<(string, string)> Gt { get; set; } + public List<(JsonPath[], string)> Gt { get; set; } /// /// **Greater or equal** filter mode. \ @@ -45,7 +46,7 @@ public class JsonParameter /// /// Example: `?parameter.balance.ge=1234` or `?parameter.time.ge=2021-02-01`. /// - public List<(string, string)> Ge { get; set; } + public List<(JsonPath[], string)> Ge { get; set; } /// /// **Less than** filter mode. \ @@ -56,7 +57,7 @@ public class JsonParameter /// /// Example: `?parameter.balance.lt=1234` or `?parameter.time.lt=2021-02-01`. /// - public List<(string, string)> Lt { get; set; } + public List<(JsonPath[], string)> Lt { get; set; } /// /// **Less or equal** filter mode. \ @@ -67,7 +68,7 @@ public class JsonParameter /// /// Example: `?parameter.balance.le=1234` or `?parameter.time.le=2021-02-01`. /// - public List<(string, string)> Le { get; set; } + public List<(JsonPath[], string)> Le { get; set; } /// /// **Same as** filter mode. \ @@ -77,7 +78,7 @@ public class JsonParameter /// Example: `?parameter.as=*mid*` or `?parameter.as=*end`. /// [JsonSchemaType(typeof(string))] - public List<(string, string)> As { get; set; } + public List<(JsonPath[], string)> As { get; set; } /// /// **Unlike** filter mode. \ @@ -87,35 +88,34 @@ public class JsonParameter /// Example: `?parameter.un=*mid*` or `?parameter.un=*end`. /// [JsonSchemaType(typeof(string))] - public List<(string, string)> Un { get; set; } + public List<(JsonPath[], string)> Un { get; set; } /// /// **In list** (any of) filter mode. \ - /// Specify a comma-separated list of strings to get items where the specified field is equal to one of the specified values. \ - /// Use `\,` as an escape symbol. + /// Specify a comma-separated list of strings or JSON array to get items where the specified field is equal to one of the specified values. \ /// - /// Example: `?parameter.in=bla,bal,abl` or `?parameter.from.in=tz1,tz2,tz3`. + /// Example: `?parameter.amount.in=1,2,3` or `?parameter.in=[{"from":"tz1","to":"tz2"},{"from":"tz2","to":"tz1"}]`. /// [JsonSchemaType(typeof(List))] - public List<(string, List)> In { get; set; } + public List<(JsonPath[], string[])> In { get; set; } /// /// **Not in list** (none of) filter mode. \ /// Specify a comma-separated list of strings to get items where the specified field is not equal to all the specified values. \ /// Use `\,` as an escape symbol. /// - /// Example: `?parameter.ni=bla,bal,abl` or `?parameter.from.ni=tz1,tz2,tz3`. + /// Example: `?parameter.amount.ni=1,2,3` or `?parameter.ni=[{"from":"tz1","to":"tz2"},{"from":"tz2","to":"tz1"}]`. /// [JsonSchemaType(typeof(List))] - public List<(string, List)> Ni { get; set; } + public List<(JsonPath[], string[])> Ni { get; set; } /// /// **Is null** filter mode. \ /// Use this mode to get items where the specified field is null or not. /// - /// Example: `?parameter.null` or `?parameter.null=false` or `?parameter.sigs.0.null=false`. + /// Example: `?parameter.null` or `?parameter.null=false` or `?parameter.sigs.[0].null=false`. /// [JsonSchemaType(typeof(bool))] - public List<(string, bool)> Null { get; set; } + public List<(JsonPath[], bool)> Null { get; set; } } } diff --git a/Tzkt.Api/Utils/JsonPath.cs b/Tzkt.Api/Utils/JsonPath.cs new file mode 100644 index 000000000..5518db298 --- /dev/null +++ b/Tzkt.Api/Utils/JsonPath.cs @@ -0,0 +1,76 @@ +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tzkt.Api.Utils +{ + public class JsonPath + { + public JsonPathType Type { get; } + public string Value { get; } + + public JsonPath(string value) + { + if (Regex.IsMatch(value, @"^[\w_]+$")) + { + Type = JsonPathType.Field; + Value = value; + } + else if (Regex.IsMatch(value, @"^"".*""$")) + { + Type = JsonPathType.Key; + Value = value[1..^1]; + } + else if (Regex.IsMatch(value, @"^\[\d+\]$")) + { + Type = JsonPathType.Index; + Value = value[1..^1]; + } + else if (value == "[*]") + { + Type = JsonPathType.Any; + Value = null; + } + else + { + Type = JsonPathType.None; + Value = value; + } + } + + public static bool TryParse(string path, out JsonPath[] res) + { + res = (path.Contains('"') + ? Regex.Matches(path, @"(?:""(?:(?:\\"")|(?:[^""]))*"")|(?:[^"".]+)").Select(x => x.Value) + : path.Split(".")) + .Select(x => new JsonPath(x)) + .ToArray(); + + return res.All(x => x.Type != JsonPathType.None); + } + + public static string Merge(JsonPath[] path, string value, int ind = 0) + { + if (ind == path.Length) + return value; + + if (path[ind].Type > JsonPathType.Key) + return $"[{Merge(path, value, ++ind)}]"; + + return $"{{\"{path[ind].Value}\":{Merge(path, value, ++ind)}}}"; + } + + public static string[] Select(JsonPath[] path) + { + return path.Select(x => x.Value).ToArray(); + } + } + + public enum JsonPathType + { + None, + Field, + Key, + Index, + Any + } +} diff --git a/Tzkt.Api/Utils/SqlBuilder.cs b/Tzkt.Api/Utils/SqlBuilder.cs index e90604cf7..f85925574 100644 --- a/Tzkt.Api/Utils/SqlBuilder.cs +++ b/Tzkt.Api/Utils/SqlBuilder.cs @@ -3,8 +3,8 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Dapper; +using Tzkt.Api.Utils; namespace Tzkt.Api { @@ -429,8 +429,9 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Eq) { - AppendFilter($@"""{column}""#>>'{{{path}}}' = @p{Counter}"); - Params.Add($"p{Counter++}", value); + AppendFilter($@"""{column}"" @> {Param(JsonPath.Merge(path, value))}::jsonb"); + if (path.Any(x => x.Type == JsonPathType.Index)) + AppendFilter($@"""{column}"" #> {Param(JsonPath.Select(path))} = {Param(value)}::jsonb"); } } @@ -438,8 +439,9 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Ne) { - AppendFilter($@"""{column}""#>>'{{{path}}}' != @p{Counter}"); - Params.Add($"p{Counter++}", value); + AppendFilter(path.Any(x => x.Type == JsonPathType.Any) + ? $@"NOT (""{column}"" @> {Param(JsonPath.Merge(path, value))}::jsonb)" + : $@"NOT (""{column}"" #> {Param(JsonPath.Select(path))} = {Param(value)}::jsonb)"); } } @@ -447,12 +449,12 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Gt) { - var col = $@"""{column}""#>>'{{{path}}}'"; - var len = $"greatest(length({col}), {value.Length})"; - AppendFilter(Regex.IsMatch(value, "^[0-9]+$") - ? $@"lpad({col}, {len}, '0') > lpad(@p{Counter}, {len}, '0')" - : $@"{col} > @p{Counter}"); - Params.Add($"p{Counter++}", value); + var val = Param(value); + var fld = $@"""{column}"" #>> {Param(JsonPath.Select(path))}"; + var len = $"greatest(length({fld}), length({val}))"; + AppendFilter(Regex.IsMatch(value, @"^\d+$") + ? $@"lpad({fld}, {len}, '0') > lpad({val}, {len}, '0')" + : $@"{fld} > {val}"); } } @@ -460,12 +462,12 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Ge) { - var col = $@"""{column}""#>>'{{{path}}}'"; - var len = $"greatest(length({col}), {value.Length})"; - AppendFilter(Regex.IsMatch(value, "^[0-9]+$") - ? $@"lpad({col}, {len}, '0') >= lpad(@p{Counter}, {len}, '0')" - : $@"{col} >= @p{Counter}"); - Params.Add($"p{Counter++}", value); + var val = Param(value); + var fld = $@"""{column}"" #>> {Param(JsonPath.Select(path))}"; + var len = $"greatest(length({fld}), length({val}))"; + AppendFilter(Regex.IsMatch(value, @"^\d+$") + ? $@"lpad({fld}, {len}, '0') >= lpad({val}, {len}, '0')" + : $@"{fld} >= {val}"); } } @@ -473,12 +475,12 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Lt) { - var col = $@"""{column}""#>>'{{{path}}}'"; - var len = $"greatest(length({col}), {value.Length})"; - AppendFilter(Regex.IsMatch(value, "^[0-9]+$") - ? $@"lpad({col}, {len}, '0') < lpad(@p{Counter}, {len}, '0')" - : $@"{col} < @p{Counter}"); - Params.Add($"p{Counter++}", value); + var val = Param(value); + var fld = $@"""{column}"" #>> {Param(JsonPath.Select(path))}"; + var len = $"greatest(length({fld}), length({val}))"; + AppendFilter(Regex.IsMatch(value, @"^\d+$") + ? $@"lpad({fld}, {len}, '0') < lpad({val}, {len}, '0')" + : $@"{fld} < {val}"); } } @@ -486,12 +488,12 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Le) { - var col = $@"""{column}""#>>'{{{path}}}'"; - var len = $"greatest(length({col}), {value.Length})"; - AppendFilter(Regex.IsMatch(value, "^[0-9]+$") - ? $@"lpad({col}, {len}, '0') <= lpad(@p{Counter}, {len}, '0')" - : $@"{col} <= @p{Counter}"); - Params.Add($"p{Counter++}", value); + var val = Param(value); + var fld = $@"""{column}"" #>> {Param(JsonPath.Select(path))}"; + var len = $"greatest(length({fld}), length({val}))"; + AppendFilter(Regex.IsMatch(value, @"^\d+$") + ? $@"lpad({fld}, {len}, '0') <= lpad({val}, {len}, '0')" + : $@"{fld} <= {val}"); } } @@ -499,8 +501,7 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.As) { - AppendFilter($@"""{column}""#>>'{{{path}}}' LIKE @p{Counter}"); - Params.Add($"p{Counter++}", value); + AppendFilter($@"""{column}"" #>> {Param(JsonPath.Select(path))} LIKE {Param(value)}"); } } @@ -508,26 +509,36 @@ public SqlBuilder Filter(string column, JsonParameter json, Func { foreach (var (path, value) in json.Un) { - AppendFilter($@"NOT (""{column}""#>>'{{{path}}}' LIKE @p{Counter})"); - Params.Add($"p{Counter++}", value); + AppendFilter($@"NOT (""{column}"" #>> {Param(JsonPath.Select(path))} LIKE {Param(value)})"); } } if (json.In != null) { - foreach (var (path, value) in json.In) + foreach (var (path, values) in json.In) { - AppendFilter($@"""{column}""#>>'{{{path}}}' = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", value); + var sqls = new List(values.Length); + foreach (var value in values) + { + var sql = $@"""{column}"" @> {Param(JsonPath.Merge(path, value))}::jsonb"; + if (path.Any(x => x.Type == JsonPathType.Index)) + sql += $@" AND ""{column}"" #> {Param(JsonPath.Select(path))} = {Param(value)}::jsonb"; + sqls.Add(sql); + } + AppendFilter(string.Join(" OR ", sqls)); } } if (json.Ni != null) { - foreach (var (path, value) in json.Ni) + foreach (var (path, values) in json.Ni) { - AppendFilter($@"NOT (""{column}""#>>'{{{path}}}' = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", value); + foreach (var value in values) + { + AppendFilter(path.Any(x => x.Type == JsonPathType.Any) + ? $@"NOT (""{column}"" @> {Param(JsonPath.Merge(path, value))}::jsonb)" + : $@"NOT (""{column}"" #> {Param(JsonPath.Select(path))} = {Param(value)}::jsonb)"); + } } } @@ -544,7 +555,7 @@ public SqlBuilder Filter(string column, JsonParameter json, Func if (value) AppendFilter($@"""{column}"" IS NOT NULL"); - AppendFilter($@"""{column}""#>>'{{{path}}}' IS {(value ? "" : "NOT ")}NULL"); + AppendFilter($@"""{column}"" #>> {Param(JsonPath.Select(path))} IS {(value ? "" : "NOT ")}NULL"); } } } @@ -1297,5 +1308,12 @@ void AppendFilter(string filter) Builder.AppendLine(filter); } + + string Param(object value) + { + var name = $"@p{Counter++}"; + Params.Add(name, value); + return name; + } } } diff --git a/Tzkt.Data/Migrations/20210412164322_JsonIndexes.Designer.cs b/Tzkt.Data/Migrations/20210412164322_JsonIndexes.Designer.cs new file mode 100644 index 000000000..c4ef61e41 --- /dev/null +++ b/Tzkt.Data/Migrations/20210412164322_JsonIndexes.Designer.cs @@ -0,0 +1,2799 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Tzkt.Data; + +namespace Tzkt.Data.Migrations +{ + [DbContext(typeof(TzktContext))] + [Migration("20210412164322_JsonIndexes")] + partial class JsonIndexes + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("Relational:MaxIdentifierLength", 63) + .HasAnnotation("ProductVersion", "5.0.4") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + modelBuilder.Entity("Tzkt.Data.Models.Account", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Address") + .IsRequired() + .HasMaxLength(36) + .HasColumnType("character(36)") + .IsFixedLength(true); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("ContractsCount") + .HasColumnType("integer"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("DelegationLevel") + .HasColumnType("integer"); + + b.Property("DelegationsCount") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("MigrationsCount") + .HasColumnType("integer"); + + b.Property("OriginationsCount") + .HasColumnType("integer"); + + b.Property("RevealsCount") + .HasColumnType("integer"); + + b.Property("Staked") + .HasColumnType("boolean"); + + b.Property("TransactionsCount") + .HasColumnType("integer"); + + b.Property("Type") + .HasColumnType("smallint"); + + b.HasKey("Id"); + + b.HasIndex("Address") + .IsUnique(); + + b.HasIndex("DelegateId"); + + b.HasIndex("FirstLevel"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Metadata") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + + b.HasIndex("Staked"); + + b.HasIndex("Type"); + + b.ToTable("Accounts"); + + b.HasDiscriminator("Type"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ActivationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccountId") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.ToTable("ActivationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.AppState", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountCounter") + .HasColumnType("integer"); + + b.Property("AccountsCount") + .HasColumnType("integer"); + + b.Property("ActivationOpsCount") + .HasColumnType("integer"); + + b.Property("BallotOpsCount") + .HasColumnType("integer"); + + b.Property("BigMapCounter") + .HasColumnType("integer"); + + b.Property("BigMapKeyCounter") + .HasColumnType("integer"); + + b.Property("BigMapUpdateCounter") + .HasColumnType("integer"); + + b.Property("BlocksCount") + .HasColumnType("integer"); + + b.Property("CommitmentsCount") + .HasColumnType("integer"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("CyclesCount") + .HasColumnType("integer"); + + b.Property("DelegationOpsCount") + .HasColumnType("integer"); + + b.Property("DoubleBakingOpsCount") + .HasColumnType("integer"); + + b.Property("DoubleEndorsingOpsCount") + .HasColumnType("integer"); + + b.Property("EndorsementOpsCount") + .HasColumnType("integer"); + + b.Property("Hash") + .HasColumnType("text"); + + b.Property("KnownHead") + .HasColumnType("integer"); + + b.Property("LastSync") + .HasColumnType("timestamp without time zone"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("ManagerCounter") + .HasColumnType("integer"); + + b.Property("MigrationOpsCount") + .HasColumnType("integer"); + + b.Property("NextProtocol") + .HasColumnType("text"); + + b.Property("NonceRevelationOpsCount") + .HasColumnType("integer"); + + b.Property("OperationCounter") + .HasColumnType("integer"); + + b.Property("OriginationOpsCount") + .HasColumnType("integer"); + + b.Property("ProposalOpsCount") + .HasColumnType("integer"); + + b.Property("ProposalsCount") + .HasColumnType("integer"); + + b.Property("Protocol") + .HasColumnType("text"); + + b.Property("ProtocolsCount") + .HasColumnType("integer"); + + b.Property("QuoteBtc") + .HasColumnType("double precision"); + + b.Property("QuoteCny") + .HasColumnType("double precision"); + + b.Property("QuoteEth") + .HasColumnType("double precision"); + + b.Property("QuoteEur") + .HasColumnType("double precision"); + + b.Property("QuoteJpy") + .HasColumnType("double precision"); + + b.Property("QuoteKrw") + .HasColumnType("double precision"); + + b.Property("QuoteLevel") + .HasColumnType("integer"); + + b.Property("QuoteUsd") + .HasColumnType("double precision"); + + b.Property("RevealOpsCount") + .HasColumnType("integer"); + + b.Property("RevelationPenaltyOpsCount") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("TransactionOpsCount") + .HasColumnType("integer"); + + b.Property("VotingEpoch") + .HasColumnType("integer"); + + b.Property("VotingPeriod") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("AppState"); + + b.HasData( + new + { + Id = -1, + AccountCounter = 0, + AccountsCount = 0, + ActivationOpsCount = 0, + BallotOpsCount = 0, + BigMapCounter = 0, + BigMapKeyCounter = 0, + BigMapUpdateCounter = 0, + BlocksCount = 0, + CommitmentsCount = 0, + Cycle = -1, + CyclesCount = 0, + DelegationOpsCount = 0, + DoubleBakingOpsCount = 0, + DoubleEndorsingOpsCount = 0, + EndorsementOpsCount = 0, + Hash = "", + KnownHead = 0, + LastSync = new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + Level = -1, + ManagerCounter = 0, + MigrationOpsCount = 0, + NextProtocol = "", + NonceRevelationOpsCount = 0, + OperationCounter = 0, + OriginationOpsCount = 0, + ProposalOpsCount = 0, + ProposalsCount = 0, + Protocol = "", + ProtocolsCount = 0, + QuoteBtc = 0.0, + QuoteCny = 0.0, + QuoteEth = 0.0, + QuoteEur = 0.0, + QuoteJpy = 0.0, + QuoteKrw = 0.0, + QuoteLevel = -1, + QuoteUsd = 0.0, + RevealOpsCount = 0, + RevelationPenaltyOpsCount = 0, + Timestamp = new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified), + TransactionOpsCount = 0, + VotingEpoch = -1, + VotingPeriod = -1 + }); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BakerCycle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("BlockDeposits") + .HasColumnType("bigint"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("DelegatedBalance") + .HasColumnType("bigint"); + + b.Property("DelegatorsCount") + .HasColumnType("integer"); + + b.Property("DoubleBakingLostDeposits") + .HasColumnType("bigint"); + + b.Property("DoubleBakingLostFees") + .HasColumnType("bigint"); + + b.Property("DoubleBakingLostRewards") + .HasColumnType("bigint"); + + b.Property("DoubleBakingRewards") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingLostDeposits") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingLostFees") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingLostRewards") + .HasColumnType("bigint"); + + b.Property("DoubleEndorsingRewards") + .HasColumnType("bigint"); + + b.Property("EndorsementDeposits") + .HasColumnType("bigint"); + + b.Property("EndorsementRewards") + .HasColumnType("bigint"); + + b.Property("Endorsements") + .HasColumnType("integer"); + + b.Property("ExpectedBlocks") + .HasColumnType("double precision"); + + b.Property("ExpectedEndorsements") + .HasColumnType("double precision"); + + b.Property("ExtraBlockFees") + .HasColumnType("bigint"); + + b.Property("ExtraBlockRewards") + .HasColumnType("bigint"); + + b.Property("ExtraBlocks") + .HasColumnType("integer"); + + b.Property("FutureBlockDeposits") + .HasColumnType("bigint"); + + b.Property("FutureBlockRewards") + .HasColumnType("bigint"); + + b.Property("FutureBlocks") + .HasColumnType("integer"); + + b.Property("FutureEndorsementDeposits") + .HasColumnType("bigint"); + + b.Property("FutureEndorsementRewards") + .HasColumnType("bigint"); + + b.Property("FutureEndorsements") + .HasColumnType("integer"); + + b.Property("MissedEndorsementRewards") + .HasColumnType("bigint"); + + b.Property("MissedEndorsements") + .HasColumnType("integer"); + + b.Property("MissedExtraBlockFees") + .HasColumnType("bigint"); + + b.Property("MissedExtraBlockRewards") + .HasColumnType("bigint"); + + b.Property("MissedExtraBlocks") + .HasColumnType("integer"); + + b.Property("MissedOwnBlockFees") + .HasColumnType("bigint"); + + b.Property("MissedOwnBlockRewards") + .HasColumnType("bigint"); + + b.Property("MissedOwnBlocks") + .HasColumnType("integer"); + + b.Property("OwnBlockFees") + .HasColumnType("bigint"); + + b.Property("OwnBlockRewards") + .HasColumnType("bigint"); + + b.Property("OwnBlocks") + .HasColumnType("integer"); + + b.Property("RevelationLostFees") + .HasColumnType("bigint"); + + b.Property("RevelationLostRewards") + .HasColumnType("bigint"); + + b.Property("RevelationRewards") + .HasColumnType("bigint"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("StakingBalance") + .HasColumnType("bigint"); + + b.Property("UncoveredEndorsementRewards") + .HasColumnType("bigint"); + + b.Property("UncoveredEndorsements") + .HasColumnType("integer"); + + b.Property("UncoveredExtraBlockFees") + .HasColumnType("bigint"); + + b.Property("UncoveredExtraBlockRewards") + .HasColumnType("bigint"); + + b.Property("UncoveredExtraBlocks") + .HasColumnType("integer"); + + b.Property("UncoveredOwnBlockFees") + .HasColumnType("bigint"); + + b.Property("UncoveredOwnBlockRewards") + .HasColumnType("bigint"); + + b.Property("UncoveredOwnBlocks") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Cycle"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Cycle", "BakerId") + .IsUnique(); + + b.ToTable("BakerCycles"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BakingRight", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Priority") + .HasColumnType("integer"); + + b.Property("Slots") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("Type") + .HasColumnType("smallint"); + + b.HasKey("Id"); + + b.HasIndex("Cycle"); + + b.HasIndex("Level"); + + b.HasIndex("Cycle", "BakerId"); + + b.ToTable("BakingRights"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BallotOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Period") + .HasColumnType("integer"); + + b.Property("ProposalId") + .HasColumnType("integer"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("Vote") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Epoch"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("Period"); + + b.HasIndex("ProposalId"); + + b.HasIndex("SenderId"); + + b.ToTable("BallotOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMap", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Active") + .HasColumnType("boolean"); + + b.Property("ActiveKeys") + .HasColumnType("integer"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("KeyType") + .HasColumnType("bytea"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Ptr") + .HasColumnType("integer"); + + b.Property("StoragePath") + .HasColumnType("text"); + + b.Property("Tags") + .HasColumnType("integer"); + + b.Property("TotalKeys") + .HasColumnType("integer"); + + b.Property("Updates") + .HasColumnType("integer"); + + b.Property("ValueType") + .HasColumnType("bytea"); + + b.HasKey("Id"); + + b.HasAlternateKey("Ptr"); + + b.HasIndex("ContractId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Ptr") + .IsUnique(); + + b.ToTable("BigMaps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMapKey", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Active") + .HasColumnType("boolean"); + + b.Property("BigMapPtr") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("JsonKey") + .HasColumnType("jsonb"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("KeyHash") + .HasMaxLength(54) + .HasColumnType("character varying(54)"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("RawKey") + .HasColumnType("bytea"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("Updates") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BigMapPtr"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("JsonKey") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + + b.HasIndex("JsonValue") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + + b.HasIndex("LastLevel"); + + b.HasIndex("BigMapPtr", "Active") + .HasFilter("\"Active\" = true"); + + b.HasIndex("BigMapPtr", "KeyHash"); + + b.ToTable("BigMapKeys"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BigMapUpdate", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("BigMapKeyId") + .HasColumnType("integer"); + + b.Property("BigMapPtr") + .HasColumnType("integer"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("TransactionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BigMapKeyId") + .HasFilter("\"BigMapKeyId\" is not null"); + + b.HasIndex("BigMapPtr"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("OriginationId") + .HasFilter("\"OriginationId\" is not null"); + + b.HasIndex("TransactionId") + .HasFilter("\"TransactionId\" is not null"); + + b.ToTable("BigMapUpdates"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Block", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Deposit") + .HasColumnType("bigint"); + + b.Property("Events") + .HasColumnType("integer"); + + b.Property("Fees") + .HasColumnType("bigint"); + + b.Property("Hash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Operations") + .HasColumnType("integer"); + + b.Property("Priority") + .HasColumnType("integer"); + + b.Property("ProtoCode") + .HasColumnType("integer"); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("RevelationId") + .HasColumnType("integer"); + + b.Property("Reward") + .HasColumnType("bigint"); + + b.Property("SoftwareId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("Validations") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Hash") + .IsUnique(); + + b.HasIndex("Level") + .IsUnique(); + + b.HasIndex("ProtoCode"); + + b.HasIndex("RevelationId") + .IsUnique(); + + b.HasIndex("SoftwareId"); + + b.ToTable("Blocks"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Commitment", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("Address") + .IsRequired() + .HasMaxLength(37) + .HasColumnType("character(37)") + .IsFixedLength(true); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Address") + .IsUnique(); + + b.HasIndex("Id") + .IsUnique(); + + b.ToTable("Commitments"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Cycle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("Index") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Seed") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character(64)") + .IsFixedLength(true); + + b.Property("SnapshotIndex") + .HasColumnType("integer"); + + b.Property("SnapshotLevel") + .HasColumnType("integer"); + + b.Property("TotalBakers") + .HasColumnType("integer"); + + b.Property("TotalDelegated") + .HasColumnType("bigint"); + + b.Property("TotalDelegators") + .HasColumnType("integer"); + + b.Property("TotalRolls") + .HasColumnType("integer"); + + b.Property("TotalStaking") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasAlternateKey("Index"); + + b.HasIndex("Index") + .IsUnique(); + + b.ToTable("Cycles"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DelegationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("Amount") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Nonce") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("PrevDelegateId") + .HasColumnType("integer"); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("DelegateId"); + + b.HasIndex("InitiatorId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("PrevDelegateId"); + + b.HasIndex("SenderId"); + + b.ToTable("DelegationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DelegatorCycle", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("DelegatorId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Cycle"); + + b.HasIndex("DelegatorId"); + + b.HasIndex("Cycle", "BakerId"); + + b.HasIndex("Cycle", "DelegatorId") + .IsUnique(); + + b.ToTable("DelegatorCycles"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleBakingOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccusedLevel") + .HasColumnType("integer"); + + b.Property("AccuserId") + .HasColumnType("integer"); + + b.Property("AccuserReward") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OffenderId") + .HasColumnType("integer"); + + b.Property("OffenderLostDeposit") + .HasColumnType("bigint"); + + b.Property("OffenderLostFee") + .HasColumnType("bigint"); + + b.Property("OffenderLostReward") + .HasColumnType("bigint"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccuserId"); + + b.HasIndex("Level"); + + b.HasIndex("OffenderId"); + + b.HasIndex("OpHash"); + + b.ToTable("DoubleBakingOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleEndorsingOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccusedLevel") + .HasColumnType("integer"); + + b.Property("AccuserId") + .HasColumnType("integer"); + + b.Property("AccuserReward") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OffenderId") + .HasColumnType("integer"); + + b.Property("OffenderLostDeposit") + .HasColumnType("bigint"); + + b.Property("OffenderLostFee") + .HasColumnType("bigint"); + + b.Property("OffenderLostReward") + .HasColumnType("bigint"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccuserId"); + + b.HasIndex("Level"); + + b.HasIndex("OffenderId"); + + b.HasIndex("OpHash"); + + b.ToTable("DoubleEndorsingOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.EndorsementOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Deposit") + .HasColumnType("bigint"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("Reward") + .HasColumnType("bigint"); + + b.Property("Slots") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("DelegateId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.ToTable("EndorsementOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.MigrationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("BalanceChange") + .HasColumnType("bigint"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("NewScriptId") + .HasColumnType("integer"); + + b.Property("NewStorageId") + .HasColumnType("integer"); + + b.Property("OldScriptId") + .HasColumnType("integer"); + + b.Property("OldStorageId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("AccountId"); + + b.HasIndex("Level"); + + b.HasIndex("NewScriptId"); + + b.HasIndex("NewStorageId"); + + b.HasIndex("OldScriptId"); + + b.HasIndex("OldStorageId"); + + b.ToTable("MigrationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.NonceRevelationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("RevealedLevel") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("SenderId"); + + b.ToTable("NonceRevelationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.OriginationOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("BigMapUpdates") + .HasColumnType("integer"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("ManagerId") + .HasColumnType("integer"); + + b.Property("Nonce") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("ScriptId") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageId") + .HasColumnType("integer"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("ContractId"); + + b.HasIndex("DelegateId"); + + b.HasIndex("InitiatorId"); + + b.HasIndex("Level"); + + b.HasIndex("ManagerId"); + + b.HasIndex("OpHash"); + + b.HasIndex("ScriptId"); + + b.HasIndex("SenderId"); + + b.HasIndex("StorageId"); + + b.ToTable("OriginationOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Proposal", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("FirstPeriod") + .HasColumnType("integer"); + + b.Property("Hash") + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("LastPeriod") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Upvotes") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Epoch"); + + b.HasIndex("Hash"); + + b.ToTable("Proposals"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ProposalOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Duplicated") + .HasColumnType("boolean"); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("Period") + .HasColumnType("integer"); + + b.Property("ProposalId") + .HasColumnType("integer"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("Epoch"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("Period"); + + b.HasIndex("ProposalId"); + + b.HasIndex("SenderId"); + + b.ToTable("ProposalOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Protocol", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BallotQuorumMax") + .HasColumnType("integer"); + + b.Property("BallotQuorumMin") + .HasColumnType("integer"); + + b.Property("BlockDeposit") + .HasColumnType("bigint"); + + b.Property("BlockReward0") + .HasColumnType("bigint"); + + b.Property("BlockReward1") + .HasColumnType("bigint"); + + b.Property("BlocksPerCommitment") + .HasColumnType("integer"); + + b.Property("BlocksPerCycle") + .HasColumnType("integer"); + + b.Property("BlocksPerSnapshot") + .HasColumnType("integer"); + + b.Property("BlocksPerVoting") + .HasColumnType("integer"); + + b.Property("ByteCost") + .HasColumnType("integer"); + + b.Property("Code") + .HasColumnType("integer"); + + b.Property("EndorsementDeposit") + .HasColumnType("bigint"); + + b.Property("EndorsementReward0") + .HasColumnType("bigint"); + + b.Property("EndorsementReward1") + .HasColumnType("bigint"); + + b.Property("EndorsersPerBlock") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("HardBlockGasLimit") + .HasColumnType("integer"); + + b.Property("HardOperationGasLimit") + .HasColumnType("integer"); + + b.Property("HardOperationStorageLimit") + .HasColumnType("integer"); + + b.Property("Hash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("NoRewardCycles") + .HasColumnType("integer"); + + b.Property("OriginationSize") + .HasColumnType("integer"); + + b.Property("PreservedCycles") + .HasColumnType("integer"); + + b.Property("ProposalQuorum") + .HasColumnType("integer"); + + b.Property("RampUpCycles") + .HasColumnType("integer"); + + b.Property("RevelationReward") + .HasColumnType("bigint"); + + b.Property("TimeBetweenBlocks") + .HasColumnType("integer"); + + b.Property("TokensPerRoll") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.ToTable("Protocols"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Quote", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Btc") + .HasColumnType("double precision"); + + b.Property("Cny") + .HasColumnType("double precision"); + + b.Property("Eth") + .HasColumnType("double precision"); + + b.Property("Eur") + .HasColumnType("double precision"); + + b.Property("Jpy") + .HasColumnType("double precision"); + + b.Property("Krw") + .HasColumnType("double precision"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.Property("Usd") + .HasColumnType("double precision"); + + b.HasKey("Id"); + + b.HasIndex("Level") + .IsUnique(); + + b.ToTable("Quotes"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevealOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("SenderId"); + + b.ToTable("RevealOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevelationPenaltyOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("LostFees") + .HasColumnType("bigint"); + + b.Property("LostReward") + .HasColumnType("bigint"); + + b.Property("MissedLevel") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("BakerId"); + + b.HasIndex("Level"); + + b.ToTable("RevelationPenaltyOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Script", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("CodeSchema") + .HasColumnType("bytea"); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("Current") + .HasColumnType("boolean"); + + b.Property("MigrationId") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("ParameterSchema") + .HasColumnType("bytea"); + + b.Property("StorageSchema") + .HasColumnType("bytea"); + + b.HasKey("Id"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("ContractId", "Current") + .HasFilter("\"Current\" = true"); + + b.ToTable("Scripts"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.SnapshotBalance", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AccountId") + .HasColumnType("integer"); + + b.Property("Balance") + .HasColumnType("bigint"); + + b.Property("DelegateId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Level"); + + b.ToTable("SnapshotBalances"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Software", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BlocksCount") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("Metadata") + .HasColumnType("jsonb"); + + b.Property("ShortHash") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character(8)") + .IsFixedLength(true); + + b.HasKey("Id"); + + b.ToTable("Software"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Statistics", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("Cycle") + .HasColumnType("integer"); + + b.Property("Date") + .HasColumnType("timestamp without time zone"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("TotalActivated") + .HasColumnType("bigint"); + + b.Property("TotalBootstrapped") + .HasColumnType("bigint"); + + b.Property("TotalBurned") + .HasColumnType("bigint"); + + b.Property("TotalCommitments") + .HasColumnType("bigint"); + + b.Property("TotalCreated") + .HasColumnType("bigint"); + + b.Property("TotalFrozen") + .HasColumnType("bigint"); + + b.Property("TotalVested") + .HasColumnType("bigint"); + + b.HasKey("Id"); + + b.HasIndex("Cycle") + .IsUnique() + .HasFilter("\"Cycle\" IS NOT NULL"); + + b.HasIndex("Date") + .IsUnique() + .HasFilter("\"Date\" IS NOT NULL"); + + b.HasIndex("Level") + .IsUnique(); + + b.ToTable("Statistics"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Storage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("ContractId") + .HasColumnType("integer"); + + b.Property("Current") + .HasColumnType("boolean"); + + b.Property("JsonValue") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("MigrationId") + .HasColumnType("integer"); + + b.Property("OriginationId") + .HasColumnType("integer"); + + b.Property("RawValue") + .HasColumnType("bytea"); + + b.Property("TransactionId") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("ContractId"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Level"); + + b.HasIndex("ContractId", "Current") + .HasFilter("\"Current\" = true"); + + b.ToTable("Storages"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.TransactionOperation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("AllocationFee") + .HasColumnType("bigint"); + + b.Property("Amount") + .HasColumnType("bigint"); + + b.Property("BakerFee") + .HasColumnType("bigint"); + + b.Property("BigMapUpdates") + .HasColumnType("integer"); + + b.Property("Counter") + .HasColumnType("integer"); + + b.Property("Entrypoint") + .HasColumnType("text"); + + b.Property("Errors") + .HasColumnType("text"); + + b.Property("GasLimit") + .HasColumnType("integer"); + + b.Property("GasUsed") + .HasColumnType("integer"); + + b.Property("InitiatorId") + .HasColumnType("integer"); + + b.Property("InternalDelegations") + .HasColumnType("smallint"); + + b.Property("InternalOperations") + .HasColumnType("smallint"); + + b.Property("InternalOriginations") + .HasColumnType("smallint"); + + b.Property("InternalTransactions") + .HasColumnType("smallint"); + + b.Property("JsonParameters") + .HasColumnType("jsonb"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Nonce") + .HasColumnType("integer"); + + b.Property("OpHash") + .IsRequired() + .HasMaxLength(51) + .HasColumnType("character(51)") + .IsFixedLength(true); + + b.Property("RawParameters") + .HasColumnType("bytea"); + + b.Property("ResetDeactivation") + .HasColumnType("integer"); + + b.Property("SenderId") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("smallint"); + + b.Property("StorageFee") + .HasColumnType("bigint"); + + b.Property("StorageId") + .HasColumnType("integer"); + + b.Property("StorageLimit") + .HasColumnType("integer"); + + b.Property("StorageUsed") + .HasColumnType("integer"); + + b.Property("TargetId") + .HasColumnType("integer"); + + b.Property("Timestamp") + .HasColumnType("timestamp without time zone"); + + b.HasKey("Id"); + + b.HasIndex("InitiatorId"); + + b.HasIndex("JsonParameters") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + + b.HasIndex("Level"); + + b.HasIndex("OpHash"); + + b.HasIndex("SenderId"); + + b.HasIndex("StorageId"); + + b.HasIndex("TargetId"); + + b.ToTable("TransactionOps"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.VotingPeriod", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BallotsQuorum") + .HasColumnType("integer"); + + b.Property("Epoch") + .HasColumnType("integer"); + + b.Property("FirstLevel") + .HasColumnType("integer"); + + b.Property("Index") + .HasColumnType("integer"); + + b.Property("Kind") + .HasColumnType("integer"); + + b.Property("LastLevel") + .HasColumnType("integer"); + + b.Property("NayBallots") + .HasColumnType("integer"); + + b.Property("NayRolls") + .HasColumnType("integer"); + + b.Property("ParticipationEma") + .HasColumnType("integer"); + + b.Property("PassBallots") + .HasColumnType("integer"); + + b.Property("PassRolls") + .HasColumnType("integer"); + + b.Property("ProposalsCount") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("Supermajority") + .HasColumnType("integer"); + + b.Property("TopRolls") + .HasColumnType("integer"); + + b.Property("TopUpvotes") + .HasColumnType("integer"); + + b.Property("TotalBakers") + .HasColumnType("integer"); + + b.Property("TotalRolls") + .HasColumnType("integer"); + + b.Property("UpvotesQuorum") + .HasColumnType("integer"); + + b.Property("YayBallots") + .HasColumnType("integer"); + + b.Property("YayRolls") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasAlternateKey("Index"); + + b.HasIndex("Epoch"); + + b.HasIndex("Id") + .IsUnique(); + + b.HasIndex("Index") + .IsUnique(); + + b.ToTable("VotingPeriods"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.VotingSnapshot", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + + b.Property("BakerId") + .HasColumnType("integer"); + + b.Property("Level") + .HasColumnType("integer"); + + b.Property("Period") + .HasColumnType("integer"); + + b.Property("Rolls") + .HasColumnType("integer"); + + b.Property("Status") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Period"); + + b.HasIndex("Period", "BakerId") + .IsUnique(); + + b.ToTable("VotingSnapshots"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Contract", b => + { + b.HasBaseType("Tzkt.Data.Models.Account"); + + b.Property("CreatorId") + .HasColumnType("integer"); + + b.Property("Kind") + .HasColumnType("smallint"); + + b.Property("ManagerId") + .HasColumnType("integer"); + + b.Property("Spendable") + .HasColumnType("boolean"); + + b.Property("Tzips") + .HasColumnType("integer"); + + b.Property("WeirdDelegateId") + .HasColumnType("integer"); + + b.HasIndex("CreatorId"); + + b.HasIndex("ManagerId"); + + b.HasIndex("WeirdDelegateId"); + + b.HasIndex("Type", "Kind") + .HasFilter("\"Type\" = 2"); + + b.HasDiscriminator().HasValue((byte)2); + }); + + modelBuilder.Entity("Tzkt.Data.Models.User", b => + { + b.HasBaseType("Tzkt.Data.Models.Account"); + + b.Property("Activated") + .HasColumnType("boolean"); + + b.Property("PublicKey") + .HasMaxLength(55) + .HasColumnType("character varying(55)"); + + b.Property("Revealed") + .HasColumnType("boolean"); + + b.HasDiscriminator().HasValue((byte)0); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Delegate", b => + { + b.HasBaseType("Tzkt.Data.Models.User"); + + b.Property("ActivationLevel") + .HasColumnType("integer"); + + b.Property("BallotsCount") + .HasColumnType("integer"); + + b.Property("BlocksCount") + .HasColumnType("integer"); + + b.Property("DeactivationLevel") + .HasColumnType("integer"); + + b.Property("DelegatorsCount") + .HasColumnType("integer"); + + b.Property("DoubleBakingCount") + .HasColumnType("integer"); + + b.Property("DoubleEndorsingCount") + .HasColumnType("integer"); + + b.Property("EndorsementsCount") + .HasColumnType("integer"); + + b.Property("FrozenDeposits") + .HasColumnType("bigint"); + + b.Property("FrozenFees") + .HasColumnType("bigint"); + + b.Property("FrozenRewards") + .HasColumnType("bigint"); + + b.Property("NonceRevelationsCount") + .HasColumnType("integer"); + + b.Property("ProposalsCount") + .HasColumnType("integer"); + + b.Property("RevelationPenaltiesCount") + .HasColumnType("integer"); + + b.Property("SoftwareId") + .HasColumnType("integer"); + + b.Property("StakingBalance") + .HasColumnType("bigint"); + + b.HasIndex("SoftwareId"); + + b.HasIndex("Type", "Staked") + .HasFilter("\"Type\" = 1"); + + b.HasDiscriminator().HasValue((byte)1); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Account", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany("DelegatedAccounts") + .HasForeignKey("DelegateId"); + + b.HasOne("Tzkt.Data.Models.Block", "FirstBlock") + .WithMany("CreatedAccounts") + .HasForeignKey("FirstLevel") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Delegate"); + + b.Navigation("FirstBlock"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ActivationOperation", b => + { + b.HasOne("Tzkt.Data.Models.User", "Account") + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Activations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Account"); + + b.Navigation("Block"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.BallotOperation", b => + { + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Ballots") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Proposal", "Proposal") + .WithMany() + .HasForeignKey("ProposalId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Proposal"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Block", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Baker") + .WithMany() + .HasForeignKey("BakerId"); + + b.HasOne("Tzkt.Data.Models.Protocol", "Protocol") + .WithMany() + .HasForeignKey("ProtoCode") + .HasPrincipalKey("Code") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.NonceRevelationOperation", "Revelation") + .WithOne("RevealedBlock") + .HasForeignKey("Tzkt.Data.Models.Block", "RevelationId") + .HasPrincipalKey("Tzkt.Data.Models.NonceRevelationOperation", "RevealedLevel"); + + b.HasOne("Tzkt.Data.Models.Software", "Software") + .WithMany() + .HasForeignKey("SoftwareId"); + + b.Navigation("Baker"); + + b.Navigation("Protocol"); + + b.Navigation("Revelation"); + + b.Navigation("Software"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DelegationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany() + .HasForeignKey("DelegateId"); + + b.HasOne("Tzkt.Data.Models.Account", "Initiator") + .WithMany() + .HasForeignKey("InitiatorId"); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Delegations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "PrevDelegate") + .WithMany() + .HasForeignKey("PrevDelegateId"); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Delegate"); + + b.Navigation("Initiator"); + + b.Navigation("PrevDelegate"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleBakingOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Accuser") + .WithMany() + .HasForeignKey("AccuserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("DoubleBakings") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Offender") + .WithMany() + .HasForeignKey("OffenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Accuser"); + + b.Navigation("Block"); + + b.Navigation("Offender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.DoubleEndorsingOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Accuser") + .WithMany() + .HasForeignKey("AccuserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("DoubleEndorsings") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Offender") + .WithMany() + .HasForeignKey("OffenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Accuser"); + + b.Navigation("Block"); + + b.Navigation("Offender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.EndorsementOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany() + .HasForeignKey("DelegateId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Endorsements") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Delegate"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.MigrationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Account", "Account") + .WithMany() + .HasForeignKey("AccountId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Migrations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Script", "NewScript") + .WithMany() + .HasForeignKey("NewScriptId"); + + b.HasOne("Tzkt.Data.Models.Storage", "NewStorage") + .WithMany() + .HasForeignKey("NewStorageId"); + + b.HasOne("Tzkt.Data.Models.Script", "OldScript") + .WithMany() + .HasForeignKey("OldScriptId"); + + b.HasOne("Tzkt.Data.Models.Storage", "OldStorage") + .WithMany() + .HasForeignKey("OldStorageId"); + + b.Navigation("Account"); + + b.Navigation("Block"); + + b.Navigation("NewScript"); + + b.Navigation("NewStorage"); + + b.Navigation("OldScript"); + + b.Navigation("OldStorage"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.NonceRevelationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Baker") + .WithMany() + .HasForeignKey("BakerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Revelations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Baker"); + + b.Navigation("Block"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.OriginationOperation", b => + { + b.HasOne("Tzkt.Data.Models.Contract", "Contract") + .WithMany() + .HasForeignKey("ContractId"); + + b.HasOne("Tzkt.Data.Models.Delegate", "Delegate") + .WithMany() + .HasForeignKey("DelegateId"); + + b.HasOne("Tzkt.Data.Models.Account", "Initiator") + .WithMany() + .HasForeignKey("InitiatorId"); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Originations") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.User", "Manager") + .WithMany() + .HasForeignKey("ManagerId"); + + b.HasOne("Tzkt.Data.Models.Script", "Script") + .WithMany() + .HasForeignKey("ScriptId"); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Storage", "Storage") + .WithMany() + .HasForeignKey("StorageId"); + + b.Navigation("Block"); + + b.Navigation("Contract"); + + b.Navigation("Delegate"); + + b.Navigation("Initiator"); + + b.Navigation("Manager"); + + b.Navigation("Script"); + + b.Navigation("Sender"); + + b.Navigation("Storage"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.ProposalOperation", b => + { + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Proposals") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Proposal", "Proposal") + .WithMany() + .HasForeignKey("ProposalId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Delegate", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Proposal"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevealOperation", b => + { + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Reveals") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Block"); + + b.Navigation("Sender"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.RevelationPenaltyOperation", b => + { + b.HasOne("Tzkt.Data.Models.Delegate", "Baker") + .WithMany() + .HasForeignKey("BakerId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("RevelationPenalties") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Baker"); + + b.Navigation("Block"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.TransactionOperation", b => + { + b.HasOne("Tzkt.Data.Models.Account", "Initiator") + .WithMany() + .HasForeignKey("InitiatorId"); + + b.HasOne("Tzkt.Data.Models.Block", "Block") + .WithMany("Transactions") + .HasForeignKey("Level") + .HasPrincipalKey("Level") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Account", "Sender") + .WithMany() + .HasForeignKey("SenderId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("Tzkt.Data.Models.Storage", "Storage") + .WithMany() + .HasForeignKey("StorageId"); + + b.HasOne("Tzkt.Data.Models.Account", "Target") + .WithMany() + .HasForeignKey("TargetId"); + + b.Navigation("Block"); + + b.Navigation("Initiator"); + + b.Navigation("Sender"); + + b.Navigation("Storage"); + + b.Navigation("Target"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Contract", b => + { + b.HasOne("Tzkt.Data.Models.Account", "Creator") + .WithMany() + .HasForeignKey("CreatorId"); + + b.HasOne("Tzkt.Data.Models.User", "Manager") + .WithMany() + .HasForeignKey("ManagerId"); + + b.HasOne("Tzkt.Data.Models.User", "WeirdDelegate") + .WithMany() + .HasForeignKey("WeirdDelegateId"); + + b.Navigation("Creator"); + + b.Navigation("Manager"); + + b.Navigation("WeirdDelegate"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Delegate", b => + { + b.HasOne("Tzkt.Data.Models.Software", "Software") + .WithMany() + .HasForeignKey("SoftwareId"); + + b.Navigation("Software"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Block", b => + { + b.Navigation("Activations"); + + b.Navigation("Ballots"); + + b.Navigation("CreatedAccounts"); + + b.Navigation("Delegations"); + + b.Navigation("DoubleBakings"); + + b.Navigation("DoubleEndorsings"); + + b.Navigation("Endorsements"); + + b.Navigation("Migrations"); + + b.Navigation("Originations"); + + b.Navigation("Proposals"); + + b.Navigation("Reveals"); + + b.Navigation("RevelationPenalties"); + + b.Navigation("Revelations"); + + b.Navigation("Transactions"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.NonceRevelationOperation", b => + { + b.Navigation("RevealedBlock"); + }); + + modelBuilder.Entity("Tzkt.Data.Models.Delegate", b => + { + b.Navigation("DelegatedAccounts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Tzkt.Data/Migrations/20210412164322_JsonIndexes.cs b/Tzkt.Data/Migrations/20210412164322_JsonIndexes.cs new file mode 100644 index 000000000..549e8548a --- /dev/null +++ b/Tzkt.Data/Migrations/20210412164322_JsonIndexes.cs @@ -0,0 +1,69 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace Tzkt.Data.Migrations +{ + public partial class JsonIndexes : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Accounts_Metadata", + table: "Accounts"); + + migrationBuilder.CreateIndex( + name: "IX_TransactionOps_JsonParameters", + table: "TransactionOps", + column: "JsonParameters") + .Annotation("Npgsql:IndexMethod", "gin") + .Annotation("Npgsql:IndexOperators", new[] { "jsonb_path_ops" }); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_JsonKey", + table: "BigMapKeys", + column: "JsonKey") + .Annotation("Npgsql:IndexMethod", "gin") + .Annotation("Npgsql:IndexOperators", new[] { "jsonb_path_ops" }); + + migrationBuilder.CreateIndex( + name: "IX_BigMapKeys_JsonValue", + table: "BigMapKeys", + column: "JsonValue") + .Annotation("Npgsql:IndexMethod", "gin") + .Annotation("Npgsql:IndexOperators", new[] { "jsonb_path_ops" }); + + migrationBuilder.CreateIndex( + name: "IX_Accounts_Metadata", + table: "Accounts", + column: "Metadata") + .Annotation("Npgsql:IndexMethod", "gin") + .Annotation("Npgsql:IndexOperators", new[] { "jsonb_path_ops" }); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_TransactionOps_JsonParameters", + table: "TransactionOps"); + + migrationBuilder.DropIndex( + name: "IX_BigMapKeys_JsonKey", + table: "BigMapKeys"); + + migrationBuilder.DropIndex( + name: "IX_BigMapKeys_JsonValue", + table: "BigMapKeys"); + + migrationBuilder.DropIndex( + name: "IX_Accounts_Metadata", + table: "Accounts"); + + migrationBuilder.CreateIndex( + name: "IX_Accounts_Metadata", + table: "Accounts", + column: "Metadata") + .Annotation("Npgsql:IndexMethod", "GIN") + .Annotation("Npgsql:IndexOperators", new[] { "jsonb_path_ops" }); + } + } +} diff --git a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs index a86ea7b88..48f47a2f5 100644 --- a/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs +++ b/Tzkt.Data/Migrations/TzktContextModelSnapshot.cs @@ -90,7 +90,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .IsUnique(); b.HasIndex("Metadata") - .HasMethod("GIN") + .HasMethod("gin") .HasOperators(new[] { "jsonb_path_ops" }); b.HasIndex("Staked"); @@ -700,6 +700,14 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("Id") .IsUnique(); + b.HasIndex("JsonKey") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + + b.HasIndex("JsonValue") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + b.HasIndex("LastLevel"); b.HasIndex("BigMapPtr", "Active") @@ -2023,6 +2031,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.HasIndex("InitiatorId"); + b.HasIndex("JsonParameters") + .HasMethod("gin") + .HasOperators(new[] { "jsonb_path_ops" }); + b.HasIndex("Level"); b.HasIndex("OpHash"); diff --git a/Tzkt.Data/Models/Accounts/Account.cs b/Tzkt.Data/Models/Accounts/Account.cs index 397437376..70eb27ea3 100644 --- a/Tzkt.Data/Models/Accounts/Account.cs +++ b/Tzkt.Data/Models/Accounts/Account.cs @@ -65,7 +65,7 @@ public static void BuildAccountModel(this ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(x => x.Metadata) - .HasMethod("GIN") + .HasMethod("gin") .HasOperators("jsonb_path_ops"); #endregion diff --git a/Tzkt.Data/Models/Operations/TransactionOperation.cs b/Tzkt.Data/Models/Operations/TransactionOperation.cs index 29a26471a..e10fe8017 100644 --- a/Tzkt.Data/Models/Operations/TransactionOperation.cs +++ b/Tzkt.Data/Models/Operations/TransactionOperation.cs @@ -48,6 +48,11 @@ public static void BuildTransactionOperationModel(this ModelBuilder modelBuilder modelBuilder.Entity() .HasIndex(x => x.TargetId); + + modelBuilder.Entity() + .HasIndex(x => x.JsonParameters) + .HasMethod("gin") + .HasOperators("jsonb_path_ops"); #endregion #region keys diff --git a/Tzkt.Data/Models/Scripts/BigMapKey.cs b/Tzkt.Data/Models/Scripts/BigMapKey.cs index ef7428b66..08548ebaa 100644 --- a/Tzkt.Data/Models/Scripts/BigMapKey.cs +++ b/Tzkt.Data/Models/Scripts/BigMapKey.cs @@ -41,15 +41,15 @@ public static void BuildBigMapKeyModel(this ModelBuilder modelBuilder) modelBuilder.Entity() .HasIndex(x => new { x.BigMapPtr, x.KeyHash }); - //modelBuilder.Entity() - // .HasIndex(x => new { x.BigMapPtr, x.JsonKey }) - // .HasMethod("GIN") - // .HasOperators("jsonb_path_ops"); + modelBuilder.Entity() + .HasIndex(x => x.JsonKey) + .HasMethod("gin") + .HasOperators("jsonb_path_ops"); - //modelBuilder.Entity() - // .HasIndex(x => new { x.BigMapPtr, x.JsonValue }) - // .HasMethod("GIN") - // .HasOperators("jsonb_path_ops"); + modelBuilder.Entity() + .HasIndex(x => x.JsonValue) + .HasMethod("gin") + .HasOperators("jsonb_path_ops"); #endregion #region keys From 8e6762e8a54d70ded01f53baf42fbefb10705abd Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 15 Apr 2021 00:22:17 +0300 Subject: [PATCH 25/35] Sql builder params refactoring --- Tzkt.Api/Utils/SqlBuilder.cs | 320 +++++++---------------------------- 1 file changed, 64 insertions(+), 256 deletions(-) diff --git a/Tzkt.Api/Utils/SqlBuilder.cs b/Tzkt.Api/Utils/SqlBuilder.cs index f85925574..c0d5315bf 100644 --- a/Tzkt.Api/Utils/SqlBuilder.cs +++ b/Tzkt.Api/Utils/SqlBuilder.cs @@ -115,16 +115,10 @@ public SqlBuilder Filter(string column, ContractKindParameter kind) AppendFilter($@"""{column}"" != {kind.Ne}"); if (kind.In != null) - { - AppendFilter($@"""{column}"" = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", kind.In); - } + AppendFilter($@"""{column}"" = ANY ({Param(kind.In)})"); if (kind.Ni != null && kind.Ni.Count > 0) - { - AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", kind.Ni); - } + AppendFilter($@"NOT (""{column}"" = ANY ({Param(kind.Ni)}))"); return this; } @@ -140,16 +134,10 @@ public SqlBuilder Filter(string column, BigMapActionParameter action) AppendFilter($@"""{column}"" != {action.Ne}"); if (action.In != null) - { - AppendFilter($@"""{column}"" = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", action.In); - } + AppendFilter($@"""{column}"" = ANY ({Param(action.In)})"); if (action.Ni != null && action.Ni.Count > 0) - { - AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", action.Ni); - } + AppendFilter($@"NOT (""{column}"" = ANY ({Param(action.Ni)}))"); return this; } @@ -181,16 +169,10 @@ public SqlBuilder Filter(string column, MigrationKindParameter kind) AppendFilter($@"""{column}"" != {kind.Ne}"); if (kind.In != null) - { - AppendFilter($@"""{column}"" = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", kind.In); - } + AppendFilter($@"""{column}"" = ANY ({Param(kind.In)})"); if (kind.Ni != null && kind.Ni.Count > 0) - { - AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", kind.Ni); - } + AppendFilter($@"NOT (""{column}"" = ANY ({Param(kind.Ni)}))"); return this; } @@ -206,16 +188,10 @@ public SqlBuilder Filter(string column, VoterStatusParameter status) AppendFilter($@"""{column}"" != {status.Ne}"); if (status.In != null) - { - AppendFilter($@"""{column}"" = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", status.In); - } + AppendFilter($@"""{column}"" = ANY ({Param(status.In)})"); if (status.Ni != null && status.Ni.Count > 0) - { - AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", status.Ni); - } + AppendFilter($@"NOT (""{column}"" = ANY ({Param(status.Ni)}))"); return this; } @@ -238,28 +214,16 @@ public SqlBuilder Filter(string column, ProtocolParameter protocol) if (protocol == null) return this; if (protocol.Eq != null) - { - AppendFilter($@"""{column}"" = @p{Counter}::character(51)"); - Params.Add($"p{Counter++}", protocol.Eq); - } + AppendFilter($@"""{column}"" = {Param(protocol.Eq)}::character(51)"); if (protocol.Ne != null) - { - AppendFilter($@"""{column}"" != @p{Counter}::character(51)"); - Params.Add($"p{Counter++}", protocol.Ne); - } + AppendFilter($@"""{column}"" != {Param(protocol.Ne)}::character(51)"); if (protocol.In != null) - { - AppendFilter($@"""{column}"" = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", protocol.In); - } + AppendFilter($@"""{column}"" = ANY ({Param(protocol.In)})"); if (protocol.Ni != null && protocol.Ni.Count > 0) - { - AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", protocol.Ni); - } + AppendFilter($@"NOT (""{column}"" = ANY ({Param(protocol.Ni)}))"); return this; } @@ -269,28 +233,16 @@ public SqlBuilder FilterA(string column, ProtocolParameter protocol) if (protocol == null) return this; if (protocol.Eq != null) - { - AppendFilter($@"{column} = @p{Counter}::character(51)"); - Params.Add($"p{Counter++}", protocol.Eq); - } + AppendFilter($@"{column} = {Param(protocol.Eq)}::character(51)"); if (protocol.Ne != null) - { - AppendFilter($@"{column} != @p{Counter}::character(51)"); - Params.Add($"p{Counter++}", protocol.Ne); - } + AppendFilter($@"{column} != {Param(protocol.Ne)}::character(51)"); if (protocol.In != null) - { - AppendFilter($@"{column} = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", protocol.In); - } + AppendFilter($@"{column} = ANY ({Param(protocol.In)})"); if (protocol.Ni != null && protocol.Ni.Count > 0) - { - AppendFilter($@"NOT ({column} = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", protocol.Ni); - } + AppendFilter($@"NOT ({column} = ANY ({Param(protocol.Ni)}))"); return this; } @@ -306,16 +258,10 @@ public SqlBuilder Filter(string column, AccountParameter account, Func 0) - { - AppendFilter($@"(""{column}"" IS NULL OR NOT (""{column}"" = ANY (@p{Counter})))"); - Params.Add($"p{Counter++}", account.Ni); - } + AppendFilter($@"(""{column}"" IS NULL OR NOT (""{column}"" = ANY ({Param(account.Ni)})))"); if (account.Eqx != null && map != null) AppendFilter($@"""{column}"" = ""{map(account.Eqx)}"""); @@ -344,16 +290,10 @@ public SqlBuilder FilterA(string column, AccountParameter account, Func 0) - { - AppendFilter($"({column} IS NULL OR NOT ({column} = ANY (@p{Counter})))"); - Params.Add($"p{Counter++}", account.Ni); - } + AppendFilter($"({column} IS NULL OR NOT ({column} = ANY ({Param(account.Ni)})))"); if (account.Eqx != null && map != null) AppendFilter($"{column} = {map(account.Eqx)}"); @@ -376,40 +316,22 @@ public SqlBuilder Filter(string column, StringParameter str, Func @p{Counter}"); - Params.Add($"p{Counter++}", value.Gt); - } + AppendFilter($@"""{column}"" > {Param(value.Gt)}"); if (value.Ge != null) - { - AppendFilter($@"""{column}"" >= @p{Counter}"); - Params.Add($"p{Counter++}", value.Ge); - } + AppendFilter($@"""{column}"" >= {Param(value.Ge)}"); if (value.Lt != null) - { - AppendFilter($@"""{column}"" < @p{Counter}"); - Params.Add($"p{Counter++}", value.Lt); - } + AppendFilter($@"""{column}"" < {Param(value.Lt)}"); if (value.Le != null) - { - AppendFilter($@"""{column}"" <= @p{Counter}"); - Params.Add($"p{Counter++}", value.Le); - } + AppendFilter($@"""{column}"" <= {Param(value.Le)}"); if (value.In != null) - { - AppendFilter($@"""{column}"" = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", value.In); - } + AppendFilter($@"""{column}"" = ANY ({Param(value.In)})"); if (value.Ni != null) - { - AppendFilter($@"NOT (""{column}"" = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", value.Ni); - } + AppendFilter($@"NOT (""{column}"" = ANY ({Param(value.Ni)}))"); return this; } @@ -1049,52 +893,28 @@ public SqlBuilder FilterA(string column, DateTimeParameter value, Func @p{Counter}"); - Params.Add($"p{Counter++}", value.Gt); - } + AppendFilter($@"{column} > {Param(value.Gt)}"); if (value.Ge != null) - { - AppendFilter($@"{column} >= @p{Counter}"); - Params.Add($"p{Counter++}", value.Ge); - } + AppendFilter($@"{column} >= {Param(value.Ge)}"); if (value.Lt != null) - { - AppendFilter($@"{column} < @p{Counter}"); - Params.Add($"p{Counter++}", value.Lt); - } + AppendFilter($@"{column} < {Param(value.Lt)}"); if (value.Le != null) - { - AppendFilter($@"{column} <= @p{Counter}"); - Params.Add($"p{Counter++}", value.Le); - } + AppendFilter($@"{column} <= {Param(value.Le)}"); if (value.In != null) - { - AppendFilter($@"{column} = ANY (@p{Counter})"); - Params.Add($"p{Counter++}", value.In); - } + AppendFilter($@"{column} = ANY ({Param(value.In)})"); if (value.Ni != null) - { - AppendFilter($@"NOT ({column} = ANY (@p{Counter}))"); - Params.Add($"p{Counter++}", value.Ni); - } + AppendFilter($@"NOT ({column} = ANY ({Param(value.Ni)}))"); return this; } @@ -1122,16 +942,10 @@ public SqlBuilder Filter(string column, TimestampParameter value, Func Date: Wed, 21 Apr 2021 18:20:02 +0300 Subject: [PATCH 26/35] Minor fixes --- Tzkt.Api/Controllers/BigMapsController.cs | 6 +++--- Tzkt.Api/Repositories/BigMapsRepository.cs | 2 +- Tzkt.Api/Swagger/Swagger.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index fa56076d5..82a727b18 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -42,8 +42,8 @@ public Task GetBigMapsCount() /// Returns a list of bigmaps. /// /// Filters bigmaps by smart contract address. - /// Filters bigmaps path in the contract storage. - /// Filters bigmaps tags (`token_metadata` - tzip-12, `metadata` - tzip-16). + /// Filters bigmaps by path in the contract storage. + /// Filters bigmaps by tags: `token_metadata` - tzip-12, `metadata` - tzip-16. /// Filters bigmaps by status: `true` - active, `false` - removed. /// Filters bigmaps by the last update level. /// Specify comma-separated list of fields to include into response or leave it undefined to return full object. If you select single field, response will be an array of values in both `.fields` and `.values` modes. @@ -104,7 +104,7 @@ public async Task>> GetBigMaps( /// Filters updates by bigmap ptr /// Filters updates by bigmap path /// Filters updates by bigmap contract - /// Filters updates by bigmap tags + /// Filters updates by bigmap tags: `token_metadata` - tzip-12, `metadata` - tzip-16 /// Filters updates by action /// Filters updates by level /// Sorts bigmaps by specified field. Supported fields: `id` (default), `ptr`, `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 4e5aca628..94e598c2e 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -1053,7 +1053,7 @@ public async Task> GetUpdates( { "level" => ("Id", "Level"), _ => ("Id", "Id") - }); + }, "uu"); using var db = GetConnection(); var updateRows = await db.QueryAsync(sql.Query, sql.Params); diff --git a/Tzkt.Api/Swagger/Swagger.cs b/Tzkt.Api/Swagger/Swagger.cs index 83057b8a9..97e1c6847 100644 --- a/Tzkt.Api/Swagger/Swagger.cs +++ b/Tzkt.Api/Swagger/Swagger.cs @@ -8,7 +8,7 @@ namespace Tzkt.Api.Swagger { public static class Swagger { - const string Version = "v1.4"; + const string Version = "v1.5"; const string Path = "/v1/swagger.json"; public static void AddOpenApiDocument(this IServiceCollection services) From 544f6bb798eef330d1d9511ed90998ab2de06e59 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Thu, 22 Apr 2021 20:32:46 +0300 Subject: [PATCH 27/35] Add bigmap events --- Tzkt.Api/Models/BigMaps/BigMapUpdate.cs | 12 + Tzkt.Api/Services/Sync/StateListener.cs | 1 + Tzkt.Api/Startup.cs | 5 +- Tzkt.Api/Websocket/Hubs/DefaultHub.cs | 12 +- .../Websocket/Parameters/BigMapsParameter.cs | 30 ++ .../Websocket/Processors/BigMapsProcessor.cs | 386 ++++++++++++++++++ Tzkt.Api/Websocket/WebsocketConfig.cs | 1 + Tzkt.Api/appsettings.json | 3 +- 8 files changed, 447 insertions(+), 3 deletions(-) create mode 100644 Tzkt.Api/Websocket/Parameters/BigMapsParameter.cs create mode 100644 Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs diff --git a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs index 15a0655f1..93fd2d0af 100644 --- a/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs +++ b/Tzkt.Api/Models/BigMaps/BigMapUpdate.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Text.Json.Serialization; using Tzkt.Data.Models; @@ -49,5 +50,16 @@ public class BigMapUpdate [JsonIgnore] public BigMapTag _Tags { get; set; } + + public IEnumerable EnumerateTags() + { + if (_Tags != BigMapTag.None) + { + if (_Tags.HasFlag(BigMapTag.Metadata)) + yield return BigMapTag.Metadata; + if (_Tags.HasFlag(BigMapTag.TokenMetadata)) + yield return BigMapTag.TokenMetadata; + } + } } } diff --git a/Tzkt.Api/Services/Sync/StateListener.cs b/Tzkt.Api/Services/Sync/StateListener.cs index 3ba17c8d8..2a6e8950a 100644 --- a/Tzkt.Api/Services/Sync/StateListener.cs +++ b/Tzkt.Api/Services/Sync/StateListener.cs @@ -68,6 +68,7 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) } catch (Exception ex) { + // TODO: reanimate listener without breaking HubProcessors state Logger.LogCritical($"State listener crashed: {ex.Message}"); } finally diff --git a/Tzkt.Api/Startup.cs b/Tzkt.Api/Startup.cs index 53d516dd9..d7470b73e 100644 --- a/Tzkt.Api/Startup.cs +++ b/Tzkt.Api/Startup.cs @@ -53,7 +53,7 @@ public void ConfigureServices(IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); - services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddTransient(); services.AddTransient(); @@ -91,6 +91,9 @@ public void ConfigureServices(IServiceCollection services) services.AddTransient>(); services.AddTransient>(); + services.AddTransient>(); + services.AddTransient>(); + services.AddSignalR(options => { options.EnableDetailedErrors = true; diff --git a/Tzkt.Api/Websocket/Hubs/DefaultHub.cs b/Tzkt.Api/Websocket/Hubs/DefaultHub.cs index 70209a9c7..2b756d9f0 100644 --- a/Tzkt.Api/Websocket/Hubs/DefaultHub.cs +++ b/Tzkt.Api/Websocket/Hubs/DefaultHub.cs @@ -11,19 +11,22 @@ public class DefaultHub : BaseHub readonly HeadProcessor Head; readonly BlocksProcessor Blocks; readonly OperationsProcessor Operations; + readonly BigMapsProcessor BigMaps; public DefaultHub( HeadProcessor head, BlocksProcessor blocks, OperationsProcessor operations, + BigMapsProcessor bigMaps, ILogger logger, IConfiguration config) : base(logger, config) { Head = head; Blocks = blocks; Operations = operations; + BigMaps = bigMaps; } - + public Task SubscribeToHead() { return Head.Subscribe(Clients.Caller, Context.ConnectionId); @@ -39,9 +42,16 @@ public Task SubscribeToOperations(OperationsParameter parameters) return Operations.Subscribe(Clients.Caller, Context.ConnectionId, parameters.Address, parameters.Types); } + public Task SubscribeToBigMaps(BigMapsParameter parameters) + { + parameters.EnsureValid(); + return BigMaps.Subscribe(Clients.Caller, Context.ConnectionId, parameters); + } + public override async Task OnDisconnectedAsync(Exception exception) { await Operations.Unsubscribe(Context.ConnectionId); + await BigMaps.Unsubscribe(Context.ConnectionId); await base.OnDisconnectedAsync(exception); } } diff --git a/Tzkt.Api/Websocket/Parameters/BigMapsParameter.cs b/Tzkt.Api/Websocket/Parameters/BigMapsParameter.cs new file mode 100644 index 000000000..e294d01a1 --- /dev/null +++ b/Tzkt.Api/Websocket/Parameters/BigMapsParameter.cs @@ -0,0 +1,30 @@ +using Microsoft.AspNetCore.SignalR; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Tzkt.Api.Websocket +{ + public class BigMapsParameter + { + public int? Ptr { get; set; } + public string Path { get; set; } + public string Contract { get; set; } + public List Tags { get; set; } + + public void EnsureValid() + { + if (Ptr != null && Ptr < 0) + throw new HubException("Invalid ptr"); + + if (Contract != null && !Regex.IsMatch(Contract, @"^KT1\w{33}$")) + throw new HubException("Invalid contract address"); + + if (Path != null && Path.Length > 256) + throw new HubException("Too long path"); + + if (Tags != null && Tags.Any(x => x != BigMapTags.Metadata && x != BigMapTags.TokenMetadata)) + throw new HubException("Invalid tags"); + } + } +} \ No newline at end of file diff --git a/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs b/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs new file mode 100644 index 000000000..b407b9ed1 --- /dev/null +++ b/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs @@ -0,0 +1,386 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.SignalR; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +using Tzkt.Api.Models; +using Tzkt.Api.Repositories; +using Tzkt.Api.Services.Cache; +using Tzkt.Data.Models; + +namespace Tzkt.Api.Websocket.Processors +{ + public class BigMapsProcessor : IHubProcessor where T : Hub + { + #region static + const string BigMapsGroup = "bigmaps"; + const string BigMapsChannel = "bigmaps"; + static readonly SemaphoreSlim Sema = new(1, 1); + + static readonly HashSet AllSubs = new(); + static readonly Dictionary> PtrSubs = new(); + static readonly Dictionary> TagSubs = new(); + static readonly Dictionary ContractSubs = new(); + + static readonly Dictionary Limits = new(); + + class ContractSub + { + public HashSet All { get; set; } + public Dictionary> Paths { get; set; } + public Dictionary> Tags { get; set; } + + public bool Empty => All == null && Paths == null && Tags == null; + } + #endregion + + readonly StateCache State; + readonly BigMapsRepository Repo; + readonly IHubContext Context; + readonly WebsocketConfig Config; + readonly ILogger Logger; + + public BigMapsProcessor(StateCache state, BigMapsRepository bigMaps, IHubContext hubContext, IConfiguration config, ILogger> logger) + { + State = state; + Repo = bigMaps; + Context = hubContext; + Config = config.GetWebsocketConfig(); + Logger = logger; + } + + public async Task OnStateChanged() + { + var sendings = new List(Limits.Count); + try + { + await Sema.WaitAsync(); + + if (Limits.Count == 0) + { + Logger.LogDebug("No bigmap subs"); + return; + } + + #region check reorg + if (State.Reorganized) + { + Logger.LogDebug("Sending reorg message with state {0}", State.ValidLevel); + sendings.Add(Context.Clients + .Group(BigMapsGroup) + .SendReorg(BigMapsChannel, State.ValidLevel)); + } + #endregion + + if (State.ValidLevel == State.Current.Level) + { + Logger.LogDebug("No bigmaps to send"); + return; + } + + #region load updates + Logger.LogDebug("Fetching bigmap updates from {0} to {1}", State.ValidLevel, State.Current.Level); + + var level = new Int32Parameter + { + Gt = State.ValidLevel, + Le = State.Current.Level + }; + var limit = State.Current.Level - State.ValidLevel; + var format = MichelineFormat.Json; + + var updates = await Repo.GetUpdates(null, null, level, null, null, limit, format); + var count = updates.Count(); + + Logger.LogDebug("{0} bigmap updates fetched", count); + #endregion + + #region prepare to send + var toSend = new Dictionary>(); + + void Add(HashSet subs, Models.BigMapUpdate update) + { + foreach (var clientId in subs) + { + if (!toSend.TryGetValue(clientId, out var list)) + { + list = new(4); + toSend.Add(clientId, list); + } + list.Add(update); + } + } + + foreach (var update in updates) + { + #region all subs + Add(AllSubs, update); + #endregion + + #region ptr subs + if (PtrSubs.TryGetValue(update.Bigmap, out var ptrSubs)) + Add(ptrSubs, update); + #endregion + + #region tag subs + foreach (var tag in update.EnumerateTags()) + if (TagSubs.TryGetValue(tag, out var tagSubs)) + Add(tagSubs, update); + #endregion + + #region contract subs + if (ContractSubs.TryGetValue(update.Contract.Address, out var contractSubs)) + { + if (contractSubs.All != null) + Add(contractSubs.All, update); + + if (contractSubs.Paths != null) + if (contractSubs.Paths.TryGetValue(update.Path, out var contractPathSubs)) + Add(contractPathSubs, update); + + if (contractSubs.Tags != null) + foreach (var tag in update.EnumerateTags()) + if (contractSubs.Tags.TryGetValue(tag, out var contractTagSubs)) + Add(contractTagSubs, update); + } + #endregion + } + #endregion + + #region send + foreach (var (connectionId, updatesList) in toSend.Where(x => x.Value.Count > 0)) + { + var data = updatesList.Count > 1 + ? Distinct(updatesList).OrderBy(x => x.Id) + : (IEnumerable)updatesList; + + sendings.Add(Context.Clients + .Client(connectionId) + .SendData(BigMapsChannel, data, State.Current.Level)); + + Logger.LogDebug("{0} bigmap updates sent to {1}", updatesList.Count, connectionId); + } + + Logger.LogDebug("{0} bigmap updates sent", count); + #endregion + } + catch (Exception ex) + { + Logger.LogError("Failed to process state change: {0}", ex.Message); + } + finally + { + Sema.Release(); + #region await sendings + try + { + await Task.WhenAll(sendings); + } + catch (Exception ex) + { + // should never get here + Logger.LogCritical("Sendings failed: {0}", ex.Message); + } + #endregion + } + } + + public async Task Subscribe(IClientProxy client, string connectionId, BigMapsParameter parameter) + { + Task sending = Task.CompletedTask; + try + { + await Sema.WaitAsync(); + Logger.LogDebug("New subscription..."); + + #region check limits + if (Limits.TryGetValue(connectionId, out var cnt) && cnt >= Config.MaxBigMapSubscriptions) + throw new HubException($"Subscriptions limit exceeded"); + + if (cnt > 0) // reuse already allocated string + connectionId = Limits.Keys.First(x => x == connectionId); + #endregion + + #region add to subs + if (parameter.Ptr != null) + { + TryAdd(PtrSubs, (int)parameter.Ptr, connectionId); + } + else if (parameter.Contract != null) + { + if (!ContractSubs.TryGetValue(parameter.Contract, out var contractSub)) + { + contractSub = new(); + ContractSubs.Add(parameter.Contract, contractSub); + } + if (parameter.Path != null) + { + contractSub.Paths ??= new(4); + TryAdd(contractSub.Paths, parameter.Path, connectionId); + } + else if (parameter.Tags != null) + { + contractSub.Tags ??= new(4); + foreach (var tag in parameter.Tags) + TryAdd(contractSub.Tags, tag == BigMapTags.Metadata ? BigMapTag.Metadata : BigMapTag.TokenMetadata, connectionId); + } + else + { + contractSub.All ??= new(4); + TryAdd(contractSub.All, connectionId); + } + } + else if (parameter.Tags?.Count > 0) + { + foreach (var tag in parameter.Tags) + TryAdd(TagSubs, tag == BigMapTags.Metadata ? BigMapTag.Metadata : BigMapTag.TokenMetadata, connectionId); + } + else + { + TryAdd(AllSubs, connectionId); + } + #endregion + + #region add to group + await Context.Groups.AddToGroupAsync(connectionId, BigMapsGroup); + #endregion + + sending = client.SendState(BigMapsChannel, State.Current.Level); + + Logger.LogDebug("Client {0} subscribed with state {1}", connectionId, State.Current.Level); + } + catch (HubException) + { + throw; + } + catch (Exception ex) + { + Logger.LogError("Failed to add subscription: {0}", ex.Message); + } + finally + { + Sema.Release(); + try + { + await sending; + } + catch (Exception ex) + { + // should never get here + Logger.LogCritical("Sending failed: {0}", ex.Message); + } + } + } + + public async Task Unsubscribe(string connectionId) + { + try + { + await Sema.WaitAsync(); + Logger.LogDebug("Remove subscription..."); + + TryRemove(AllSubs, connectionId); + TryRemove(PtrSubs, connectionId); + TryRemove(TagSubs, connectionId); + + foreach (var contractSub in ContractSubs.Values) + { + if (contractSub.All != null) + { + TryRemove(contractSub.All, connectionId); + if (contractSub.All.Count == 0) + contractSub.All = null; + } + if (contractSub.Paths != null) + { + TryRemove(contractSub.Paths, connectionId); + if (contractSub.Paths.Count == 0) + contractSub.Paths = null; + } + if (contractSub.Tags != null) + { + TryRemove(contractSub.Tags, connectionId); + if (contractSub.Tags.Count == 0) + contractSub.Tags = null; + } + } + + foreach (var contract in ContractSubs.Where(x => x.Value.Empty).Select(x => x.Key).ToList()) + ContractSubs.Remove(contract); + + if (Limits[connectionId] != 0) + Logger.LogCritical("Failed to unsibscribe {0}: {1} subs left", connectionId, Limits[connectionId]); + Limits.Remove(connectionId); + + Logger.LogDebug("Client {0} unsubscribed", connectionId); + } + catch (Exception ex) + { + Logger.LogError("Failed to remove subscription: {0}", ex.Message); + } + finally + { + Sema.Release(); + } + } + + private static void TryAdd(Dictionary> subs, TSubKey key, string connectionId) + { + if (!subs.TryGetValue(key, out var set)) + { + set = new(4); + subs.Add(key, set); + } + if (!set.Contains(connectionId)) + { + set.Add(connectionId); + Limits[connectionId] = Limits.GetValueOrDefault(connectionId) + 1; + } + } + + private static void TryAdd(HashSet set, string connectionId) + { + if (!set.Contains(connectionId)) + { + set.Add(connectionId); + Limits[connectionId] = Limits.GetValueOrDefault(connectionId) + 1; + } + } + + private static void TryRemove(Dictionary> subs, string connectionId) + { + foreach (var (key, value) in subs) + { + if (value.Remove(connectionId)) + Limits[connectionId]--; + + if (value.Count == 0) + subs.Remove(key); + } + } + + private static void TryRemove(HashSet set, string connectionId) + { + if (set.Remove(connectionId)) + Limits[connectionId]--; + } + + private static IEnumerable Distinct(List items) + { + var hashset = new HashSet(items.Count); + foreach (var item in items) + { + if (!hashset.Contains(item.Id)) + { + hashset.Add(item.Id); + yield return item; + } + } + } + } +} diff --git a/Tzkt.Api/Websocket/WebsocketConfig.cs b/Tzkt.Api/Websocket/WebsocketConfig.cs index 53c7076ee..85e3dc125 100644 --- a/Tzkt.Api/Websocket/WebsocketConfig.cs +++ b/Tzkt.Api/Websocket/WebsocketConfig.cs @@ -7,6 +7,7 @@ public class WebsocketConfig public bool Enabled { get; set; } = true; public int MaxConnections { get; set; } = 1000; public int MaxAccountSubscriptions { get; set; } = 50; + public int MaxBigMapSubscriptions { get; set; } = 50; } public static class CacheConfigExt diff --git a/Tzkt.Api/appsettings.json b/Tzkt.Api/appsettings.json index 7fe6bca1d..3bddbe6ed 100644 --- a/Tzkt.Api/appsettings.json +++ b/Tzkt.Api/appsettings.json @@ -11,7 +11,8 @@ "Websocket": { "Enabled": true, "MaxConnections": 1000, - "MaxAccountSubscriptions": 50 + "MaxAccountSubscriptions": 50, + "MaxBigMapSubscriptions": 50 }, "ConnectionStrings": { "DefaultConnection": "server=db;port=5432;database=tzkt_db;username=tzkt;password=qwerty;" From 05e8e68e1851f1170552b976ccb3a7e10189aaf7 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Thu, 22 Apr 2021 21:08:58 +0300 Subject: [PATCH 28/35] Update WS docs --- Tzkt.Api/Swagger/Description.md | 1 - Tzkt.Api/Swagger/WsExamples.md | 2 + Tzkt.Api/Swagger/WsGetStarted.md | 14 ++--- Tzkt.Api/Swagger/WsSubscriptions.md | 87 ++++++++++++++++++++++++++++- 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/Tzkt.Api/Swagger/Description.md b/Tzkt.Api/Swagger/Description.md index 039bd73e8..3d22738bf 100644 --- a/Tzkt.Api/Swagger/Description.md +++ b/Tzkt.Api/Swagger/Description.md @@ -6,7 +6,6 @@ TzKT is an open-source project, so you can easily clone and build it and use it TzKT API is available for the following Tezos networks with the following base URLs: - Mainnet: `https://api.tzkt.io/` or `https://api.mainnet.tzkt.io/` ([view docs](https://api.tzkt.io)) -- Delphinet: `https://api.delphinet.tzkt.io/` ([view docs](https://api.delphinet.tzkt.io)) - Edo2net: `https://api.edo2net.tzkt.io/` ([view docs](https://api.edo2net.tzkt.io)) - Florencenet: `https://api.florencenet.tzkt.io/` ([view docs](https://api.florencenet.tzkt.io)) diff --git a/Tzkt.Api/Swagger/WsExamples.md b/Tzkt.Api/Swagger/WsExamples.md index 7e12eff1e..fd03b4b82 100644 --- a/Tzkt.Api/Swagger/WsExamples.md +++ b/Tzkt.Api/Swagger/WsExamples.md @@ -75,6 +75,8 @@ Install SignalR package via npm: ````sh > npm install @microsoft/signalr + +const signalR = require("@microsoft/signalr"); ```` or via CDN: diff --git a/Tzkt.Api/Swagger/WsGetStarted.md b/Tzkt.Api/Swagger/WsGetStarted.md index 8286ce786..9c1f65af2 100644 --- a/Tzkt.Api/Swagger/WsGetStarted.md +++ b/Tzkt.Api/Swagger/WsGetStarted.md @@ -15,8 +15,8 @@ There are three message types that the server sends to the client: ````js { - type: 0, // 0 - state message - state: 0 // subscription state + type: 0, // 0 - state message + state: 0 // subscription state } ```` @@ -34,9 +34,9 @@ State can be used to fetch historical data from the REST API right after opening ````js { - type: 1, // 1 - data message - state: 0, // subscription state - data: {} // data object (or array, depending on subscription) + type: 1, // 1 - data message + state: 0, // subscription state + data: {} // data: object or array, depending on subscription } ```` @@ -50,8 +50,8 @@ TzKT Events operates with the same data models as the REST API to achieve full c ````js { - type: 2, // 2 - reorg message - state: 0 // subscription state + type: 2, // 2 - reorg message + state: 0 // subscription state } ```` diff --git a/Tzkt.Api/Swagger/WsSubscriptions.md b/Tzkt.Api/Swagger/WsSubscriptions.md index 6e6e52a70..a8fb22d53 100644 --- a/Tzkt.Api/Swagger/WsSubscriptions.md +++ b/Tzkt.Api/Swagger/WsSubscriptions.md @@ -87,9 +87,8 @@ Sends operations of specified types or related to specified accounts, included i // such as 'transaction', 'delegation', etc. } ```` - -> **Note:** Currently, you can subscribe to no more than 50 addresses per single connection. If you need more, let us know and consider opening more connections in the meantime, -> or subscribe to all operations and filter them on the client side. + +> **Note:** you can invoke this method multiple times with different parameters to register multiple subscriptions. ### Data model @@ -109,4 +108,86 @@ await connection.invoke("SubscribeToOperations", { types: 'transaction' }); await connection.invoke("SubscribeToOperations", { address: 'tz1234...', types: 'delegation,origination' }); ```` +--- + +## SubscribeToBigMaps + +Sends bigmap updates + +### Method + +`SubscribeToBigMaps` + +### Channel + +`bigmaps` + +### Parameters + +This method accepts the following parameters: + +````js +{ + ptr: 0, // ptr of the bigmap you want to subscribe to + tags: [], // array of bigmap tags ('metadata' or 'token_metadata') + contract: '', // contract address + path: '' // path to the bigmap in the contract strage +} +```` + +You can set various combinations of these fields to configure what you want to subscribe to. For example: + +````js +// subscribe to all bigmaps +{ +} + +// subscribe to all bigmaps with specific tags +{ + tags: ['metadata', 'token_metadata'] +} + +// subscribe to all bigmaps of the specific contract +{ + contract: 'KT1...' +} + +// subscribe to all bigmaps of the specific contract with specific tags +{ + contract: 'KT1...', + tags: ['metadata'] +} + +// subscribe to specific bigmap by ptr +{ + ptr: 123 +} + +// subscribe to specific bigmap by path +{ + contract: 'KT1...', + path: 'ledger' +} +```` + +> **Note:** you can invoke this method multiple times with different parameters to register multiple subscriptions. + +### Data model + +Same as in [/bigmaps/updates](#operation/BigMaps_GetBigMapUpdates). + +### State + +State contains level (`int`) of the last processed block. + +### Example + +````js +connection.on("bigmaps", (msg) => { console.log(msg); }); +// subscribe to all bigmaps of the 'KT123...' contract +await connection.invoke("SubscribeToBigMaps", { contract: 'KT123...' }); +// subscribe to bigmap with ptr 123 +await connection.invoke("SubscribeToBigMaps", { ptr: 123 }); +```` + --- \ No newline at end of file From f2c6ac166ce803336a949e1c50a613110e7cd6a9 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 22 Apr 2021 23:57:45 +0300 Subject: [PATCH 29/35] Fix wrong updates limit and optimize unsubscribing --- Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs | 3 ++- Tzkt.Api/Websocket/Processors/OperationsProcessor.cs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs b/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs index b407b9ed1..dd119ec79 100644 --- a/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs +++ b/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs @@ -91,7 +91,7 @@ public async Task OnStateChanged() Gt = State.ValidLevel, Le = State.Current.Level }; - var limit = State.Current.Level - State.ValidLevel; + var limit = 1_000_000; var format = MichelineFormat.Json; var updates = await Repo.GetUpdates(null, null, level, null, null, limit, format); @@ -282,6 +282,7 @@ public async Task Unsubscribe(string connectionId) try { await Sema.WaitAsync(); + if (!Limits.ContainsKey(connectionId)) return; Logger.LogDebug("Remove subscription..."); TryRemove(AllSubs, connectionId); diff --git a/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs b/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs index 31bef9e67..68978e7c2 100644 --- a/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs +++ b/Tzkt.Api/Websocket/Processors/OperationsProcessor.cs @@ -377,6 +377,7 @@ public async Task Unsubscribe(string connectionId) try { await Sema.WaitAsync(); + if (!Subs.ContainsKey(connectionId) && !AddressSubs.ContainsKey(connectionId)) return; Logger.LogDebug("Remove subscription..."); Subs.Remove(connectionId); From 84fd2973c1637a1801a2e21ea2dad54b76555bf9 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Sat, 24 Apr 2021 20:27:52 +0300 Subject: [PATCH 30/35] Smooth changes in software API model --- Tzkt.Api/Models/Software.cs | 55 +++++++++++++++++++ .../Software/SoftwareMetadataService.cs | 4 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Tzkt.Api/Models/Software.cs b/Tzkt.Api/Models/Software.cs index ef156484d..ae838a8c3 100644 --- a/Tzkt.Api/Models/Software.cs +++ b/Tzkt.Api/Models/Software.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Text.Json; namespace Tzkt.Api.Models @@ -39,5 +41,58 @@ public class Software /// Offchain metadata /// public RawJson Metadata { get; set; } + + /// + /// **DEPRECATED**. Use `metadata` instead. + /// + public DateTime? CommitDate + { + get + { + if (Metadata?.Json == null) return null; + using var doc = JsonDocument.Parse(Metadata.Json); + return doc.RootElement.TryGetProperty("commitDate", out var v) && v.TryGetDateTime(out var dt) ? dt : null; + } + } + + /// + /// **DEPRECATED**. Use `metadata` instead. + /// + public string CommitHash + { + get + { + if (Metadata?.Json == null) return null; + using var doc = JsonDocument.Parse(Metadata.Json); + return doc.RootElement.TryGetProperty("commitHash", out var v) ? v.GetString() : null; + } + } + + /// + /// **DEPRECATED**. Use `metadata` instead. + /// + public string Version + { + get + { + if (Metadata?.Json == null) return null; + using var doc = JsonDocument.Parse(Metadata.Json); + return doc.RootElement.TryGetProperty("version", out var v) ? v.GetString() : null; + } + } + + /// + /// **DEPRECATED**. Use `metadata` instead. + /// + public List Tags + { + get + { + if (Metadata?.Json == null) return null; + using var doc = JsonDocument.Parse(Metadata.Json); + return doc.RootElement.TryGetProperty("tags", out var v) && v.ValueKind == JsonValueKind.Array + ? v.EnumerateArray().Select(x => x.GetString()).ToList() : null; + } + } } } diff --git a/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs b/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs index 167a580d9..3e58f6154 100644 --- a/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs +++ b/Tzkt.Api/Services/Metadata/Software/SoftwareMetadataService.cs @@ -2,10 +2,10 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Dapper; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Dapper; using Tzkt.Api.Models; using Tzkt.Api.Services.Cache; @@ -32,7 +32,7 @@ public SoftwareMetadataService(TimeCache time, IConfiguration config, ILogger (int)row.Id, row => new SoftwareAlias { Version = row.Version, - Date = DateTime.TryParse(row.CommitDate, out DateTime dt) ? dt : Time[row.FirstLevel] + Date = DateTimeOffset.TryParse(row.CommitDate, out DateTimeOffset dt) ? dt.DateTime : Time[row.FirstLevel] }); Logger.LogDebug($"Loaded {Aliases.Count} software metadata"); From 33c62dbcefc2f558f1de7b7e51a41b6fa0a69dff Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Mon, 26 Apr 2021 04:23:04 +0300 Subject: [PATCH 31/35] Add cycle to state API model and fix missed bigmap commit --- Tzkt.Api/Models/State.cs | 5 ++++ Tzkt.Api/Repositories/StateRepository.cs | 1 + .../Cache/State/RawModels/RawState.cs | 2 ++ .../Handlers/Proto9/Commits/BigMapCommit.cs | 7 ++++++ .../Handlers/Proto9/Proto9Handler.cs | 24 +++++++++++++++---- 5 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 Tzkt.Sync/Protocols/Handlers/Proto9/Commits/BigMapCommit.cs diff --git a/Tzkt.Api/Models/State.cs b/Tzkt.Api/Models/State.cs index 78acd44dc..bd4ae041e 100644 --- a/Tzkt.Api/Models/State.cs +++ b/Tzkt.Api/Models/State.cs @@ -7,6 +7,11 @@ namespace Tzkt.Api.Models { public class State { + /// + /// Current cycle + /// + public int Cycle { get; set; } + /// /// The height of the last block from the genesis block /// diff --git a/Tzkt.Api/Repositories/StateRepository.cs b/Tzkt.Api/Repositories/StateRepository.cs index 4d1583b55..4120db425 100644 --- a/Tzkt.Api/Repositories/StateRepository.cs +++ b/Tzkt.Api/Repositories/StateRepository.cs @@ -26,6 +26,7 @@ public State Get() KnownLevel = appState.KnownHead, LastSync = appState.LastSync, Hash = appState.Hash, + Cycle = appState.Cycle, Level = appState.Level, Protocol = appState.Protocol, Timestamp = appState.Timestamp, diff --git a/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs b/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs index 63a27ab45..a1233949a 100644 --- a/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs +++ b/Tzkt.Api/Services/Cache/State/RawModels/RawState.cs @@ -8,6 +8,8 @@ public class RawState public DateTime LastSync { get; set; } + public int Cycle { get; set; } + public int Level { get; set; } public string Hash { get; set; } diff --git a/Tzkt.Sync/Protocols/Handlers/Proto9/Commits/BigMapCommit.cs b/Tzkt.Sync/Protocols/Handlers/Proto9/Commits/BigMapCommit.cs new file mode 100644 index 000000000..825cee6ca --- /dev/null +++ b/Tzkt.Sync/Protocols/Handlers/Proto9/Commits/BigMapCommit.cs @@ -0,0 +1,7 @@ +namespace Tzkt.Sync.Protocols.Proto9 +{ + class BigMapCommit : Proto1.BigMapCommit + { + public BigMapCommit(ProtocolHandler protocol) : base(protocol) { } + } +} diff --git a/Tzkt.Sync/Protocols/Handlers/Proto9/Proto9Handler.cs b/Tzkt.Sync/Protocols/Handlers/Proto9/Proto9Handler.cs index cdbb80f70..68e63c59f 100644 --- a/Tzkt.Sync/Protocols/Handlers/Proto9/Proto9Handler.cs +++ b/Tzkt.Sync/Protocols/Handlers/Proto9/Proto9Handler.cs @@ -21,7 +21,7 @@ class Proto9Handler : ProtocolHandler public override IValidator Validator { get; } public override IRpc Rpc { get; } - public Proto9Handler(TezosNode node, TzktContext db, CacheService cache, QuotesService quotes, IServiceProvider services, IConfiguration config, ILogger logger) + public Proto9Handler(TezosNode node, TzktContext db, CacheService cache, QuotesService quotes, IServiceProvider services, IConfiguration config, ILogger logger) : base(node, db, cache, quotes, services, config, logger) { Rpc = new Rpc(node); @@ -110,6 +110,8 @@ public override async Task Commit(JsonElement block) } #endregion + var bigMapCommit = new BigMapCommit(this); + #region operations 3 foreach (var operation in operations[3].EnumerateArray()) { @@ -126,11 +128,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).Apply(blockCommit.Block, operation, content); break; case "origination": - await new OriginationsCommit(this).Apply(blockCommit.Block, operation, content); + var orig = new OriginationsCommit(this); + await orig.Apply(blockCommit.Block, operation, content); + if (orig.BigMapDiffs != null) + bigMapCommit.Append(orig.Origination, orig.Origination.Contract, orig.BigMapDiffs); break; case "transaction": var parent = new TransactionsCommit(this); await parent.Apply(blockCommit.Block, operation, content); + if (parent.BigMapDiffs != null) + bigMapCommit.Append(parent.Transaction, parent.Transaction.Target as Contract, parent.BigMapDiffs); if (content.Required("metadata").TryGetProperty("internal_operation_results", out var internalResult)) { @@ -142,10 +149,16 @@ public override async Task Commit(JsonElement block) await new DelegationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); break; case "origination": - await new OriginationsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalOrig = new OriginationsCommit(this); + await internalOrig.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalOrig.BigMapDiffs != null) + bigMapCommit.Append(internalOrig.Origination, internalOrig.Origination.Contract, internalOrig.BigMapDiffs); break; case "transaction": - await new TransactionsCommit(this).ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + var internalTx = new TransactionsCommit(this); + await internalTx.ApplyInternal(blockCommit.Block, parent.Transaction, internalContent); + if (internalTx.BigMapDiffs != null) + bigMapCommit.Append(internalTx.Transaction, internalTx.Transaction.Target as Contract, internalTx.BigMapDiffs); break; default: throw new NotImplementedException($"internal '{content.RequiredString("kind")}' is not implemented"); @@ -160,6 +173,8 @@ public override async Task Commit(JsonElement block) } #endregion + await bigMapCommit.Apply(); + var brCommit = new BakingRightsCommit(this); await brCommit.Apply(blockCommit.Block); @@ -252,6 +267,7 @@ public override async Task Revert() await new DelegatorCycleCommit(this).Revert(currBlock); await new CycleCommit(this).Revert(currBlock); await new BakingRightsCommit(this).Revert(currBlock); + await new BigMapCommit(this).Revert(currBlock); foreach (var operation in operations.OrderByDescending(x => x.Id)) { From 50ee5a3c4e2721b4253d81f452a779d5b73db231 Mon Sep 17 00:00:00 2001 From: Groxan <257byte@gmail.com> Date: Mon, 26 Apr 2021 18:24:29 +0300 Subject: [PATCH 32/35] Fix quotes service --- Tzkt.Sync/Services/Quotes/QuotesService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tzkt.Sync/Services/Quotes/QuotesService.cs b/Tzkt.Sync/Services/Quotes/QuotesService.cs index 8f961f94b..f6c59d62f 100644 --- a/Tzkt.Sync/Services/Quotes/QuotesService.cs +++ b/Tzkt.Sync/Services/Quotes/QuotesService.cs @@ -247,7 +247,7 @@ void SaveAndUpdate(AppState state, IEnumerable quotes) param.Add(last.Krw); param.Add(last.Eth); - var p = 6; + var p = 7; var i = 0; foreach (var q in quotes) From 0cb6336cea0c26e395585e18cb44e4b0335fba5a Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 29 Apr 2021 00:27:43 +0300 Subject: [PATCH 33/35] Better contract storage path validation --- Tzkt.Api/Controllers/ContractsController.cs | 24 +++++++++----------- Tzkt.Api/Repositories/AccountRepository.cs | 25 ++++++++++++--------- Tzkt.Api/Utils/SqlBuilder.cs | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/Tzkt.Api/Controllers/ContractsController.cs b/Tzkt.Api/Controllers/ContractsController.cs index a88b9d779..ace091ba7 100644 --- a/Tzkt.Api/Controllers/ContractsController.cs +++ b/Tzkt.Api/Controllers/ContractsController.cs @@ -9,6 +9,7 @@ using Netezos.Encoding; using Tzkt.Api.Models; using Tzkt.Api.Repositories; +using Tzkt.Api.Utils; namespace Tzkt.Api.Controllers { @@ -282,27 +283,22 @@ public async Task BuildEntrypointParameters([Address] string addre public async Task GetStorage([Address] string address, [Min(0)] int level = 0, string path = null) { #region safe path - string[] safePath = null; + JsonPath[] jsonPath = null; if (path != null) { - var arr = path.Replace("..", "*").Split(".", StringSplitOptions.RemoveEmptyEntries); + if (!JsonPath.TryParse(path, out jsonPath)) + return new BadRequest(nameof(path), + $"Path contains invalid item: {jsonPath.First(x => x.Type == JsonPathType.None).Value}"); - for (int i = 0; i < arr.Length; i++) - { - arr[i] = arr[i].Replace("*", "."); - - if (!Regex.IsMatch(arr[i], "^[0-9A-z_.%@]+$")) - return new BadRequest(nameof(path), $"Invalid path value '{arr[i]}'"); - } - - if (arr.Length > 0) - safePath = arr; + if (jsonPath.Any(x => x.Type == JsonPathType.Any)) + return new BadRequest(nameof(path), + "Path contains invalid item: [*]"); } #endregion if (level == 0) - return this.Json(await Accounts.GetStorageValue(address, safePath)); - return this.Json(await Accounts.GetStorageValue(address, safePath, level)); + return this.Json(await Accounts.GetStorageValue(address, jsonPath)); + return this.Json(await Accounts.GetStorageValue(address, jsonPath, level)); } /// diff --git a/Tzkt.Api/Repositories/AccountRepository.cs b/Tzkt.Api/Repositories/AccountRepository.cs index 35a583684..af47148f6 100644 --- a/Tzkt.Api/Repositories/AccountRepository.cs +++ b/Tzkt.Api/Repositories/AccountRepository.cs @@ -11,6 +11,7 @@ using Tzkt.Api.Models; using Tzkt.Api.Services.Cache; using Tzkt.Api.Services.Metadata; +using Tzkt.Api.Utils; namespace Tzkt.Api.Repositories { @@ -2410,7 +2411,7 @@ public async Task> GetEntrypoints(string address, bool a }); } - public async Task GetStorageValue(string address, string[] path) + public async Task GetStorageValue(string address, JsonPath[] path) { var rawAccount = await Accounts.GetAsync(address); if (rawAccount is not RawContract contract) return null; @@ -2421,20 +2422,21 @@ public async Task GetStorageValue(string address, string[] path) return path?.Length > 0 ? null : manager.Address; } - // path value should already be valid - var jsonPath = path == null ? string.Empty : $@"#>'{{{string.Join(',', path)}}}'"; + var pathSelector = path == null ? string.Empty : " #> @path"; + var pathParam = path == null ? null : new { path = JsonPath.Select(path) }; using var db = GetConnection(); var row = await db.QueryFirstOrDefaultAsync($@" - SELECT ""JsonValue""{jsonPath} as ""JsonValue"" + SELECT ""JsonValue""{pathSelector} as ""JsonValue"" FROM ""Storages"" WHERE ""ContractId"" = {contract.Id} AND ""Current"" = true - LIMIT 1"); + LIMIT 1", + pathParam); return row?.JsonValue; } - public async Task GetStorageValue(string address, string[] path, int level) + public async Task GetStorageValue(string address, JsonPath[] path, int level) { var rawAccount = await Accounts.GetAsync(address); if (rawAccount is not RawContract contract) return null; @@ -2451,17 +2453,18 @@ public async Task GetStorageValue(string address, string[] path, int lev return path?.Length > 0 ? null : manager.Address; } - // path value should already be valid - var jsonPath = path == null ? string.Empty : $@"#>'{{{string.Join(',', path)}}}'"; - + var pathSelector = path == null ? string.Empty : " #> @path"; + var pathParam = path == null ? null : new { path = JsonPath.Select(path) }; + using var db = GetConnection(); var row = await db.QueryFirstOrDefaultAsync($@" - SELECT ""JsonValue""{jsonPath} as ""JsonValue"" + SELECT ""JsonValue""{pathSelector} as ""JsonValue"" FROM ""Storages"" WHERE ""ContractId"" = {contract.Id} AND ""Level"" <= {level} ORDER BY ""Level"" DESC, ""TransactionId"" DESC - LIMIT 1"); + LIMIT 1", + pathParam); return row?.JsonValue; } diff --git a/Tzkt.Api/Utils/SqlBuilder.cs b/Tzkt.Api/Utils/SqlBuilder.cs index c0d5315bf..f67ba37ed 100644 --- a/Tzkt.Api/Utils/SqlBuilder.cs +++ b/Tzkt.Api/Utils/SqlBuilder.cs @@ -447,7 +447,7 @@ public SqlBuilder Filter(string column, JsonParameter json, Func sql += $@" AND ""{column}"" #> {Param(JsonPath.Select(path))} = {Param(value)}::jsonb"; sqls.Add(sql); } - AppendFilter(string.Join(" OR ", sqls)); + AppendFilter($"({string.Join(" OR ", sqls)})"); } } From 6caef1ae06ecd74d3ee441bea169c8ba31996f4c Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 29 Apr 2021 01:16:52 +0300 Subject: [PATCH 34/35] Add filtering bigmap updates by value --- Tzkt.Api/Controllers/BigMapsController.cs | 7 +++++-- Tzkt.Api/Repositories/BigMapsRepository.cs | 4 ++++ Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Tzkt.Api/Controllers/BigMapsController.cs b/Tzkt.Api/Controllers/BigMapsController.cs index 82a727b18..04fadb55d 100644 --- a/Tzkt.Api/Controllers/BigMapsController.cs +++ b/Tzkt.Api/Controllers/BigMapsController.cs @@ -106,6 +106,8 @@ public async Task>> GetBigMaps( /// Filters updates by bigmap contract /// Filters updates by bigmap tags: `token_metadata` - tzip-12, `metadata` - tzip-16 /// Filters updates by action + /// Filters updates by JSON value. Note, this query parameter supports the following format: `?value{.path?}{.mode?}=...`, + /// so you can specify a path to a particular field to filter by, for example: `?value.balance.gt=...`. /// Filters updates by level /// Sorts bigmaps by specified field. Supported fields: `id` (default), `ptr`, `firstLevel`, `lastLevel`, `totalKeys`, `activeKeys`, `updates`. /// Specifies which or how many items should be skipped @@ -119,6 +121,7 @@ public async Task>> GetBigMapUpdates( AccountParameter contract, BigMapTagsParameter tags, BigMapActionParameter action, + JsonParameter value, Int32Parameter level, SortParameter sort, OffsetParameter offset, @@ -131,9 +134,9 @@ public async Task>> GetBigMapUpdates( #endregion if (path == null && contract == null && tags == null) - return Ok(await BigMaps.GetUpdates(bigmap, action, level, sort, offset, limit, micheline)); + return Ok(await BigMaps.GetUpdates(bigmap, action, value, level, sort, offset, limit, micheline)); - return Ok(await BigMaps.GetUpdates(bigmap, path, contract, action, tags, level, sort, offset, limit, micheline)); + return Ok(await BigMaps.GetUpdates(bigmap, path, contract, action, value, tags, level, sort, offset, limit, micheline)); } /// diff --git a/Tzkt.Api/Repositories/BigMapsRepository.cs b/Tzkt.Api/Repositories/BigMapsRepository.cs index 94e598c2e..8f89f7b92 100644 --- a/Tzkt.Api/Repositories/BigMapsRepository.cs +++ b/Tzkt.Api/Repositories/BigMapsRepository.cs @@ -948,6 +948,7 @@ public async Task> GetKeyByHashUpdates( public async Task> GetUpdates( Int32Parameter ptr, BigMapActionParameter action, + JsonParameter value, Int32Parameter level, SortParameter sort, OffsetParameter offset, @@ -959,6 +960,7 @@ public async Task> GetUpdates( var sql = new SqlBuilder($@"SELECT ""Id"", ""BigMapPtr"", ""Action"", ""Level"", ""BigMapKeyId"", ""{fCol}Value"" FROM ""BigMapUpdates""") .Filter("BigMapPtr", ptr) .Filter("Action", action) + .Filter("JsonValue", value) .Filter("Level", level) .Take(sort, offset, limit, x => x switch { @@ -1029,6 +1031,7 @@ public async Task> GetUpdates( StringParameter path, AccountParameter contract, BigMapActionParameter action, + JsonParameter value, BigMapTagsParameter tags, Int32Parameter level, SortParameter sort, @@ -1046,6 +1049,7 @@ public async Task> GetUpdates( .Filter("BigMapPtr", ptr) .Filter("StoragePath", path) .Filter("Action", action) + .Filter("JsonValue", value) .Filter("Tags", tags) .Filter("ContractId", contract) .Filter("Level", level) diff --git a/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs b/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs index dd119ec79..be5bd4553 100644 --- a/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs +++ b/Tzkt.Api/Websocket/Processors/BigMapsProcessor.cs @@ -94,7 +94,7 @@ public async Task OnStateChanged() var limit = 1_000_000; var format = MichelineFormat.Json; - var updates = await Repo.GetUpdates(null, null, level, null, null, limit, format); + var updates = await Repo.GetUpdates(null, null, null, level, null, null, limit, format); var count = updates.Count(); Logger.LogDebug("{0} bigmap updates fetched", count); From ce3e591b854e663f77fd4034bb2ce7c79d331845 Mon Sep 17 00:00:00 2001 From: 257Byte <257Byte@gmail.com> Date: Thu, 29 Apr 2021 02:24:27 +0300 Subject: [PATCH 35/35] Update packages and docs --- Dockerfile-snapshot | 2 +- README.md | 61 +++++++++++++++-------------- Tzkt.Api/Swagger/WsSubscriptions.md | 6 ++- Tzkt.Api/Tzkt.Api.csproj | 4 +- Tzkt.Data/Tzkt.Data.csproj | 6 +-- Tzkt.Sync/Tzkt.Sync.csproj | 6 +-- 6 files changed, 44 insertions(+), 41 deletions(-) diff --git a/Dockerfile-snapshot b/Dockerfile-snapshot index 3d6efebd7..ff5eadb33 100644 --- a/Dockerfile-snapshot +++ b/Dockerfile-snapshot @@ -1,3 +1,3 @@ FROM cirrusci/wget -RUN wget "https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/tzkt_v1.4.backup" -O tzkt_db.backup +RUN wget "https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/tzkt_v1.5.backup" -O tzkt_db.backup diff --git a/README.md b/README.md index 18665f156..5d4bfde61 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ The indexer fetches raw data from the Tezos node, then processes it and stores i ## Features: - **More detailed data.** TzKT not only collects blockchain data, but also processes and extends it with unique properties or even entities. For example, TzKT was the first indexer introduced synthetic operation types such as "migration" or "revelation penalty", which fill in the gaps in account history (because this data is missed in the blockchain), and the only indexer that correctly distinguishes smart contracts among all contracts. -- **Data quality comes first!** You will never see an incorrect account balance, or total rolls, or missed operations, etc. TzKT was built by professionals who know Tezos from A to Z (or, in other words, from TZ to KT 😼). -- **Advanced API.** TzKT provides a REST-like API, so you don't have to connect to the database directly. In addition to basic data access TzKT API has a lot of cool features such as exporting account statements, calculating historical balances (at any block), injecting metadata and much more. See the [API documentation](https://api.tzkt.io), automatically generated using Swagger (Open API 3 specification). +- **Micheline-to-JSON conversion** TzKT automatically converts raw Micheline JSON to human-readable JSON, so it's extremely handy to work with transaction parameters, contract storages, bigmaps keys, etc. +- **Data quality comes first!** You will never see an incorrect account balance, or total rolls, or missed operations, etc. TzKT was built by professionals who know Tezos from A to Z (or, in other words, from tz to KT 😼). +- **Advanced API.** TzKT provides a REST-like API, so you don't have to connect to the database directly. In addition to basic data access TzKT API has a lot of cool features such as deep filtering, sorting, data selection, exporting .csv statements, calculating historical data (at any block) such as balances or BigMap keys, injecting historical quotes and metadata, optimized caching and much more. See the complete [API documentation](https://api.tzkt.io). - **WebSocket API.** TzKT allows to subscribe to real-time blockchain data, such as new blocks or new operations, etc. via WebSocket. TzKT uses SignalR, which is very easy to use and for which there are many client libraries for different languages. -- **Low resource consumption.** TzKT is fairly lightweight. The indexer consumes up to 128MB of RAM, and the API up to 256MB-1024MB, depending on the configured cache size. +- **Low resource consumption.** TzKT is fairly lightweight. The indexer consumes up to 128MB of RAM, and the API up to 256MB-1024MB, depending on the network and configured cache size. - **No local node needed.** TzKT indexer works well even with remote RPC node. By default it uses [tezos.giganode.io](https://tezos.giganode.io/), the most performant public RPC node in Tezos, which is more than enough for most cases. - **Quick start.** Indexer bootstrap takes ~15 minutes by using snapshots publicly available for all supported networks. Of course, you can run full synchronization from scratch as well. - **Validation and diagnostics.** TzKT indexer validates all incoming data so you will never get to the wrong chain and will never commit corrupted data. Also, the indexer performs self-diagnostics after each block, which guarantees the correct commiting. @@ -37,7 +38,7 @@ make stop ## Installation (from source) -This is the preferred way, because you have more control over each TzKT component (database, indexer, API). This guide is for Ubuntu 18.04, but if you are using a different OS, the installation process will probably differ only in the "Install packages" step. +This is the preferred way, because you have more control over each TzKT component (database, indexer, API). This guide is for Ubuntu 20.04, but if you are using a different OS, the installation process will probably differ only in the "Install packages" step. ### Install packages @@ -87,7 +88,7 @@ postgres=# \q #### Download fresh snapshot ````c -wget "https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/tzkt_v1.4.backup" -O /tmp/tzkt_db.backup +wget "https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/tzkt_v1.5.backup" -O /tmp/tzkt_db.backup ```` #### Restore database from the snapshot @@ -258,16 +259,16 @@ That's it. By default API is available on ports 5000 (HTTP) and 5001 (HTTPS). If ## Install Tzkt Indexer and API for testnets In general the steps are the same as for the mainnet, you just need to use different database, different snapshot and different appsettings (chain id and RPC endpoint). Here are some presets for testnets: - - Delphinet: - - Snapshot: https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/delphi_tzkt_v1.4.backup - - RPC node: https://rpc.tzkt.io/delphinet/ - - Chain id: NetXm8tYqnMWky1 - Edonet: - - Snapshot: https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/edo2_tzkt_v1.4.backup + - Snapshot: https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/edo2_tzkt_v1.5.backup - RPC node: https://rpc.tzkt.io/edo2net/ - Chain id: NetXSp4gfdanies + - Florencenet: + - Snapshot: https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/flor_tzkt_v1.5.backup + - RPC node: https://rpc.tzkt.io/florencenobanet/ + - Chain id: NetXxkAx4woPLyu -Anyway, let's do it, for reference, from scratch for the delphinet. +Anyway, let's do it, for reference, from scratch for the `florencenet`. ### Prepare database @@ -276,23 +277,23 @@ Anyway, let's do it, for reference, from scratch for the delphinet. ```` sudo -u postgres psql -postgres=# create database delphi_tzkt_db; +postgres=# create database flor_tzkt_db; postgres=# create user tzkt with encrypted password 'qwerty'; -postgres=# grant all privileges on database delphi_tzkt_db to tzkt; +postgres=# grant all privileges on database flor_tzkt_db to tzkt; postgres=# \q ```` #### Download fresh snapshot ````c -wget "https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/delphi_tzkt_v1.4.backup" -O /tmp/delphi_tzkt_db.backup +wget "https://tzkt-snapshots.s3.eu-central-1.amazonaws.com/flor_tzkt_v1.5.backup" -O /tmp/flor_tzkt_db.backup ```` #### Restore database from the snapshot ````c -// delphinet restoring takes ~1 min -sudo -u postgres pg_restore -c --if-exists -v -d delphi_tzkt_db -1 /tmp/delphi_tzkt_db.backup +// florencenet restoring takes ~1 min +sudo -u postgres pg_restore -c --if-exists -v -d flor_tzkt_db -1 /tmp/flor_tzkt_db.backup ```` ### Clone, build, configure and run Tzkt Indexer @@ -308,12 +309,12 @@ git clone https://github.com/baking-bad/tzkt.git ```` cd ~/tzkt/Tzkt.Sync/ -dotnet publish -o ~/delphi-tzkt-sync +dotnet publish -o ~/flor-tzkt-sync ```` #### Configure indexer -Edit configuration file `~/delphi-tzkt-sync/appsettings.json` with your favorite text editor. What you need is to specify `Diagnostics` (just disable it), `TezosNode.ChainId`, `TezosNode.Endpoint` and `ConnectionStrings.DefaultConnection`. +Edit configuration file `~/flor-tzkt-sync/appsettings.json` with your favorite text editor. What you need is to specify `Diagnostics` (just disable it), `TezosNode.ChainId`, `TezosNode.Endpoint` and `ConnectionStrings.DefaultConnection`. Like this: @@ -325,8 +326,8 @@ Like this: }, "TezosNode": { - "ChainId": "NetXm8tYqnMWky1", - "Endpoint": "https://rpc.tzkt.io/delphinet/", + "ChainId": "NetXxkAx4woPLyu", + "Endpoint": "https://rpc.tzkt.io/florencenobanet/", "Timeout": 30 }, @@ -336,7 +337,7 @@ Like this: }, "ConnectionStrings": { - "DefaultConnection": "server=localhost;port=5432;database=delphi_tzkt_db;username=tzkt;password=qwerty;" + "DefaultConnection": "server=localhost;port=5432;database=flor_tzkt_db;username=tzkt;password=qwerty;" }, "Logging": { @@ -352,7 +353,7 @@ Like this: #### Run indexer ````c -cd ~/delphi-tzkt-sync +cd ~/flor-tzkt-sync dotnet Tzkt.Sync.dll // info: Microsoft.Hosting.Lifetime[0] @@ -360,7 +361,7 @@ dotnet Tzkt.Sync.dll // info: Microsoft.Hosting.Lifetime[0] // Hosting environment: Production // info: Microsoft.Hosting.Lifetime[0] -// Content root path: /home/tzkt/delphi-tzkt-sync +// Content root path: /home/tzkt/flor-tzkt-sync // warn: Tzkt.Sync.Services.Observer[0] // Observer is started // info: Tzkt.Sync.Services.Observer[0] @@ -372,20 +373,20 @@ dotnet Tzkt.Sync.dll That's it. If you want to run the indexer as a daemon, take a look at this guide: https://docs.microsoft.com/aspnet/core/host-and-deploy/linux-nginx?view=aspnetcore-3.1#create-the-service-file. -### Build, configure and run Tzkt API for the delphinet indexer +### Build, configure and run Tzkt API for the florencenet indexer -Suppose you have already created database `delphi_tzkt_db`, database user `tzkt` and cloned Tzkt repo to `~/tzkt`. +Suppose you have already created database `flor_tzkt_db`, database user `tzkt` and cloned Tzkt repo to `~/tzkt`. #### Build API ```` cd ~/tzkt/Tzkt.Api/ -dotnet publish -o ~/delphi-tzkt-api +dotnet publish -o ~/flor-tzkt-api ```` #### Configure API -Edit configuration file `~/delphi-tzkt-api/appsettings.json` with your favorite text editor. What you need is to specify `ConnectionStrings.DefaultConnection`, a connection string for the database created above. +Edit configuration file `~/flor-tzkt-api/appsettings.json` with your favorite text editor. What you need is to specify `ConnectionStrings.DefaultConnection`, a connection string for the database created above. By default API is available on ports 5000 (HTTP) and 5001 (HTTPS). If you want to use HTTPS, you also need to configure certificates. @@ -413,7 +414,7 @@ Like this: }, "ConnectionStrings": { - "DefaultConnection": "server=localhost;port=5432;database=delphi_tzkt_db;username=tzkt;password=qwerty;" + "DefaultConnection": "server=localhost;port=5432;database=flor_tzkt_db;username=tzkt;password=qwerty;" }, "Kestrel": { @@ -439,7 +440,7 @@ Like this: #### Run API ````c -cd ~/delphi-tzkt-api +cd ~/flor-tzkt-api dotnet Tzkt.Api.dll // info: Tzkt.Api.Services.Metadata.AccountMetadataService[0] @@ -457,7 +458,7 @@ dotnet Tzkt.Api.dll // info: Microsoft.Hosting.Lifetime[0] // Hosting environment: Production // info: Microsoft.Hosting.Lifetime[0] -// Content root path: /home/tzkt/delphi-tzkt-api +// Content root path: /home/tzkt/flor-tzkt-api // .... ```` diff --git a/Tzkt.Api/Swagger/WsSubscriptions.md b/Tzkt.Api/Swagger/WsSubscriptions.md index a8fb22d53..7d303818f 100644 --- a/Tzkt.Api/Swagger/WsSubscriptions.md +++ b/Tzkt.Api/Swagger/WsSubscriptions.md @@ -83,8 +83,10 @@ Sends operations of specified types or related to specified accounts, included i address: '', // address you want to subscribe to, // or null if you want to subscribe for all operations - types: '' // comma-separated list of operation types - // such as 'transaction', 'delegation', etc. + types: '' // comma-separated list of operation types, any of: + // 'transaction', 'origination', 'delegation', 'reveal' + // 'double_baking', 'double_endorsing', 'nonce_revelation', 'activation' + // 'proposal', 'ballot', 'endorsement. } ```` diff --git a/Tzkt.Api/Tzkt.Api.csproj b/Tzkt.Api/Tzkt.Api.csproj index e25c59164..0b298c443 100644 --- a/Tzkt.Api/Tzkt.Api.csproj +++ b/Tzkt.Api/Tzkt.Api.csproj @@ -2,7 +2,7 @@ net5.0 - 1.4 + 1.5 @@ -38,7 +38,7 @@ - + diff --git a/Tzkt.Data/Tzkt.Data.csproj b/Tzkt.Data/Tzkt.Data.csproj index 7feff3f82..5a03a21cb 100644 --- a/Tzkt.Data/Tzkt.Data.csproj +++ b/Tzkt.Data/Tzkt.Data.csproj @@ -2,12 +2,12 @@ net5.0 - 1.4 + 1.5 - - + + diff --git a/Tzkt.Sync/Tzkt.Sync.csproj b/Tzkt.Sync/Tzkt.Sync.csproj index 97914dedb..5de1e189b 100644 --- a/Tzkt.Sync/Tzkt.Sync.csproj +++ b/Tzkt.Sync/Tzkt.Sync.csproj @@ -2,16 +2,16 @@ net5.0 - 1.4 + 1.5 - + all runtime; build; native; contentfiles; analyzers; buildtransitive - +