Fix bug in module schema (#124)

* Fix bug in module schema * Update CHANGELOG.md * Update backend/CHANGELOG.md Co-authored-by: Søren Hjort <[email protected]> * Update ContractExtensions.cs * Update ContractExtensions.cs * Added test and fix wrong key for module v1 where schema not embedded * Update UpdateModuleSourceCatchup.cs * Update UpdateModuleSourceCatchup.cs --------- Co-authored-by: Søren Hjort <[email protected]>
Concordium · Oct 27, 2023 · eb7d009 · eb7d009
1 parent 50de657
commit eb7d009
Show file tree

Hide file tree

Showing 10 changed files with 171 additions and 3 deletions.
diff --git a/backend/Application/Aggregates/Contract/Entities/ModuleReferenceEvent.cs b/backend/Application/Aggregates/Contract/Entities/ModuleReferenceEvent.cs
@@ -119,6 +119,17 @@ internal static async Task<ModuleSourceInfo> Create(IContractNodeClient client,
             return (versionedModuleSource, moduleSourceHex, moduleWasm);
         }
 
+        /// <summary>
+        /// The module can contain a schema in one of two different custom sections.
+        /// The supported sections depend on the module version.
+        /// The schema version can be either defined by the section name or embedded into the actual schema:
+        /// - Both v0 and v1 modules support the section 'concordium-schema' where the schema includes the version.
+        ///   - For v0 modules this is always a v0 schema.
+        ///   - For v1 modules this can be a v1, v2, or v3 schema.
+        ///- V0 modules additionally support section 'concordium-schema-v1' which always contain a v0 schema (not a typo).
+        /// - V1 modules additionally support section 'concordium-schema-v2' which always contain a v1 schema (not a typo).
+        /// The section 'concordium-schema' is the most common and is what the current tooling produces.
+        /// </summary>
         private static (string Schema, ModuleSchemaVersion SchemaVersion)? GetModuleSchema(WebAssembly.Module module, VersionedModuleSource moduleSource)
         {
             switch (moduleSource)
@@ -138,7 +149,7 @@ private static (string Schema, ModuleSchemaVersion SchemaVersion)? GetModuleSche
                     {
                         return (moduleV1SchemaUndefined!, Application.Aggregates.Contract.Types.ModuleSchemaVersion.Undefined); // v1, v2, or v3
                     }
-                    if (GetSchemaFromWasmCustomSection(module, "concordium-schema-v1", out var moduleV1SchemaV1))
+                    if (GetSchemaFromWasmCustomSection(module, "concordium-schema-v2", out var moduleV1SchemaV1))
                     {
                         return (moduleV1SchemaV1!, Application.Aggregates.Contract.Types.ModuleSchemaVersion.V1); // v1 (not a typo)
                     }
@@ -152,7 +163,7 @@ private static bool GetSchemaFromWasmCustomSection(WebAssembly.Module module, st
         {
             schema = null;
             var customSection = module.CustomSections
-                .SingleOrDefault(section => section.Name.StartsWith(entryKey, StringComparison.InvariantCulture));
+                .SingleOrDefault(section => section.Name.Equals(entryKey, StringComparison.InvariantCulture));
 
             if (customSection == null) return false;
 

diff --git a/backend/Application/Aggregates/Contract/Extensions/ContractExtensions.cs b/backend/Application/Aggregates/Contract/Extensions/ContractExtensions.cs
@@ -70,6 +70,7 @@ private static void AddContractJobs(this IServiceCollection collection)
 
         collection.AddSingleton<IContractJobRepository, ContractJobRepository>();
         collection.AddTransient<IContractJob, InitialModuleSourceCatchup>();
+        collection.AddTransient<IContractJob, UpdateModuleSourceCatchup>();
         collection.AddTransient<IContractJob, ParallelBatchBlockHeightJob<InitialContractAggregateCatchUpJob>>();
         collection.AddTransient<InitialContractAggregateCatchUpJob>();
     }

diff --git a/backend/Application/Aggregates/Contract/Jobs/UpdateModuleSourceCatchup.cs b/backend/Application/Aggregates/Contract/Jobs/UpdateModuleSourceCatchup.cs
@@ -0,0 +1,112 @@
+using System.Threading;
+using System.Threading.Tasks;
+using Application.Aggregates.Contract.Configurations;
+using Application.Aggregates.Contract.Entities;
+using Application.Aggregates.Contract.Observability;
+using Application.Aggregates.Contract.Resilience;
+using Application.Api.GraphQL.EfCore;
+using Application.Observability;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Options;
+using Serilog.Context;
+
+namespace Application.Aggregates.Contract.Jobs;
+
+/// <summary>
+/// Update all <see cref="ModuleReferenceEvent"/> where <see cref="ModuleReferenceEvent.Schema"/> isn't null.
+///
+/// Overrides existing <see cref="ModuleReferenceEvent.Schema"/> and <see cref="ModuleReferenceEvent.SchemaVersion"/>
+/// with logic from <see cref="ModuleReferenceEvent.ModuleSourceInfo.GetModuleSchema"/>.
+/// </summary>
+public class UpdateModuleSourceCatchup : IContractJob
+{
+    private readonly IContractNodeClient _client;
+    private readonly IDbContextFactory<GraphQlDbContext> _dbContextFactory;
+    private readonly ContractHealthCheck _healthCheck;
+    private readonly ILogger _logger;
+    private readonly ContractAggregateOptions _contractAggregateOptions;
+    private readonly ContractAggregateJobOptions _jobOptions;
+
+    /// <summary>
+    /// WARNING - Do not change this if job already executed on environment, since it will trigger rerun of job.
+    /// </summary>
+    private const string JobName = "UpdateModuleSourceCatchup";
+
+    public UpdateModuleSourceCatchup(
+        IContractNodeClient client,
+        IDbContextFactory<GraphQlDbContext> dbContextFactory,
+        IOptions<ContractAggregateOptions> options,
+        ContractHealthCheck healthCheck)
+    {
+        _client = client;
+        _dbContextFactory = dbContextFactory;
+        _healthCheck = healthCheck;
+        _logger = Log.ForContext<UpdateModuleSourceCatchup>();
+        _contractAggregateOptions = options.Value;
+        var gotJobOptions = _contractAggregateOptions.Jobs.TryGetValue(GetUniqueIdentifier(), out var jobOptions);
+        _jobOptions = gotJobOptions ? jobOptions! : new ContractAggregateJobOptions();    
+    }
+
+    private async Task<IList<string>> GetModuleReferences()
+    {
+        await using var context = await _dbContextFactory.CreateDbContextAsync();
+
+        return await context.ModuleReferenceEvents
+            .AsNoTracking()
+            .Where(m => m.ModuleSource != null)
+            .Select(m => m.ModuleReference)
+            .ToListAsync();
+    }
+
+    private async ValueTask Process(string moduleReference, ulong lastFinalized, CancellationToken token)
+    {
+        await Policies.GetTransientPolicy(_logger, _contractAggregateOptions.RetryCount, _contractAggregateOptions.RetryDelay)
+            .ExecuteAsync(async () =>
+            {
+                await using var context = await _dbContextFactory.CreateDbContextAsync(token);
+
+                var module = await context.ModuleReferenceEvents
+                    .SingleAsync(m => m.ModuleReference == moduleReference, cancellationToken: token);
+
+                var moduleSourceInfo = await ModuleReferenceEvent.ModuleSourceInfo.Create(_client, lastFinalized, moduleReference);
+                module.UpdateWithModuleSourceInfo(moduleSourceInfo);
+                await context.SaveChangesAsync(token);
+
+                _logger.Information($"Updated module {moduleReference}");
+            });
+    }
+
+    public async Task StartImport(CancellationToken token)
+    {
+        using var _ = TraceContext.StartActivity(GetUniqueIdentifier());
+        using var __ = LogContext.PushProperty("Job", GetUniqueIdentifier());
+
+        try
+        {
+            var moduleReferences = await GetModuleReferences();
+
+            _logger.Information($"Starts process {moduleReferences.Count} modules");
+
+            var consensusInfo = await _client.GetConsensusInfoAsync(token);
+
+            var cycle = Parallel.ForEachAsync(moduleReferences,
+                new ParallelOptions
+                {
+                    MaxDegreeOfParallelism = _jobOptions.MaxParallelTasks
+                }, (moduleRef, cancellationToken) => Process(moduleRef, consensusInfo.LastFinalizedBlockHeight, cancellationToken));
+            await cycle;
+
+            _logger.Information($"Done with job {GetUniqueIdentifier()}");
+        }
+        catch (Exception e)
+        {
+            _healthCheck.AddUnhealthyJobWithMessage(GetUniqueIdentifier(), "Job stopped due to exception.");
+            _logger.Fatal(e, $"{GetUniqueIdentifier()} stopped due to exception.");
+            throw;
+        }
+    }
+
+    public string GetUniqueIdentifier() => JobName;
+
+    public bool ShouldNodeImportAwait() => false;
+}
diff --git a/backend/CHANGELOG.md b/backend/CHANGELOG.md
@@ -1,7 +1,10 @@
 ## Unreleased changes
 
 ## 1.7.7
-- Added endpoint to display module schema in a human interpretable form. The schema is present if it is embedded in the module source.
+- Added
+    - Endpoint to display module schema in a human interpretable form. The schema is present if it is embedded in the module source.
+- Bugfix
+    - Schema was incorrectly always mapped to undefined schema version. Fix implemented and job added which cleans up corrupted data.
 
 ## 1.7.4
 - Change contract- and module queries to use offset pagination.

diff --git a/backend/Tests/Aggregates/Contract/Entities/ModuleReferenceEventTests.cs b/backend/Tests/Aggregates/Contract/Entities/ModuleReferenceEventTests.cs
@@ -3,6 +3,7 @@
 using System.Threading;
 using Application.Aggregates.Contract;
 using Application.Aggregates.Contract.Entities;
+using Application.Aggregates.Contract.Types;
 using Concordium.Sdk.Client;
 using Concordium.Sdk.Types;
 using FluentAssertions;
@@ -14,6 +15,34 @@ namespace Tests.Aggregates.Contract.Entities;
 
 public sealed class ModuleReferenceEventTests
 {
+    [Theory]
+    [InlineData("cis1-wccd-embedded-schema-v0-unversioned.wasm", ModuleSchemaVersion.V0, false, 0)]
+    [InlineData("cis1-wccd-embedded-schema-v0-versioned.wasm.v0", ModuleSchemaVersion.Undefined, true, 0)]
+    [InlineData("cis2-wccd-embedded-schema-v1-unversioned.wasm.v1", ModuleSchemaVersion.V1, true, 1)]
+    [InlineData("cis2-wccd-embedded-schema-v1-versioned.wasm.v1", ModuleSchemaVersion.Undefined, true, 1)]
+    public async Task WhenCreateModuleSchema_ThenParseWithCorrectVersion(string fileName, ModuleSchemaVersion version, bool trim, int moduleVersion)
+    {
+        var client = new Mock<IContractNodeClient>();
+        var bytes = (await File.ReadAllBytesAsync($"./TestUtilities/TestData/{fileName}"));
+        if (trim)
+        {
+            bytes = bytes[8..];
+        }
+        VersionedModuleSource module = moduleVersion == 0 ? new ModuleV0(bytes) : new ModuleV1(bytes);
+        var queryResponseModuleSource = new QueryResponse<VersionedModuleSource>(BlockHash.From(new byte[32]), module);
+
+        client.Setup(c => c.GetModuleSourceAsync(It.IsAny<IBlockHashInput>(), It.IsAny<ModuleReference>(), It.IsAny<CancellationToken>()))
+            .Returns(Task.FromResult(queryResponseModuleSource));
+
+        // Act
+        var moduleSchema = await ModuleReferenceEvent.ModuleSourceInfo.Create(client.Object, 0, Convert.ToHexString(new byte[32]));
+
+        // Assert
+        moduleSchema.ModuleSource.Should().Be(Convert.ToHexString(bytes));
+        moduleSchema.ModuleSchemaVersion.Should().Be(version);
+    }
+
+
     [Theory]
     [InlineData("module.schema_embedded.wasm.hex", "FFFF03010000000C00000054657374436F6E7472616374000000000001150200000003000000466F6F020300000042617202")]
     [InlineData("module.wasm.hex", null)]

diff --git a/backend/Tests/TestUtilities/TestData/cis1-wccd-embedded-schema-v0-unversioned.wasm b/backend/Tests/TestUtilities/TestData/cis1-wccd-embedded-schema-v0-unversioned.wasm
diff --git a/backend/Tests/TestUtilities/TestData/cis1-wccd-embedded-schema-v0-versioned.wasm.v0 b/backend/Tests/TestUtilities/TestData/cis1-wccd-embedded-schema-v0-versioned.wasm.v0
diff --git a/backend/Tests/TestUtilities/TestData/cis2-wccd-embedded-schema-v1-unversioned.wasm.v1 b/backend/Tests/TestUtilities/TestData/cis2-wccd-embedded-schema-v1-unversioned.wasm.v1
diff --git a/backend/Tests/TestUtilities/TestData/cis2-wccd-embedded-schema-v1-versioned.wasm.v1 b/backend/Tests/TestUtilities/TestData/cis2-wccd-embedded-schema-v1-versioned.wasm.v1
diff --git a/backend/Tests/Tests.csproj b/backend/Tests/Tests.csproj
@@ -54,5 +54,17 @@
       <None Update="TestUtilities\TestData\cis2-nft-schema">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
       </None>
+      <None Update="TestUtilities\TestData\cis2-wccd-embedded-schema-v1-unversioned.wasm.v1">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="TestUtilities\TestData\cis2-wccd-embedded-schema-v1-versioned.wasm.v1">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="TestUtilities\TestData\cis1-wccd-embedded-schema-v0-unversioned.wasm">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
+      <None Update="TestUtilities\TestData\cis1-wccd-embedded-schema-v0-versioned.wasm.v0">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      </None>
     </ItemGroup>
 </Project>