Export fix

backend/Application/Api/Rest/EnumerableFileResult.cs

@@ -0,0 +1,75 @@
+/// https://github.com/philipmat/EnumerableStreamFileResult
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using System.IO;
+
+namespace EnumerableStreamFileResult
+{
+    class EnumerableFileResult<T> : FileResult
+    {
+        private readonly IEnumerable<T> _enumeration;
+        private readonly IStreamWritingAdapter<T> _writer;
+
+        public EnumerableFileResult(IEnumerable<T> enumeration, IStreamWritingAdapter<T> writer)
+            : base(writer.ContentType)
+        {
+            _enumeration = enumeration ?? throw new ArgumentNullException(nameof(enumeration));
+            _writer = writer ?? throw new ArgumentNullException(nameof(writer));
+        }
+
+        public override async Task ExecuteResultAsync(ActionContext context)
+        {
+            SetContentType(context);
+            SetContentDispositionHeader(context);
+
+            await WriteContentAsync(context).ConfigureAwait(false);
+        }
+
+        private async Task WriteContentAsync(ActionContext context)
+        {
+            var body = context.HttpContext.Response.Body;
+            await _writer.WriteHeaderAsync(body).ConfigureAwait(false);
+            int recordCount = 0;
+            foreach (var item in _enumeration)
+            {
+                await _writer.WriteAsync(item, body).ConfigureAwait(false);
+                recordCount++;
+            }
+
+            await _writer.WriteFooterAsync(body, recordCount);
+
+            await base.ExecuteResultAsync(context).ConfigureAwait(false);
+        }
+
+        private void SetContentDispositionHeader(ActionContext context)
+        {
+            var headers = context.HttpContext.Response.Headers;
+            var cd = new System.Net.Mime.ContentDisposition
+            {
+                FileName = FileDownloadName,
+                Inline = false, // false = attachement
+            };
+
+            headers.Add(
+                "Content-Disposition",
+                new Microsoft.Extensions.Primitives.StringValues(cd.ToString()));
+        }
+
+        private void SetContentType(ActionContext context)
+        {
+            var response = context.HttpContext.Response;
+            response.ContentType = ContentType;
+        }
+    }
+
+    internal interface IStreamWritingAdapter<T>
+    {
+       string ContentType { get; }
+
+        Task WriteHeaderAsync(Stream stream);
+
+        Task WriteAsync(T item, Stream stream);
+
+        Task WriteFooterAsync(Stream stream, int recordCount); 
+    }
+}

backend/Application/Api/Rest/ExportController.cs

@@ -1,11 +1,14 @@
 using System.Globalization;
 using System.Threading.Tasks;
 using System.Text;
+
 using Application.Api.GraphQL.EfCore;
 using Concordium.Sdk.Types;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using AccountAddress = Application.Api.GraphQL.Accounts.AccountAddress;
+using System.IO;
+using EnumerableStreamFileResult;
 
 namespace Application.Api.Rest;
 
@@ -24,7 +27,7 @@ public ExportController(IDbContextFactory<GraphQlDbContext> dbContextFactory)
     [Route("rest/export/statement")]
     public async Task<ActionResult> GetStatementExport(string accountAddress, DateTime? fromTime, DateTime? toTime)
     {
-        await using var dbContext = await _dbContextFactory.CreateDbContextAsync();
+        var dbContext = await _dbContextFactory.CreateDbContextAsync();
 
         if (!Concordium.Sdk.Types.AccountAddress.TryParse(accountAddress, out var parsed))
         {
@@ -42,21 +45,22 @@ public async Task<ActionResult> GetStatementExport(string accountAddress, DateTi
 
         var query = dbContext.AccountStatementEntries
             .AsNoTracking()
+            .OrderByDescending(x => x.Timestamp)
             .Where(x => x.AccountId == account.Id);
 
         if (fromTime != null && fromTime.Value.Kind != DateTimeKind.Utc)
         {
             return BadRequest("Time zone missing on 'fromTime'.");
         }
 
-        DateTimeOffset from = fromTime ?? DateTime.Now.AddDays(-31);
+        DateTimeOffset from = fromTime ?? DateTime.UtcNow.AddDays(-31);
 
         if (toTime != null && toTime.Value.Kind != DateTimeKind.Utc)
         {
             return BadRequest("Time zone missing on 'toTime'.");
         }
 
-        DateTimeOffset to = toTime ?? DateTime.Now;
+        DateTimeOffset to = toTime ?? DateTime.UtcNow;
 
         if ((to - from).TotalDays > MAX_DAYS)
         {
@@ -66,37 +70,36 @@ public async Task<ActionResult> GetStatementExport(string accountAddress, DateTi
         query = query.Where(e => e.Timestamp >= from);
         query = query.Where(e => e.Timestamp <= to);
 
-        var result = query.Select(x => new
-        {
-            x.Timestamp,
-            x.EntryType,
-            x.Amount,
-            x.AccountBalance
-        });
-        var values = await result.ToListAsync();
-        if (values.Count == 0)
+        var result = query.Select(x => string.Format(
+            "{0}, {1}, {2}, {3}\n",
+            x.Timestamp.ToString("u"),
+            (x.Amount / (decimal)CcdAmount.MicroCcdPerCcd).ToString(CultureInfo.InvariantCulture),
+            (x.AccountBalance / (decimal)CcdAmount.MicroCcdPerCcd).ToString(CultureInfo.InvariantCulture),
+            x.EntryType
+            )
+        );
+
+        return new EnumerableFileResult<string>(result, new StreamWritingAdapter())
         {
-            return new NoContentResult();
-        }
+            FileDownloadName = $"statement-{accountAddress}_{from:yyyyMMdd}Z-{to:yyyyMMdd}Z.csv"
+        };
+    }
+    private class StreamWritingAdapter : IStreamWritingAdapter<string>
+    {
+        public string ContentType => "text/csv";
 
-        var csv = new StringBuilder("Time,Amount (CCD),Balance (CCD),Label\n");
-        foreach (var v in values)
+        public async Task WriteAsync(string item, Stream stream)
         {
-            csv.Append(v.Timestamp.ToString("u"));
-            csv.Append(',');
-            csv.Append((v.Amount / (decimal)CcdAmount.MicroCcdPerCcd).ToString(CultureInfo.InvariantCulture));
-            csv.Append(',');
-            csv.Append((v.AccountBalance / (decimal)CcdAmount.MicroCcdPerCcd).ToString(CultureInfo.InvariantCulture));
-            csv.Append(',');
-            csv.Append(v.EntryType);
-            csv.Append('\n');
+            byte[] line = Encoding.ASCII.GetBytes(item);
+            await stream.WriteAsync(line);
         }
 
-        var firstTime = values.First().Timestamp;
-        var lastTime = values.Last().Timestamp;
-        return new FileContentResult(Encoding.ASCII.GetBytes(csv.ToString()), "text/csv")
+        public Task WriteFooterAsync(Stream stream, int recordCount) => Task.CompletedTask;
+        public async Task WriteHeaderAsync(Stream stream)
         {
-            FileDownloadName = $"statement-{accountAddress}_{firstTime:yyyyMMddHHmmss}Z-{lastTime:yyyyMMddHHmmss}Z.csv"
-        };
+            byte[] line = Encoding.ASCII.GetBytes("Time,Amount (CCD),Balance (CCD),Label\n");
+            await stream.WriteAsync(line);
+        }
     }
 }
+

backend/Application/Application.csproj

@@ -4,7 +4,7 @@
         <TargetFramework>net6.0</TargetFramework>
         <Nullable>enable</Nullable>
         <ImplicitUsings>disable</ImplicitUsings>
-        <Version>1.8.18</Version>
+        <Version>1.8.19</Version>
         <IsWindows Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindows>
         <IsOSX Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsOSX>
         <IsLinux Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinux>

backend/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## Unreleased changes
 
+## 1.8.19
+- Bugfix
+    - Fix performance of account statement export, by adding an index on table `graphql_account_statement_entries`, and streaming the data.
+
 ## 1.8.18
 - Bugfix
     - Fix performance view `graphql_account_rewards` by adding an index on table `graphql_account_statement_entries`.

backend/DatabaseScripts/SqlScripts/0054_create_index_on_graphql_account_statement_for_timestamp.sql

@@ -0,0 +1,4 @@
+/*
+ Creates an index on time on graphql_account_statement_entries which is used by account statement export
+ */
+create index account_statement_entries_account_id_index_rewards_timestamp on graphql_account_statement_entries (account_id, time);

backend/Tests/Api/Rest/ExportControllerTest.cs

@@ -5,6 +5,12 @@
 using FluentAssertions;
 using Tests.TestUtilities;
 using Tests.TestUtilities.Builders.GraphQL;
+using EnumerableStreamFileResult;
+using System.IO;
+using Microsoft.AspNetCore.Http;
+using Moq;
+using Microsoft.AspNetCore.Routing;
+using Microsoft.AspNetCore.Mvc.Controllers;
 
 namespace Tests.Api.Rest;
 
@@ -30,7 +36,7 @@ public async Task DisposeAsync()
     }
 
     [Fact]
-    public async void accountStatementWith33DaySpanIsNotAllowed()
+    public async void AccountStatementWith33DaySpanIsNotAllowed()
     {
         // Arrange
         var startDate = DateTime.SpecifyKind(new DateTime(2020, 11, 1), DateTimeKind.Utc);
@@ -54,7 +60,7 @@ public async void accountStatementWith33DaySpanIsNotAllowed()
     }
 
     [Fact]
-    public async void transactionOutsideSpecifiedTimeStampsAreNotReturned()
+    public async void TransactionOutsideSpecifiedTimeStampsAreNotReturned()
     {
         // Arrange
         var date1 = new DateTime(2020, 12, 1, 0, 0, 0, DateTimeKind.Utc);
@@ -66,6 +72,21 @@ public async void transactionOutsideSpecifiedTimeStampsAreNotReturned()
         var controller = new ExportController(_testHelper.dbContextFactory);
         var address = "3XSLuJcXg6xEua6iBPnWacc3iWh93yEDMCqX8FbE3RDSbEnT9P";
 
+        MemoryStream stream = new();
+        var _headers = new HeaderDictionary();
+
+        var httpResponseMock = new Mock<HttpResponse>();
+        httpResponseMock.Setup(mock => mock.Body).Returns(stream);
+        httpResponseMock.Setup(mock => mock.Headers).Returns(_headers);
+
+        var httpContextMock = new Mock<HttpContext>();
+        httpContextMock.Setup(mock => mock.Response).Returns(httpResponseMock.Object);
+
+        var _controllerContext = new ControllerContext
+        {
+            HttpContext = httpContextMock.Object
+        };
+
         _testHelper.DbContext.Accounts.Add(new AccountBuilder()
             .WithId(42)
             .WithCanonicalAddress(address, true)
@@ -80,8 +101,9 @@ public async void transactionOutsideSpecifiedTimeStampsAreNotReturned()
 
         // Act
         var actionResult = await controller.GetStatementExport(address, startDate, endDate);
-        var result = Assert.IsType<FileContentResult>(actionResult);
-        string csv = System.Text.Encoding.UTF8.GetString(result.FileContents);
+        var result = Assert.IsType<EnumerableFileResult<string>>(actionResult);
+        await result.ExecuteResultAsync(_controllerContext);
+        string csv = System.Text.Encoding.ASCII.GetString(stream.ToArray());
 
         // Assert
         Regex.Matches(csv, "\n").Count.Should().Be(2);