Skip to content

Commit

Permalink
Surrogate id from UTC (#3879)
Browse files Browse the repository at this point in the history 
* Surrogate id from utc

* Test
  • Loading branch information
@SergeyGaluzo
SergeyGaluzo authored May 30, 2024
1 parent bec76c4 commit 0d443c0
Show file tree
Hide file tree
Showing 16 changed files with 76 additions and 41 deletions.
13 changes: 7 additions & 6 deletions src/Microsoft.Health.Fhir.Core/Extensions/IdHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,20 +20,21 @@ internal static class IdHelper
{
private const int ShiftFactor = 3;

internal static readonly DateTime MaxDateTime = new DateTime(long.MaxValue >> ShiftFactor, DateTimeKind.Utc).TruncateToMillisecond().AddTicks(-1);
internal static readonly DateTimeOffset MaxDateTime = new DateTime(long.MaxValue >> ShiftFactor, DateTimeKind.Utc).TruncateToMillisecond().AddTicks(-1);

public static long DateToId(this DateTime dateTime)
public static long ToId(this DateTimeOffset dateTimeOffset)
{
EnsureArg.IsLte(dateTime, MaxDateTime, nameof(dateTime));
long id = dateTime.TruncateToMillisecond().Ticks << ShiftFactor;
EnsureArg.IsLte(dateTimeOffset, MaxDateTime, nameof(dateTimeOffset));
long id = dateTimeOffset.UtcDateTime.TruncateToMillisecond().Ticks << ShiftFactor;

Debug.Assert(id >= 0, "The ID should not have become negative");
return id;
}

public static DateTime IdToDate(this long resourceSurrogateId)
public static DateTimeOffset ToDate(this long resourceSurrogateId)
{
var dateTime = new DateTime(resourceSurrogateId >> ShiftFactor, DateTimeKind.Utc);
return dateTime.TruncateToMillisecond();
var offset = new DateTimeOffset(dateTime.TruncateToMillisecond(), TimeSpan.Zero);
return offset;
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ await container.Value.UpsertItemAsync(

private static long GetLongId()
{
return Clock.UtcNow.DateTime.DateToId() + RandomNumberGenerator.GetInt32(100, 999);
return Clock.UtcNow.ToId() + RandomNumberGenerator.GetInt32(100, 999);
}

/// <summary>
Expand Down
2 changes: 1 addition & 1 deletion ...er.UnitTests/Features/Search/Expressions/LastUpdatedToResourceSurrogateIdRewriterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public void GivenAnExpressionOverLastUpdated_WhenTranslatedToResourceSurrogateId
BinaryExpression binaryOutput = Assert.IsType<BinaryExpression>(output);
Assert.Equal(SqlFieldName.ResourceSurrogateId, binaryOutput.FieldName);
Assert.Equal(expectedOperator, binaryOutput.BinaryOperator);
Assert.Equal(DateTimeOffset.Parse(expectedDateTimeOffset), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated((long)binaryOutput.Value));
Assert.Equal(DateTimeOffset.Parse(expectedDateTimeOffset), ((long)binaryOutput.Value).ToLastUpdated());
}
}
}
16 changes: 8 additions & 8 deletions src/Microsoft.Health.Fhir.SqlServer.UnitTests/ResourceSurrogateIdHelperTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,22 @@ public class ResourceSurrogateIdHelperTests
[Fact]
public void GivenADateTime_WhenRepresentedAsASurrogateId_HasTheExpectedRange()
{
var baseDate = DateTime.MinValue;
long baseId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(baseDate);
var baseDate = DateTimeOffset.MinValue;
long baseId = baseDate.ToSurrogateId();

Assert.Equal(baseDate, ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(baseId + 79999));
Assert.Equal(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(baseId + 80000) - baseDate);
Assert.Equal(baseDate, (baseId + 79999).ToLastUpdated());
Assert.Equal(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond), (baseId + 80000).ToLastUpdated() - baseDate);

long maxBaseId = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(ResourceSurrogateIdHelper.MaxDateTime);
long maxBaseId = ResourceSurrogateIdHelper.MaxDateTime.ToSurrogateId();

Assert.Equal(ResourceSurrogateIdHelper.MaxDateTime.TruncateToMillisecond(), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(maxBaseId));
Assert.Equal(ResourceSurrogateIdHelper.MaxDateTime.TruncateToMillisecond(), ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(maxBaseId + 79999));
Assert.Equal(ResourceSurrogateIdHelper.MaxDateTime.UtcDateTime.TruncateToMillisecond(), maxBaseId.ToLastUpdated());
Assert.Equal(ResourceSurrogateIdHelper.MaxDateTime.UtcDateTime.TruncateToMillisecond(), (maxBaseId + 79999).ToLastUpdated());
}

[Fact]
public void GivenADateTimeLargerThanTheLargestThatCanBeRepresentedAsASurrogateId_WhenTurnedIntoASurrogateId_Throws()
{
Assert.Throws<ArgumentOutOfRangeException>(() => ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.MaxValue));
Assert.Throws<ArgumentOutOfRangeException>(() => DateTimeOffset.MaxValue.ToSurrogateId());
}
}
}
4 changes: 2 additions & 2 deletions src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@ public async Task<string> ExecuteAsync(JobInfo jobInfo, CancellationToken cancel

var since = record.Since == null ? new PartialDateTime(DateTime.MinValue).ToDateTimeOffset() : record.Since.ToDateTimeOffset();

var globalStartId = since.DateTime.DateToId();
var globalStartId = since.ToId();
var till = record.Till.ToDateTimeOffset();
var globalEndId = till.DateTime.DateToId() - 1; // -1 is so _till value can be used as _since in the next time based export
var globalEndId = till.ToId() - 1; // -1 is so _till value can be used as _since in the next time based export

var enqueued = groupJobs.Where(x => x.Id != jobInfo.Id) // exclude coord
.Select(x => JsonConvert.DeserializeObject<ExportJobRecord>(x.Definition))
Expand Down
8 changes: 4 additions & 4 deletions ...qlServer/Features/Search/Expressions/Visitors/LastUpdatedToResourceSurrogateIdRewriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,14 +56,14 @@ public override Expression VisitBinary(BinaryExpression expression, object conte
return Expression.GreaterThanOrEqual(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated.AddTicks(TimeSpan.TicksPerMillisecond)));
new DateTimeOffset(truncated.AddTicks(TimeSpan.TicksPerMillisecond)).ToSurrogateId());
case BinaryOperator.GreaterThanOrEqual:
if (original == truncated)
{
return Expression.GreaterThanOrEqual(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated));
new DateTimeOffset(truncated).ToSurrogateId());
}

goto case BinaryOperator.GreaterThan;
Expand All @@ -73,15 +73,15 @@ public override Expression VisitBinary(BinaryExpression expression, object conte
return Expression.LessThan(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated));
new DateTimeOffset(truncated).ToSurrogateId());
}

goto case BinaryOperator.LessThanOrEqual;
case BinaryOperator.LessThanOrEqual:
return Expression.LessThan(
SqlFieldName.ResourceSurrogateId,
null,
ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(truncated.AddTicks(TimeSpan.TicksPerMillisecond)));
new DateTimeOffset(truncated.AddTicks(TimeSpan.TicksPerMillisecond)).ToSurrogateId());
case BinaryOperator.NotEqual:
case BinaryOperator.Equal: // expecting eq to have been rewritten as a range
default:
Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ await _sqlRetryService.ExecuteSql(
_model.GetResourceTypeName(resourceTypeId),
clonedSearchOptions.OnlyIds ? null : new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet),
new ResourceRequest(requestMethod),
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
resourceSurrogateId.ToLastUpdated(),
isDeleted,
null,
null,
Expand Down Expand Up @@ -648,7 +648,7 @@ await _sqlRetryService.ExecuteSql(
resourceType,
new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet),
new ResourceRequest(requestMethod),
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
resourceSurrogateId.ToLastUpdated(),
isDeleted,
null,
null,
Expand Down
16 changes: 11 additions & 5 deletions src/Microsoft.Health.Fhir.SqlServer/Features/Storage/ResourceSurrogateIdHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
// -------------------------------------------------------------------------------------------------

using System;
using Microsoft.Health.Core.Extensions;
using Microsoft.Health.Fhir.Core.Extensions;

namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
Expand All @@ -14,16 +15,21 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage
/// </summary>
internal static class ResourceSurrogateIdHelper
{
public static DateTime MaxDateTime => IdHelper.MaxDateTime;
public static DateTimeOffset MaxDateTime => IdHelper.MaxDateTime;

public static long LastUpdatedToResourceSurrogateId(DateTime dateTime)
public static long ToSurrogateId(this DateTimeOffset dateTimeOffset)
{
return dateTime.DateToId();
return dateTimeOffset.ToId();
}

public static DateTime ResourceSurrogateIdToLastUpdated(long resourceSurrogateId)
public static DateTimeOffset ToLastUpdated(this long resourceSurrogateId)
{
return resourceSurrogateId.IdToDate();
return resourceSurrogateId.ToDate();
}

public static DateTimeOffset TruncateToMillisecond(this DateTimeOffset dateTimeOffset)
{
return new DateTimeOffset(dateTimeOffset.DateTime.TruncateToMillisecond(), dateTimeOffset.Offset);
}
}
}
2 changes: 1 addition & 1 deletion src/Microsoft.Health.Fhir.SqlServer/Features/Storage/ResourceWrapperExtention.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public static class ResourceWrapperExtention
{
public static ResourceDateKey ToResourceDateKey(this ResourceWrapper wrapper, Func<string, short> getResourceTypeId, bool ignoreVersion = false)
{
return new ResourceDateKey(getResourceTypeId(wrapper.ResourceTypeName), wrapper.ResourceId, ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(wrapper.LastModified.DateTime), ignoreVersion ? null : wrapper.Version);
return new ResourceDateKey(getResourceTypeId(wrapper.ResourceTypeName), wrapper.ResourceId, wrapper.LastModified.ToSurrogateId(), ignoreVersion ? null : wrapper.Version);
}
}
}
6 changes: 3 additions & 3 deletions src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,12 +302,12 @@ private async Task<IDictionary<DataStoreOperationIdentifier, DataStoreOperationO
if (!keepLastUpdated || _ignoreInputLastUpdated.IsEnabled())
{
surrId = transactionId + index;
resource.LastModified = new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(surrId), TimeSpan.Zero);
resource.LastModified = surrId.ToLastUpdated();
ReplaceVersionIdAndLastUpdatedInMeta(resource);
}
else
{
var surrIdBase = ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(resource.LastModified.DateTime);
var surrIdBase = resource.LastModified.ToSurrogateId();
surrId = surrIdBase + minSequenceId + index;
ReplaceVersionIdInMeta(resource);
singleTransaction = true; // There is no way to rollback until TransactionId is added to Resource table
Expand Down Expand Up @@ -457,7 +457,7 @@ List<string> GetErrors(IEnumerable<ImportResource> dups, IEnumerable<ImportResou
.GroupBy(_ => new ResourceDateKey(
_model.GetResourceTypeId(_.ResourceWrapper.ResourceTypeName),
_.ResourceWrapper.ResourceId,
_.KeepLastUpdated ? ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(_.ResourceWrapper.LastModified.DateTime) : 0,
_.KeepLastUpdated ? _.ResourceWrapper.LastModified.ToSurrogateId() : 0,
null))
.Select(_ => _.OrderByDescending(_ => _.ResourceWrapper.Version).First())
.ToList();
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ private static ResourceWrapper ReadResourceWrapper(SqlDataReader reader, bool re
getResourceTypeName(resourceTypeId),
new RawResource(rawResource, FhirResourceFormat.Json, isMetaSet: isRawResourceMetaSet),
readRequestMethod ? new ResourceRequest(requestMethod) : null,
new DateTimeOffset(ResourceSurrogateIdHelper.ResourceSurrogateIdToLastUpdated(resourceSurrogateId), TimeSpan.Zero),
resourceSurrogateId.ToLastUpdated(),
isDeleted,
searchIndices: null,
compartmentIndices: null,
Expand Down
4 changes: 2 additions & 2 deletions test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/HistoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -648,13 +648,13 @@ private void AssertCount<TBase>(int expected, ICollection<TBase> collection, Dat
if (since.HasValue)
{
sb.AppendLine($"since={since.Value.ToString("o")}");
sb.AppendLine($"sinceSurr={SqlServer.Features.Storage.ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(since.Value.DateTime)}");
sb.AppendLine($"sinceSurr={since.Value.ToId()}");
}

if (before.HasValue)
{
sb.AppendLine($"before={before.Value.ToString("o")}");
sb.AppendLine($"beforeSurr={SqlServer.Features.Storage.ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(before.Value.DateTime)}");
sb.AppendLine($"beforeSurr={before.Value.ToId()}");
}

throw new XunitException(sb.ToString());
Expand Down
28 changes: 28 additions & 0 deletions test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
using Microsoft.Health.Fhir.Core.Features.Operations.Import;
using Microsoft.Health.Fhir.Core.Features.Operations.Import.Models;
using Microsoft.Health.Fhir.SqlServer.Features.Operations.Import;
using Microsoft.Health.Fhir.SqlServer.Features.Storage;
using Microsoft.Health.Fhir.Tests.Common;
using Microsoft.Health.Fhir.Tests.Common.FixtureParameters;
using Microsoft.Health.Fhir.Tests.E2E.Common;
Expand Down Expand Up @@ -182,6 +183,33 @@ public async Task GivenIncrementalLoad_80KSurrogateIds_BadRequestIsReturned()
Assert.Contains(ImportProcessingJob.SurrogateIdsErrorMessage, await message.Content.ReadAsStringAsync());
}

[Fact]
public async Task GivenIncrementalLoad_TruncatedLastUpdatedPreservedWithOffset()
{
var id = Guid.NewGuid().ToString("N");
var lastUpdated = new DateTimeOffset(DateTime.Parse(DateTime.Now.AddYears(-1).ToString()).AddSeconds(0.0001), TimeSpan.FromHours(10));
var ndJson = CreateTestPatient(id, lastUpdated);
var location = (await ImportTestHelper.UploadFileAsync(ndJson, _fixture.StorageAccount)).location;
var request = CreateImportRequest(location, ImportMode.IncrementalLoad, false);
await ImportCheckAsync(request, null, 0);

var history = await _client.SearchAsync($"Patient/{id}/_history");
Assert.Single(history.Resource.Entry);
Assert.Equal("1", history.Resource.Entry[0].Resource.VersionId);
Assert.Equal(lastUpdated.TruncateToMillisecond(), history.Resource.Entry[0].Resource.Meta.LastUpdated);

var lastUpdatedUtc = new DateTimeOffset(lastUpdated.DateTime.AddHours(-10), TimeSpan.Zero);
Assert.Equal(lastUpdated.UtcDateTime, lastUpdatedUtc.UtcDateTime); // the same date in UTC sense
ndJson = CreateTestPatient(id, lastUpdatedUtc);
location = (await ImportTestHelper.UploadFileAsync(ndJson, _fixture.StorageAccount)).location;
request = CreateImportRequest(location, ImportMode.IncrementalLoad, false);
await ImportCheckAsync(request, null, 0); // reported loaded

// but was not inserted TODO: uncomment Single once version 82 is released
history = await _client.SearchAsync($"Patient/{id}/_history");
////Assert.Single(history.Resource.Entry);
}

[Fact]
public async Task GivenIncrementalLoad_MultipleImportsWithSameLastUpdatedAndExplicitVersion()
{
Expand Down
2 changes: 1 addition & 1 deletion ...t.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ private void PrepareData()
{
ExecuteSql("TRUNCATE TABLE dbo.JobQueue");
ExecuteSql("TRUNCATE TABLE dbo.Resource");
var surrId = DateTime.UtcNow.DateToId();
var surrId = DateTimeOffset.UtcNow.ToId();
ExecuteSql(@$"
INSERT INTO Resource
(ResourceTypeId,ResourceId,Version,IsHistory,ResourceSurrogateId,IsDeleted,RequestMethod,RawResource,IsRawResourceMetaSet,SearchParamHash)
Expand Down
6 changes: 3 additions & 3 deletions test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public async Task TimeTravel()
await Mediator.UpsertResourceAsync(patient.ToResourceElement());

await Task.Delay(100); // avoid time -> surrogate id -> time round trip error
var till = DateTime.UtcNow;
var till = DateTimeOffset.UtcNow;

var results = await _fixture.SearchService.SearchAsync(type, new List<Tuple<string, string>>(), CancellationToken.None);
Assert.Single(results.Results);
Expand All @@ -150,7 +150,7 @@ public async Task TimeTravel()
Assert.Empty(results.Results);

// add magic parameters
var maxId = till.DateToId();
var maxId = till.ToId();
var range = (await _fixture.SearchService.GetSurrogateIdRanges(type, 0, maxId, 100, 1, true, CancellationToken.None)).First();
queryParameters = new[]
{
Expand All @@ -176,7 +176,7 @@ public async Task TimeTravel()
Assert.False(resource.IsHistory); // current

// use surr id interval that covers all changes
maxId = DateTime.UtcNow.DateToId();
maxId = DateTimeOffset.UtcNow.ToId();
range = (await _fixture.SearchService.GetSurrogateIdRanges(type, 0, maxId, 100, 1, true, CancellationToken.None)).First();
queryParameters = new[]
{
Expand Down
2 changes: 1 addition & 1 deletion tools/PerfTester/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public static void Main()

if (_callType == "SingleId" || _callType == "HttpUpdate" || _callType == "HttpCreate" || _callType == "BundleUpdate")
{
Console.WriteLine($"Start at {DateTime.UtcNow.ToString("s")} surrogate Id = {ResourceSurrogateIdHelper.LastUpdatedToResourceSurrogateId(DateTime.UtcNow)}");
Console.WriteLine($"Start at {DateTime.UtcNow.ToString("s")} surrogate Id = {DateTimeOffset.UtcNow.ToSurrogateId()}");
ExecuteParallelHttpPuts();
return;
}
Expand Down

0 comments on commit 0d443c0

Please sign in to comment.