From b9967fd1951f50f40f5dade2f6ac1a8d81439150 Mon Sep 17 00:00:00 2001 From: Nick Chapsas Date: Tue, 15 May 2018 16:37:40 +0100 Subject: [PATCH 1/3] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f95d341..ae7eaee 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,7 @@ It is HIGHLY recommended that you use one of the `Async` methods to get the resu ```csharp var user = await cosmoStore.Query().FirstOrDefaultAsync(x => x.Username == "elfocrash"); -var users = await cosmoStore.Query().ToListAsync(x => x.HairColor == HairColor.Black); -var otherUsers = await cosmosStore.Query().Where(x => x.Name.StartsWith("Smit")).ToListAsync(cancellationToken) +var users = await cosmoStore.Query().Where(x => x.HairColor == HairColor.Black).ToListAsync(cancellationToken); // or you can use SQL From b067690809e3ecfa7a62b15244225caa4559d8b4 Mon Sep 17 00:00:00 2001 From: Nick Chapsas Date: Mon, 21 May 2018 09:18:17 +0100 Subject: [PATCH 2/3] Update README.md --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ae7eaee..fc26de9 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,12 @@ Cosmonaut is an object mapper that enables .NET developers to work with a CosmosDB using .NET objects. It eliminates the need for most of the data-access code that developers usually need to write. +### Getting started + +- [How to easily start using CosmosDB in your C# application in no time with Cosmonaut](http://chapsas.com/how-to-easily-start-using-cosmosdb-in-your-c-application-in-no-time-with-cosmonaut/) +- [(Video) Getting started with .NET Core and CosmosDB using Cosmonaut](http://chapsas.com/video-getting-started-with-net-core-and-cosmosdb-using-cosmonaut/) +- [(Video) How to save money in CosmosDB with Cosmonaut's Collection Sharing](http://chapsas.com/video-how-to-save-money-in-cosmosdb-with-cosmonauts-collection-sharing/) + ### Usage The idea is pretty simple. You can have one CosmoStore per entity (POCO/dtos etc) This entity will be used to create a collection in the cosmosdb and it will offer all the data access for this object @@ -172,7 +178,7 @@ Example: [CosmosCollection("somename")] ``` -#### Benchmarks +#### Benchmarks (Outdated. To be recalculated) ##### Averages of 1000 iterations for 500 documents per operation on collection with default indexing and 5000 RU/s (POCO serialization) From 1b1f84a712f77c1b0461893c38a6b9822c83a36a Mon Sep 17 00:00:00 2001 From: Nick Chapsas Date: Mon, 21 May 2018 21:29:41 +0100 Subject: [PATCH 3/3] Refactoring --- .../Extensions/CosmosSqlQueryExtensions.cs | 58 +++++++----- .../Extensions/DocumentEntityExtensions.cs | 89 ++++++++++++------- Cosmonaut/Extensions/ExpressionExtensions.cs | 4 +- Cosmonaut/Operations/CosmosScaler.cs | 26 +++--- Cosmonaut/Storage/CosmosCollectionCreator.cs | 4 +- 5 files changed, 109 insertions(+), 72 deletions(-) diff --git a/Cosmonaut/Extensions/CosmosSqlQueryExtensions.cs b/Cosmonaut/Extensions/CosmosSqlQueryExtensions.cs index a405151..051127f 100644 --- a/Cosmonaut/Extensions/CosmosSqlQueryExtensions.cs +++ b/Cosmonaut/Extensions/CosmosSqlQueryExtensions.cs @@ -27,16 +27,21 @@ internal static string EnsureQueryIsCollectionSharingFriendly(this stri var hasExistingWhereClause = sql.IndexOf(" where ", StringComparison.OrdinalIgnoreCase) >= 0; - if (hasExistingWhereClause) - { - var splitQuery = Regex.Split(sql, " where ", RegexOptions.IgnoreCase); - var firstPartQuery = splitQuery[0]; - var secondPartQuery = splitQuery[1]; - var sharedCollectionExpressionQuery = $"{identifier}.{nameof(ISharedCosmosEntity.CosmosEntityName)} = '{cosmosEntityNameValue}'"; - return $"{firstPartQuery} where {sharedCollectionExpressionQuery} and {secondPartQuery}"; - } + if (!hasExistingWhereClause) + return $"{sql} where {identifier}.{nameof(ISharedCosmosEntity.CosmosEntityName)} = '{cosmosEntityNameValue}'"; + + return GetQueryWithExistingWhereClauseInjectedWithSharedCollection(sql, identifier, cosmosEntityNameValue); + } - return $"{sql} where {identifier}.{nameof(ISharedCosmosEntity.CosmosEntityName)} = '{cosmosEntityNameValue}'"; + private static string GetQueryWithExistingWhereClauseInjectedWithSharedCollection(string sql, + string identifier, string cosmosEntityNameValue) + { + var splitQuery = Regex.Split(sql, " where ", RegexOptions.IgnoreCase); + var firstPartQuery = splitQuery[0]; + var secondPartQuery = splitQuery[1]; + var sharedCollectionExpressionQuery = + $"{identifier}.{nameof(ISharedCosmosEntity.CosmosEntityName)} = '{cosmosEntityNameValue}'"; + return $"{firstPartQuery} where {sharedCollectionExpressionQuery} and {secondPartQuery}"; } private static string GetCollectionIdentifier(string sql) @@ -45,26 +50,13 @@ private static string GetCollectionIdentifier(string sql) if (matchedWithAs.Success) { - var potentialIdentifierFromAs = matchedWithAs.Groups[2].Value; - if (PostSelectCosmosSqlOperators.Contains(potentialIdentifierFromAs, StringComparer.OrdinalIgnoreCase)) - { - throw new InvalidSqlQueryException(sql); - } - - return potentialIdentifierFromAs; + return GetPorentialIdentifierWithTheAsKeyword(sql, matchedWithAs); } var matchedGroups = IdentifierMatchRegex.Match(sql); if (matchedGroups.Success && matchedGroups.Groups.Count == 3) { - var potentialIdentifier = matchedGroups.Groups[2].Value; - - if (PostSelectCosmosSqlOperators.Contains(potentialIdentifier, StringComparer.OrdinalIgnoreCase)) - { - return matchedGroups.Groups[1].Value; - } - - return potentialIdentifier; + return GetPotentialIdentifierWith3MatchedGroups(matchedGroups); } if (matchedGroups.Success && matchedGroups.Groups.Count == 2) @@ -86,5 +78,23 @@ private static string GetCollectionIdentifier(string sql) throw new InvalidSqlQueryException(sql); } + + private static string GetPotentialIdentifierWith3MatchedGroups(Match matchedGroups) + { + var potentialIdentifier = matchedGroups.Groups[2].Value; + + return PostSelectCosmosSqlOperators.Contains(potentialIdentifier, StringComparer.OrdinalIgnoreCase) ? matchedGroups.Groups[1].Value : potentialIdentifier; + } + + private static string GetPorentialIdentifierWithTheAsKeyword(string sql, Match matchedWithAs) + { + var potentialIdentifierFromAs = matchedWithAs.Groups[2].Value; + if (PostSelectCosmosSqlOperators.Contains(potentialIdentifierFromAs, StringComparer.OrdinalIgnoreCase)) + { + throw new InvalidSqlQueryException(sql); + } + + return potentialIdentifierFromAs; + } } } \ No newline at end of file diff --git a/Cosmonaut/Extensions/DocumentEntityExtensions.cs b/Cosmonaut/Extensions/DocumentEntityExtensions.cs index bec4e5d..20dbcd2 100644 --- a/Cosmonaut/Extensions/DocumentEntityExtensions.cs +++ b/Cosmonaut/Extensions/DocumentEntityExtensions.cs @@ -45,9 +45,7 @@ internal static PartitionKeyDefinition GetPartitionKeyForEntity(this Type type) var partitionKeyProperty = partitionKeyProperties.Single(); var porentialJsonPropertyAttribute = partitionKeyProperty.GetCustomAttribute(); - if (porentialJsonPropertyAttribute.HasJsonPropertyAttributeId() - || partitionKeyProperty.Name.Equals(nameof(ICosmosEntity.CosmosId)) - || partitionKeyProperty.Name.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase)) + if (IsCosmosIdThePartitionKey(porentialJsonPropertyAttribute, partitionKeyProperty)) { return DocumentHelpers.GetPartitionKeyDefinition(CosmosConstants.CosmosId); } @@ -59,13 +57,19 @@ internal static PartitionKeyDefinition GetPartitionKeyForEntity(this Type type) return DocumentHelpers.GetPartitionKeyDefinition(partitionKeyProperty.Name); } + private static bool IsCosmosIdThePartitionKey(JsonPropertyAttribute porentialJsonPropertyAttribute, PropertyInfo partitionKeyProperty) + { + return porentialJsonPropertyAttribute.HasJsonPropertyAttributeId() + || partitionKeyProperty.Name.Equals(nameof(ICosmosEntity.CosmosId)) + || partitionKeyProperty.Name.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase); + } + internal static PartitionKey GetPartitionKeyValueForEntity(this TEntity entity, bool isShared) where TEntity : class { var partitionKeyValue = entity.GetPartitionKeyValueAsStringForEntity(isShared); return !string.IsNullOrEmpty(partitionKeyValue) ? new PartitionKey(entity.GetPartitionKeyValueAsStringForEntity(isShared)) : null; } - internal static string GetPartitionKeyValueAsStringForEntity(this TEntity entity, bool isShared) where TEntity : class { if (isShared) @@ -73,15 +77,13 @@ internal static string GetPartitionKeyValueAsStringForEntity(this TEnti var type = entity.GetType(); var partitionKeyProperty = type.GetProperties() - .Where(x => x.GetCustomAttribute() != null).ToList(); + .Where(x => x.GetCustomAttribute() != null) + .ToList(); if (partitionKeyProperty.Count > 1) throw new MultiplePartitionKeysException(type); - if (partitionKeyProperty.Count == 0) - return null; - - return partitionKeyProperty.Single().GetValue(entity).ToString(); + return partitionKeyProperty.Count == 0 ? null : partitionKeyProperty.Single().GetValue(entity).ToString(); } internal static bool HasPartitionKey(this Type type) @@ -99,12 +101,7 @@ internal static TEntity ValidateEntityForCosmosDb(this TEntity entity) { var propertyInfos = entity.GetType().GetProperties(); - var containsJsonAttributeIdCount = - propertyInfos.Count(x => x.GetCustomAttributes() - .Any(attr => attr.PropertyName.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase))) - + entity.GetType().GetInterfaces().Count(x => x.GetProperties() - .Any(prop => prop.GetCustomAttributes() - .Any(attr => attr.PropertyName.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase)))); + var containsJsonAttributeIdCount = GetCountOfJsonPropertiesWithNameIdForObject(entity, propertyInfos); if (containsJsonAttributeIdCount > 1) throw new MultipleCosmosIdsException( @@ -115,11 +112,7 @@ internal static TEntity ValidateEntityForCosmosDb(this TEntity entity) if (idProperty != null && containsJsonAttributeIdCount == 1) { - if (!idProperty.GetCustomAttributes().Any(x => - x.PropertyName.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase))) - throw new MultipleCosmosIdsException( - "An entity can only have one cosmos db id. Either rename the Id property or remove the [JsonAttribute(\"id\")]."); - return entity; + return HandleEntityWithMultipleIds(entity, idProperty); } if (idProperty != null && idProperty.GetValue(entity) == null) @@ -130,6 +123,34 @@ internal static TEntity ValidateEntityForCosmosDb(this TEntity entity) return entity; } + private static TEntity HandleEntityWithMultipleIds(TEntity entity, PropertyInfo idProperty) + where TEntity : class + { + if (!idProperty.GetCustomAttributes().Any(x => + x.PropertyName.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase))) + throw new MultipleCosmosIdsException( + "An entity can only have one cosmos db id. Either rename the Id property or remove the [JsonAttribute(\"id\")]."); + return entity; + } + + private static int GetCountOfJsonPropertiesWithNameIdForObject(TEntity entity, PropertyInfo[] propertyInfos) where TEntity : class + { + return GetCountOfJsonPropertiesWithNameId(propertyInfos) + GetCountOfJsonPropertyWithNameIdInInterfaces(entity); + } + + private static int GetCountOfJsonPropertyWithNameIdInInterfaces(TEntity entity) where TEntity : class + { + return entity.GetType().GetInterfaces().Count(x => x.GetProperties() + .Any(prop => prop.GetCustomAttributes() + .Any(attr => attr.PropertyName.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase)))); + } + + private static int GetCountOfJsonPropertiesWithNameId(PropertyInfo[] propertyInfos) + { + return propertyInfos.Count(x => x.GetCustomAttributes() + .Any(attr => attr.PropertyName.Equals(CosmosConstants.CosmosId, StringComparison.OrdinalIgnoreCase))); + } + internal static bool HasJsonPropertyAttributeId(this JsonPropertyAttribute porentialJsonPropertyAttribute) { return porentialJsonPropertyAttribute != null && @@ -154,21 +175,16 @@ internal static string GetDocumentId(this TEntity entity) where TEntity if (propertyNamedId != null) { - if (!String.IsNullOrEmpty(propertyNamedId.GetValue(entity)?.ToString())) - { - return propertyNamedId.GetValue(entity).ToString(); - } - - propertyNamedId.SetValue(entity, Guid.NewGuid().ToString()); - return propertyNamedId.GetValue(entity).ToString(); + return HandlePropertyNamedId(entity, propertyNamedId); } - var potentialCosmosEntityId = entity.GetType().GetInterface(nameof(ICosmosEntity))? - .GetProperties()?.SingleOrDefault(x => - x.GetCustomAttribute()?.PropertyName == CosmosConstants.CosmosId); + var potentialCosmosEntityId = entity.GetType() + .GetInterface(nameof(ICosmosEntity)) + ?.GetProperties() + .SingleOrDefault(x => x.GetCustomAttribute()?.PropertyName == CosmosConstants.CosmosId); if (potentialCosmosEntityId != null && - !String.IsNullOrEmpty(potentialCosmosEntityId.GetValue(entity)?.ToString())) + !string.IsNullOrEmpty(potentialCosmosEntityId.GetValue(entity)?.ToString())) { return potentialCosmosEntityId.GetValue(entity).ToString(); } @@ -176,6 +192,17 @@ internal static string GetDocumentId(this TEntity entity) where TEntity throw new CosmosEntityWithoutIdException(entity); } + private static string HandlePropertyNamedId(TEntity entity, PropertyInfo propertyNamedId) where TEntity : class + { + if (!string.IsNullOrEmpty(propertyNamedId.GetValue(entity)?.ToString())) + { + return propertyNamedId.GetValue(entity).ToString(); + } + + propertyNamedId.SetValue(entity, Guid.NewGuid().ToString()); + return propertyNamedId.GetValue(entity).ToString(); + } + internal static void RemovePotentialDuplicateIdProperties(ref dynamic mapped) { if (mapped.Id != null) diff --git a/Cosmonaut/Extensions/ExpressionExtensions.cs b/Cosmonaut/Extensions/ExpressionExtensions.cs index 52d88d8..5dfea47 100644 --- a/Cosmonaut/Extensions/ExpressionExtensions.cs +++ b/Cosmonaut/Extensions/ExpressionExtensions.cs @@ -45,9 +45,7 @@ public ReplaceExpressionVisitor(Expression oldValue, Expression newValue) public override Expression Visit(Expression node) { - if (node == _oldValue) - return _newValue; - return base.Visit(node); + return node == _oldValue ? _newValue : base.Visit(node); } } } diff --git a/Cosmonaut/Operations/CosmosScaler.cs b/Cosmonaut/Operations/CosmosScaler.cs index c15dc90..5bb349c 100644 --- a/Cosmonaut/Operations/CosmosScaler.cs +++ b/Cosmonaut/Operations/CosmosScaler.cs @@ -3,11 +3,9 @@ using System.Linq; using System.Net; using System.Threading.Tasks; -using Cosmonaut.Exceptions; using Cosmonaut.Extensions; using Cosmonaut.Response; using Microsoft.Azure.Documents; -using Microsoft.Azure.Documents.Client; namespace Cosmonaut.Operations { @@ -30,11 +28,9 @@ internal async Task UpscaleCollectionRequestUnitsForRequest(DocumentCollection c var upscaleRequestUnits = (int)(Math.Round(documentCount * operationCost / 100d, 0) * 100); - var collectionOffer = (OfferV2)_cosmosStore.DocumentClient.CreateOfferQuery() - .Where(x => x.ResourceLink == collection.SelfLink).AsEnumerable().Single(); - _cosmosStore.CollectionThrouput = upscaleRequestUnits >= _cosmosStore.Settings.MaximumUpscaleRequestUnits ? _cosmosStore.Settings.MaximumUpscaleRequestUnits : upscaleRequestUnits; - var replaced = await _cosmosStore.DocumentClient.ReplaceOfferAsync(new OfferV2(collectionOffer, _cosmosStore.CollectionThrouput)); - _cosmosStore.IsUpscaled = replaced.StatusCode == HttpStatusCode.OK; + await ChangeCollectionThroughput(collection, upscaleRequestUnits >= _cosmosStore.Settings.MaximumUpscaleRequestUnits + ? _cosmosStore.Settings.MaximumUpscaleRequestUnits + : upscaleRequestUnits); } internal async Task DownscaleCollectionRequestUnitsToDefault(DocumentCollection collection) @@ -45,11 +41,8 @@ internal async Task DownscaleCollectionRequestUnitsToDefault(DocumentCollection if (!_cosmosStore.IsUpscaled) return; - var collectionOffer = (OfferV2)_cosmosStore.DocumentClient.CreateOfferQuery() - .Where(x => x.ResourceLink == collection.SelfLink).AsEnumerable().Single(); - _cosmosStore.CollectionThrouput = typeof(TEntity).GetCollectionThroughputForEntity(_cosmosStore.Settings.DefaultCollectionThroughput); - var replaced = await _cosmosStore.DocumentClient.ReplaceOfferAsync(new OfferV2(collectionOffer, _cosmosStore.CollectionThrouput)); - _cosmosStore.IsUpscaled = replaced.StatusCode != HttpStatusCode.OK; + var throughput = typeof(TEntity).GetCollectionThroughputForEntity(_cosmosStore.Settings.DefaultCollectionThroughput); + await ChangeCollectionThroughput(collection, throughput); } internal async Task> HandleUpscalingForRangeOperation( @@ -81,5 +74,14 @@ internal async Task> UpscaleCollectionIfConfigur multipleResponse.AddResponse(sampleResponse); return multipleResponse; } + + private async Task ChangeCollectionThroughput(DocumentCollection collection, int requestUnits) + { + var collectionOffer = (OfferV2)_cosmosStore.DocumentClient.CreateOfferQuery() + .Where(x => x.ResourceLink == collection.SelfLink).AsEnumerable().Single(); + _cosmosStore.CollectionThrouput = requestUnits; + var replaced = await _cosmosStore.DocumentClient.ReplaceOfferAsync(new OfferV2(collectionOffer, _cosmosStore.CollectionThrouput)); + _cosmosStore.IsUpscaled = replaced.StatusCode == HttpStatusCode.OK; + } } } \ No newline at end of file diff --git a/Cosmonaut/Storage/CosmosCollectionCreator.cs b/Cosmonaut/Storage/CosmosCollectionCreator.cs index ca75306..3f39af5 100644 --- a/Cosmonaut/Storage/CosmosCollectionCreator.cs +++ b/Cosmonaut/Storage/CosmosCollectionCreator.cs @@ -38,7 +38,7 @@ public async Task EnsureCreatedAsync( Id = collectionName }; - SetPartitionKeyIsCollectionIsNotShared(typeof(TEntity), isSharedCollection, collection); + SetPartitionKeyIfCollectionIsNotShared(typeof(TEntity), isSharedCollection, collection); SetPartitionKeyAsIdIfCollectionIsShared(isSharedCollection, collection); if (indexingPolicy != null) @@ -60,7 +60,7 @@ private static void SetPartitionKeyAsIdIfCollectionIsShared(bool isSharedCollect } } - private static void SetPartitionKeyIsCollectionIsNotShared(Type entityType, bool isSharedCollection, DocumentCollection collection) + private static void SetPartitionKeyIfCollectionIsNotShared(Type entityType, bool isSharedCollection, DocumentCollection collection) { if (isSharedCollection) return; var partitionKey = entityType.GetPartitionKeyForEntity();