From 75caccccd52730fd17b954aa90e58b40ea8bc991 Mon Sep 17 00:00:00 2001 From: jkeane Date: Tue, 10 Nov 2020 08:43:21 -0600 Subject: [PATCH] Decouple HttpTrigger from discovery logic and ActivityService/History (#317) * Added ActivityHistory refactored some classes into their own files removed old DAL/DALResolver * added collection/queue for activity support * Shell for classifier logic * changes to support ActivityHistory for long running activity (graph queries) * Changes to support redirect on request. * moved interface definitions to Interfaces added some file headers --- .../GraphDeltaProcessorFunctionsTests.cs | 3 +- .../ServicePrincipalGraphHelperTests.cs | 2 +- .../ModelValidationTests.cs | 6 +- .../CSE.Automation/Base/KeyVaultBase.cs | 97 +-- .../CSE.Automation/CSE.Automation.csproj | 4 - .../CSE.Automation/Config/CosmosConfig.cs | 5 +- src/Automation/CSE.Automation/Constants.cs | 7 +- .../DataAccess/ActivityHistoryRepository.cs | 44 ++ .../ActivityHistoryRepositorySettings.cs | 31 + .../DataAccess/AuditRepository.cs | 23 +- .../DataAccess/AuditRepositorySettings.cs | 31 + .../DataAccess/CosmosDBRepository.cs | 747 +++++++++--------- .../DataAccess/CosmosDBSettings.cs | 58 ++ .../CSE.Automation/DataAccess/DAL.cs | 293 ------- .../CSE.Automation/DataAccess/DALResolver.cs | 66 -- .../CSE.Automation/GlobalSuppressions.cs | 2 +- .../CSE.Automation/Graph/GraphHelperBase.cs | 15 +- .../Graph/GraphOperationMetrics.cs | 24 + .../Graph/ServicePrincipalGraphHelper.cs | 37 +- .../CSE.Automation/Graph/UserGraphHelper.cs | 2 +- .../CSE.Automation/GraphDeltaProcessor.cs | 252 ++++-- .../Interfaces/IActivityHistoryRepository.cs | 12 + .../Interfaces/IActivityService.cs | 18 + .../Interfaces/IAuditRepository.cs | 6 + .../CSE.Automation/Interfaces/IClassifier.cs | 15 + .../Interfaces/IConfigService.cs | 3 +- .../Interfaces/ICosmosDBRepository.cs | 4 +- .../CSE.Automation/Interfaces/IDAL.cs | 18 - .../Interfaces/IDeltaProcessor.cs | 6 +- .../CSE.Automation/Interfaces/IGraphModel.cs | 2 - .../CSE.Automation/Model/ActivityContext.cs | 70 +- .../CSE.Automation/Model/ActivityHistory.cs | 70 ++ .../Model/EvaluateServicePrincipalCommand.cs | 12 + .../CSE.Automation/Model/GraphModel.cs | 9 +- .../Model/ObjectClassification.cs | 13 + .../Model/RequestDiscoveryCommand.cs | 29 + .../Model/ServicePrincipalClassification.cs | 14 + .../Processors/DeltaProcessorBase.cs | 89 +-- .../Processors/DeltaProcessorSettings.cs | 48 ++ .../Processors/ServicePrincipalProcessor.cs | 186 +++-- .../Services/ActivityService.cs | 85 ++ .../CSE.Automation/Services/AuditService.cs | 9 +- .../Services/ObjectTrackingService.cs | 44 +- .../Services/ServicePrincipalClassifier.cs | 17 + src/Automation/CSE.Automation/Startup.cs | 43 +- .../Validators/GraphModelValidator.cs | 2 - .../ServicePrincipalModelValidator.cs | 14 +- .../appsettings.Development.json | 7 +- src/Automation/CSE.Automation/host.json | 2 +- .../SolutionScripts/SolutionCommands.ps1 | 3 +- 50 files changed, 1456 insertions(+), 1143 deletions(-) create mode 100644 src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepository.cs create mode 100644 src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepositorySettings.cs create mode 100644 src/Automation/CSE.Automation/DataAccess/AuditRepositorySettings.cs create mode 100644 src/Automation/CSE.Automation/DataAccess/CosmosDBSettings.cs delete mode 100644 src/Automation/CSE.Automation/DataAccess/DAL.cs delete mode 100644 src/Automation/CSE.Automation/DataAccess/DALResolver.cs create mode 100644 src/Automation/CSE.Automation/Graph/GraphOperationMetrics.cs create mode 100644 src/Automation/CSE.Automation/Interfaces/IActivityHistoryRepository.cs create mode 100644 src/Automation/CSE.Automation/Interfaces/IActivityService.cs create mode 100644 src/Automation/CSE.Automation/Interfaces/IAuditRepository.cs create mode 100644 src/Automation/CSE.Automation/Interfaces/IClassifier.cs delete mode 100644 src/Automation/CSE.Automation/Interfaces/IDAL.cs create mode 100644 src/Automation/CSE.Automation/Model/ActivityHistory.cs create mode 100644 src/Automation/CSE.Automation/Model/EvaluateServicePrincipalCommand.cs create mode 100644 src/Automation/CSE.Automation/Model/ObjectClassification.cs create mode 100644 src/Automation/CSE.Automation/Model/RequestDiscoveryCommand.cs create mode 100644 src/Automation/CSE.Automation/Model/ServicePrincipalClassification.cs create mode 100644 src/Automation/CSE.Automation/Processors/DeltaProcessorSettings.cs create mode 100644 src/Automation/CSE.Automation/Services/ActivityService.cs create mode 100644 src/Automation/CSE.Automation/Services/ServicePrincipalClassifier.cs diff --git a/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/GraphDeltaProcessorFunctionsTests.cs b/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/GraphDeltaProcessorFunctionsTests.cs index 045a36d2..833028cb 100644 --- a/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/GraphDeltaProcessorFunctionsTests.cs +++ b/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/GraphDeltaProcessorFunctionsTests.cs @@ -16,6 +16,7 @@ public class GraphDeltaProcessorFunctionsTests private readonly ISecretClient _secretClient; private readonly IGraphHelper _graphHelper; private readonly IServicePrincipalProcessor _processor; + private readonly IActivityService _activityService; IServiceProvider _serviceProvider; ILogger _logger; @@ -31,7 +32,7 @@ public GraphDeltaProcessorFunctionsTests() _serviceProvider = Substitute.For(); _logger = Substitute.For>(); - _subject = new GraphDeltaProcessor(_serviceProvider, _processor, _logger); + _subject = new GraphDeltaProcessor(_serviceProvider, _activityService, _processor, _logger); } [Fact] diff --git a/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/ServicePrincipalGraphHelperTests.cs b/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/ServicePrincipalGraphHelperTests.cs index 9df0ed68..a92bbf08 100644 --- a/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/ServicePrincipalGraphHelperTests.cs +++ b/src/Automation/CSE.Automation.Tests/FunctionsUnitTests/ServicePrincipalGraphHelperTests.cs @@ -104,7 +104,7 @@ public async Task GetDeltaGraphObjects_GetAll() var config = GetConfiguration(); var service = serviceScope.ServiceProvider.GetService>(); - var results = await service.GetDeltaGraphObjects(new ActivityContext(), config); + var results = await service.GetDeltaGraphObjects(new ActivityContext(null), config); } Assert.True(true); diff --git a/src/Automation/CSE.Automation.Tests/ModelValidationUnitTests/ModelValidationTests.cs b/src/Automation/CSE.Automation.Tests/ModelValidationUnitTests/ModelValidationTests.cs index 2cf8d5a5..f5e4d3e8 100644 --- a/src/Automation/CSE.Automation.Tests/ModelValidationUnitTests/ModelValidationTests.cs +++ b/src/Automation/CSE.Automation.Tests/ModelValidationUnitTests/ModelValidationTests.cs @@ -11,7 +11,6 @@ using CSE.Automation.Graph; using Microsoft.Graph; using System.Threading.Tasks; -using Status = CSE.Automation.Model.Status; namespace CSE.Automation.Tests.FunctionsUnitTests { @@ -25,7 +24,7 @@ class MockUserGraphHelper : IGraphHelper throw new NotImplementedException(); } - public Task GetGraphObject(string id) + public Task GetGraphObjectWithOwners(string id) { return Task.FromResult(new User()); } @@ -68,7 +67,6 @@ public void ServicePrinciapalModelValidate_ReturnsTrueIfValid() Deleted = new DateTime(2001, 1, 1), LastUpdated = new DateTime(2002, 1, 1), ObjectType = ObjectType.ServicePrincipal, - Status = Status.Remediated }; var results = servicePrincipalValidator.Validate(servicePrincipal); @@ -91,7 +89,7 @@ public void AuditEntryModelValidate_ReturnsValidationFailuresIfInvalid() [Fact] public void AuditEntryModelValidate_ReturnsTrueIfValid() { - var context = new ActivityContext(); + var context = new ActivityContext(null); var auditItem = new AuditEntry() { diff --git a/src/Automation/CSE.Automation/Base/KeyVaultBase.cs b/src/Automation/CSE.Automation/Base/KeyVaultBase.cs index e7e0c4b0..856c58bc 100644 --- a/src/Automation/CSE.Automation/Base/KeyVaultBase.cs +++ b/src/Automation/CSE.Automation/Base/KeyVaultBase.cs @@ -1,31 +1,34 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace CSE.Automation.Base -{ -#pragma warning disable CA1052 // Static holder types should be Static or NotInheritable - public class KeyVaultBase -#pragma warning restore CA1052 // Static holder types should be Static or NotInheritable - { -#pragma warning disable CA1034 // Nested types should not be visible - public static class KeyVaultHelper -#pragma warning restore CA1034 // Nested types should not be visible +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace CSE.Automation.Base +{ +#pragma warning disable CA1052 // Static holder types should be Static or NotInheritable + public class KeyVaultBase +#pragma warning restore CA1052 // Static holder types should be Static or NotInheritable + { +#pragma warning disable CA1034 // Nested types should not be visible + public static class KeyVaultHelper +#pragma warning restore CA1034 // Nested types should not be visible { - /// - /// Build the Key Vault URL from the name - /// - /// Key Vault Name - /// URL to Key Vault - public static bool BuildKeyVaultConnectionString(string keyVaultName, out string keyvaultConnection) - { - // name is required - if (string.IsNullOrWhiteSpace(keyVaultName)) - { - throw new ArgumentNullException(nameof(keyVaultName)); - } - + /// + /// Build the Key Vault URL from the name + /// + /// Key Vault Name + /// URL to Key Vault + public static bool BuildKeyVaultConnectionString(string keyVaultName, out string keyvaultConnection) + { + // name is required + if (string.IsNullOrWhiteSpace(keyVaultName)) + { + throw new ArgumentNullException(nameof(keyVaultName)); + } + var uriBuilder = new UriBuilder { Scheme = Uri.UriSchemeHttps, @@ -34,24 +37,24 @@ public static bool BuildKeyVaultConnectionString(string keyVaultName, out string keyvaultConnection = uriBuilder.Uri.AbsoluteUri; - return true; - } - - /// - /// Validate the keyvault name - /// - /// string - /// bool - public static bool ValidateName(string name) - { - if (string.IsNullOrWhiteSpace(name)) - { - return false; - } - name = name.Trim(); - - return name.Length >= 3 && name.Length <= 24; - } - } - } -} + return true; + } + + /// + /// Validate the keyvault name + /// + /// string + /// bool + public static bool ValidateName(string name) + { + if (string.IsNullOrWhiteSpace(name)) + { + return false; + } + name = name.Trim(); + + return name.Length >= 3 && name.Length <= 24; + } + } + } +} diff --git a/src/Automation/CSE.Automation/CSE.Automation.csproj b/src/Automation/CSE.Automation/CSE.Automation.csproj index 1589a26a..2864c511 100644 --- a/src/Automation/CSE.Automation/CSE.Automation.csproj +++ b/src/Automation/CSE.Automation/CSE.Automation.csproj @@ -22,8 +22,6 @@ - - @@ -40,8 +38,6 @@ - - diff --git a/src/Automation/CSE.Automation/Config/CosmosConfig.cs b/src/Automation/CSE.Automation/Config/CosmosConfig.cs index bfa44505..1310bf53 100644 --- a/src/Automation/CSE.Automation/Config/CosmosConfig.cs +++ b/src/Automation/CSE.Automation/Config/CosmosConfig.cs @@ -1,4 +1,7 @@ -using System; +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; using System.Collections.Generic; using System.Text; using Microsoft.Azure.Cosmos; diff --git a/src/Automation/CSE.Automation/Constants.cs b/src/Automation/CSE.Automation/Constants.cs index e2954cbc..08d22d79 100644 --- a/src/Automation/CSE.Automation/Constants.cs +++ b/src/Automation/CSE.Automation/Constants.cs @@ -1,4 +1,7 @@ -namespace CSE.Automation +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +namespace CSE.Automation { public sealed class Constants { @@ -21,9 +24,11 @@ public sealed class Constants public const string CosmosDBConfigCollectionName = "SPConfigurationCollection"; public const string CosmosDBAuditCollectionName = "SPAuditCollection"; public const string CosmosDBObjectTrackingCollectionName = "SPObjectTrackingCollection"; + public const string CosmosDBActivityHistoryCollectionName = "SPActivityHistoryCollection"; // Azure Storage Queue Constants public const string SPStorageConnectionString = "SPStorageConnectionString"; + public const string DiscoverQueueAppSetting = "%SPDiscoverQueue%"; public const string EvaluateQueueAppSetting = "%SPEvaluateQueue%"; public const string UpdateQueueAppSetting = "%SPUpdateQueue%"; diff --git a/src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepository.cs b/src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepository.cs new file mode 100644 index 00000000..ccb46990 --- /dev/null +++ b/src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepository.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Text; +using System.Threading.Tasks; +using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Table; +using Microsoft.Extensions.Logging; +using Microsoft.Graph; + +namespace CSE.Automation.DataAccess +{ + internal class ActivityHistoryRepository : CosmosDBRepository, IActivityHistoryRepository + { + private readonly ActivityHistoryRepositorySettings settings; + public ActivityHistoryRepository(ActivityHistoryRepositorySettings settings, ILogger logger) + : base(settings, logger) + { + this.settings = settings; + } + + public override string GenerateId(ActivityHistory entity) + { + if (string.IsNullOrWhiteSpace(entity.Id)) + { + entity.Id = Guid.NewGuid().ToString(); + } + + return entity.Id; + } + + public async Task> GetCorrelated(string correlationId) + { + return await InternalCosmosDBSqlQuery($"select * from c where c.correlationId = '{correlationId}' order by c.created").ConfigureAwait(false); + } + + public override string CollectionName => settings.CollectionName; + } +} diff --git a/src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepositorySettings.cs b/src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepositorySettings.cs new file mode 100644 index 00000000..718bfa85 --- /dev/null +++ b/src/Automation/CSE.Automation/DataAccess/ActivityHistoryRepositorySettings.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Text; +using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Table; +using Microsoft.Extensions.Logging; +using Microsoft.Graph; + +namespace CSE.Automation.DataAccess +{ + internal class ActivityHistoryRepositorySettings : CosmosDBSettings + { + public ActivityHistoryRepositorySettings(ISecretClient secretClient) + : base(secretClient) + { + } + + public string CollectionName { get; set; } + public override void Validate() + { + base.Validate(); + if (string.IsNullOrEmpty(this.CollectionName)) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: CollectionName is invalid"); + } + } + } +} diff --git a/src/Automation/CSE.Automation/DataAccess/AuditRepository.cs b/src/Automation/CSE.Automation/DataAccess/AuditRepository.cs index 1a19df1b..9226d129 100644 --- a/src/Automation/CSE.Automation/DataAccess/AuditRepository.cs +++ b/src/Automation/CSE.Automation/DataAccess/AuditRepository.cs @@ -11,25 +11,13 @@ namespace CSE.Automation.DataAccess { - internal class AuditRespositorySettings : CosmosDBSettings - { - public AuditRespositorySettings(ISecretClient secretClient) : base(secretClient) { } - - public string CollectionName { get; set; } - public override void Validate() - { - base.Validate(); - if (string.IsNullOrEmpty(this.CollectionName)) throw new ConfigurationErrorsException($"{this.GetType().Name}: CollectionName is invalid"); - } - } - - internal interface IAuditRepository : ICosmosDBRepository { } internal class AuditRepository : CosmosDBRepository, IAuditRepository { - private readonly AuditRespositorySettings _settings; - public AuditRepository(AuditRespositorySettings settings, ILogger logger) : base(settings, logger) + private readonly AuditRepositorySettings settings; + public AuditRepository(AuditRepositorySettings settings, ILogger logger) + : base(settings, logger) { - _settings = settings; + this.settings = settings; } public override string GenerateId(AuditEntry entity) @@ -38,9 +26,10 @@ public override string GenerateId(AuditEntry entity) { entity.Id = Guid.NewGuid().ToString(); } + return entity.Id; } - public override string CollectionName => _settings.CollectionName; + public override string CollectionName => settings.CollectionName; } } diff --git a/src/Automation/CSE.Automation/DataAccess/AuditRepositorySettings.cs b/src/Automation/CSE.Automation/DataAccess/AuditRepositorySettings.cs new file mode 100644 index 00000000..3153e549 --- /dev/null +++ b/src/Automation/CSE.Automation/DataAccess/AuditRepositorySettings.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Text; +using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Table; +using Microsoft.Extensions.Logging; +using Microsoft.Graph; + +namespace CSE.Automation.DataAccess +{ + internal class AuditRepositorySettings : CosmosDBSettings + { + public AuditRepositorySettings(ISecretClient secretClient) + : base(secretClient) + { + } + + public string CollectionName { get; set; } + public override void Validate() + { + base.Validate(); + if (string.IsNullOrEmpty(this.CollectionName)) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: CollectionName is invalid"); + } + } + } +} diff --git a/src/Automation/CSE.Automation/DataAccess/CosmosDBRepository.cs b/src/Automation/CSE.Automation/DataAccess/CosmosDBRepository.cs index b1abdc66..f06b30bc 100644 --- a/src/Automation/CSE.Automation/DataAccess/CosmosDBRepository.cs +++ b/src/Automation/CSE.Automation/DataAccess/CosmosDBRepository.cs @@ -1,395 +1,352 @@ -using CSE.Automation.Config; -using CSE.Automation.Interfaces; -using CSE.Automation.Model; -using Microsoft.Azure.Cosmos; -using Microsoft.Azure.Cosmos.Fluent; -using Microsoft.Extensions.Logging; -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Globalization; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using SettingsBase = CSE.Automation.Model.SettingsBase; - -namespace CSE.Automation.DataAccess -{ - - class CosmosDBSettings : SettingsBase, ICosmosDBSettings - { - private string _uri; - private string _key; - private string _databaseName; - - public CosmosDBSettings(ISecretClient secretClient) : base(secretClient) { } - - [Secret(Constants.CosmosDBURLName)] - public string Uri - { - get { return _uri ?? base.GetSecret(); } - set { _uri = value; } - } - - [Secret(Constants.CosmosDBKeyName)] - public string Key - { - get { return _key ?? base.GetSecret(); } - set { _key = value; } - } - - [Secret(Constants.CosmosDBDatabaseName)] - public string DatabaseName - { - get { return _databaseName ?? base.GetSecret(); } - set { _databaseName = value; } - } - - - public override void Validate() - { - if (string.IsNullOrWhiteSpace(this.Uri)) throw new ConfigurationErrorsException($"{this.GetType().Name}: Uri is invalid"); - if (string.IsNullOrWhiteSpace(this.Key)) throw new ConfigurationErrorsException($"{this.GetType().Name}: Key is invalid"); - if (string.IsNullOrWhiteSpace(this.DatabaseName)) throw new ConfigurationErrorsException($"{this.GetType().Name}: DatabaseName is invalid"); - } - } - - internal abstract class CosmosDBRepository : ICosmosDBRepository, IDisposable - where TEntity : class - { - const string pagedOffsetString = " offset {0} limit {1}"; - - private readonly CosmosConfig _options; - private static CosmosClient _client; - private Container _container; - private ContainerProperties _containerProperties; - private readonly ICosmosDBSettings _settings; - private readonly ILogger _logger; - private PropertyInfo _partitionKeyPI; - - /// - /// Data Access Layer Constructor - /// - /// Instance of settings for a ComosDB - /// Instance of logger. - protected CosmosDBRepository(ICosmosDBSettings settings, ILogger logger) - { - _settings = settings; - _logger = logger; - - _options = new CosmosConfig - { - MaxRows = MaxPageSize, - Timeout = CosmosTimeout, - }; - } - - public int DefaultPageSize { get; set; } = 100; - public int MaxPageSize { get; set; } = 1000; - public int CosmosTimeout { get; set; } = 60; - public int CosmosMaxRetries { get; set; } = 10; - public abstract string CollectionName { get; } - public string DatabaseName => _settings.DatabaseName; - private object lockObj = new object(); - - // NOTE: CosmosDB library currently wraps the Newtonsoft JSON serializer. Align Attributes and Converters to Newtonsoft on the domain models. - private CosmosClient Client => _client ??= new CosmosClientBuilder(_settings.Uri, _settings.Key) - .WithRequestTimeout(TimeSpan.FromSeconds(CosmosTimeout)) - .WithThrottlingRetryOptions(TimeSpan.FromSeconds(CosmosTimeout), CosmosMaxRetries) - .WithSerializerOptions(new CosmosSerializationOptions - { - PropertyNamingPolicy = CosmosPropertyNamingPolicy.CamelCase, - Indented = false, - IgnoreNullValues = true, - }) - .Build(); - private Container Container { get { lock (lockObj) { return _container ??= GetContainer(Client); } } } - - - public abstract string GenerateId(TEntity entity); - - - /// - /// Recreate the Cosmos Client / Container (after a key rotation) - /// - /// force reconnection even if no params changed - /// Task - public async Task Reconnect(bool force = false) - { - if (force || _container.Id != this.CollectionName) - { - - // open and test a new client / container - _client = null; - if (await Test().ConfigureAwait(true) == false) - { - _logger.LogError($"Failed to reconnect to CosmosDB {_settings.DatabaseName}:{this.CollectionName}"); - } - - } - } - - public virtual PartitionKey ResolvePartitionKey(TEntity entity) - { - try - { - var value = new PartitionKey(_partitionKeyPI.GetValue(entity).ToString()); - return value; - } - catch (Exception ex) - { - ex.Data["partitionKeyPath"] = _containerProperties.PartitionKeyPath; - ex.Data["entityType"] = typeof(TEntity); - throw; - } - } - - - public async Task Test() - { - if (string.IsNullOrEmpty(this.CollectionName)) - { - throw new ArgumentException($"CosmosCollection cannot be null"); - } - - // open and test a new client / container - try - { - var containers = await GetContainerNames().ConfigureAwait(false); - var containerNames = string.Join(',', containers); - _logger.LogDebug($"Test {this.Id} -- '{containerNames}'"); - if (containers.Any(x => x == this.CollectionName) == false) - { - throw new ApplicationException(); // use same error path - } - return true; - } -#pragma warning disable CA1031 // Do not catch general exception types - catch (Exception ex) -#pragma warning restore CA1031 // Do not catch general exception types - { - _logger.LogError(ex, $"Failed to find collection in CosmosDB {_settings.DatabaseName}:{this.CollectionName}"); - return false; - } - - } - - public string Id => $"{DatabaseName}:{CollectionName}"; - - /// - /// Query the database for all the containers defined and return a list of the container names. - /// - /// - private async Task> GetContainerNames() - { - var containerNames = new List(); - var database = this.Client.GetDatabase(_settings.DatabaseName); - using var iter = database.GetContainerQueryIterator(); - while (iter.HasMoreResults) - { - var response = await iter.ReadNextAsync().ConfigureAwait(false); - - containerNames.AddRange(response.Select(c => c.Id)); - } - - return containerNames; - } - - /// - /// Get a proxy to the container. - /// - /// An instance of - /// An instance of . - Container GetContainer(CosmosClient client) - { - try - { - var container = client.GetContainer(_settings.DatabaseName, this.CollectionName); - - _containerProperties = GetContainerProperties(container).Result; - var partitionKeyName = _containerProperties.PartitionKeyPath.TrimStart('/'); - _partitionKeyPI = typeof(TEntity).GetProperty(partitionKeyName, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase); - if (_partitionKeyPI is null) - { - throw new ApplicationException($"Failed to find partition key property {partitionKeyName} on {typeof(TEntity).Name}. Collection definition does not match Entity definition"); - } - - _logger.LogDebug($"{CollectionName} partition key path {_containerProperties.PartitionKeyPath}"); - - return container; - } - catch (Exception ex) - { - _logger.LogCritical(ex, $"Failed to connect to CosmosDB {_settings.DatabaseName}:{this.CollectionName}"); - throw; - } - } - - /// - /// Get the properties for the container. - /// - /// Instance of a container or null. - /// An instance of or null. - protected async Task GetContainerProperties(Container container = null) - { - return (await (container ?? Container).ReadContainerAsync().ConfigureAwait(false)).Resource; - } - - /// - /// Generic function to be used by subclasses to execute arbitrary queries and return type T. - /// - /// POCO type to which results are serialized and returned. - /// Query to be executed. - /// Enumerable list of objects of type T. - private async Task> InternalCosmosDBSqlQuery(QueryDefinition queryDefinition) - { - // run query - var query = this.Container.GetItemQueryIterator(queryDefinition, requestOptions: _options.QueryRequestOptions); - - var results = new List(); - - while (query.HasMoreResults) - { - foreach (var doc in await query.ReadNextAsync().ConfigureAwait(false)) - { - results.Add(doc); - } - } - - return results; - } - - /// - /// Generic function to be used by subclasses to execute arbitrary queries and return type T. - /// - /// POCO type to which results are serialized and returned. - /// Query to be executed. - /// Enumerable list of objects of type T. - private async Task> InternalCosmosDBSqlQuery(string sql, QueryRequestOptions options = null) - { - // run query - var query = this.Container.GetItemQueryIterator(sql, requestOptions: options ?? _options.QueryRequestOptions); - - var results = new List(); - - while (query.HasMoreResults) - { - foreach (var doc in await query.ReadNextAsync().ConfigureAwait(false)) - { - results.Add(doc); - } - } - return results; - } - - /// - /// Given a document id and its partition value, retrieve the document, if it exists. - /// - /// Id of the document. - /// Value of the partitionkey for the document. - /// An instance of the document or null. - public async Task GetByIdAsync(string id, string partitionKey) - { - var result = await GetByIdWithMetaAsync(id, partitionKey).ConfigureAwait(false); - return result?.Resource; - } - - public async Task> GetByIdWithMetaAsync(string id, string partitionKey) - { - ItemResponse entityWithMeta = null; - try - { - var result = await this.Container.ReadItemAsync(id, new PartitionKey(partitionKey)).ConfigureAwait(false); - - entityWithMeta = result; - } -#pragma warning disable CA1031 // Do not catch general exception types - catch (Exception) -#pragma warning restore CA1031 // Do not catch general exception types - { - // swallow exception - } - return entityWithMeta; - } - - public async Task ReplaceDocumentAsync(string id, TEntity newDocument, ItemRequestOptions reqOptions) - { - return await this.Container.ReplaceItemAsync(newDocument, id, ResolvePartitionKey(newDocument), reqOptions).ConfigureAwait(false); - } - - public async Task CreateDocumentAsync(TEntity newDocument) - { - return await this.Container.CreateItemAsync(newDocument, ResolvePartitionKey(newDocument)).ConfigureAwait(false); - } - - public async Task UpsertDocumentAsync(TEntity newDocument) - { - return await this.Container.UpsertItemAsync(newDocument, ResolvePartitionKey(newDocument)).ConfigureAwait(false); - } - - public async Task DeleteDocumentAsync(string id, string partitionKey) - { - return await this.Container.DeleteItemAsync(id, new PartitionKey(partitionKey)).ConfigureAwait(false); - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Using lower case with cosmos queries as tested.")] - public async Task> GetPagedAsync(string q, int offset = 0, int limit = Constants.DefaultPageSize) - { - string sql = q; - - - if (limit < 1) - { - limit = Constants.DefaultPageSize; - } - else if (limit > Constants.MaxPageSize) - { - limit = Constants.MaxPageSize; - } - - string offsetLimit = string.Format(CultureInfo.InvariantCulture, pagedOffsetString, offset, limit); - - if (!string.IsNullOrEmpty(q)) - { - // convert to lower and escape embedded ' - q = q.Trim().ToLowerInvariant().Replace("'", "''", System.StringComparison.OrdinalIgnoreCase); - - if (!string.IsNullOrEmpty(q)) - { - // get actors by a "like" search on name - sql += string.Format(CultureInfo.InvariantCulture, $" and contains(m.textSearch, @q) "); - } - } - - sql += offsetLimit; - - QueryDefinition queryDefinition = new QueryDefinition(sql); - - if (!string.IsNullOrEmpty(q)) - { - queryDefinition.WithParameter("@q", q); - } - - return await InternalCosmosDBSqlQuery(queryDefinition).ConfigureAwait(false); - - } - - public async Task> GetAllAsync(TypeFilter filter = TypeFilter.any) - { - string sql = "select * from m"; - if (filter != TypeFilter.any) - { - sql += ($" m.objectType='{0}'", Enum.GetName(typeof(TypeFilter), filter)); - } - - return await InternalCosmosDBSqlQuery(sql).ConfigureAwait(false); - } - - #region IDisposable - public void Dispose() - { - //_client?.Dispose(); - } - #endregion - } -} +using CSE.Automation.Config; +using CSE.Automation.Interfaces; +using Microsoft.Azure.Cosmos; +using Microsoft.Azure.Cosmos.Fluent; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; + +namespace CSE.Automation.DataAccess +{ + + internal abstract class CosmosDBRepository : ICosmosDBRepository, IDisposable + where TEntity : class + { + const string pagedOffsetString = " offset {0} limit {1}"; + + private readonly CosmosConfig _options; + private static CosmosClient _client; + private Container _container; + private ContainerProperties _containerProperties; + private readonly ICosmosDBSettings _settings; + private readonly ILogger _logger; + private PropertyInfo _partitionKeyPI; + + /// + /// Data Access Layer Constructor + /// + /// Instance of settings for a ComosDB + /// Instance of logger. + protected CosmosDBRepository(ICosmosDBSettings settings, ILogger logger) + { + _settings = settings; + _logger = logger; + + _options = new CosmosConfig + { + MaxRows = MaxPageSize, + Timeout = CosmosTimeout, + }; + } + + public int DefaultPageSize { get; set; } = 100; + public int MaxPageSize { get; set; } = 1000; + public int CosmosTimeout { get; set; } = 60; + public int CosmosMaxRetries { get; set; } = 10; + public abstract string CollectionName { get; } + public string DatabaseName => _settings.DatabaseName; + private object lockObj = new object(); + + // NOTE: CosmosDB library currently wraps the Newtonsoft JSON serializer. Align Attributes and Converters to Newtonsoft on the domain models. + private CosmosClient Client => _client ??= new CosmosClientBuilder(_settings.Uri, _settings.Key) + .WithRequestTimeout(TimeSpan.FromSeconds(CosmosTimeout)) + .WithThrottlingRetryOptions(TimeSpan.FromSeconds(CosmosTimeout), CosmosMaxRetries) + .WithSerializerOptions(new CosmosSerializationOptions + { + PropertyNamingPolicy = CosmosPropertyNamingPolicy.CamelCase, + Indented = false, + IgnoreNullValues = true, + }) + .Build(); + private Container Container { get { lock (lockObj) { return _container ??= GetContainer(Client); } } } + + public abstract string GenerateId(TEntity entity); + + /// + /// Recreate the Cosmos Client / Container (after a key rotation) + /// + /// force reconnection even if no params changed + /// Task + public async Task Reconnect(bool force = false) + { + if (force || _container.Id != this.CollectionName) + { + + // open and test a new client / container + _client = null; + if (await Test().ConfigureAwait(true) == false) + { + _logger.LogError($"Failed to reconnect to CosmosDB {_settings.DatabaseName}:{this.CollectionName}"); + } + } + } + + public virtual PartitionKey ResolvePartitionKey(TEntity entity) + { + try + { + var value = new PartitionKey(_partitionKeyPI.GetValue(entity).ToString()); + return value; + } + catch (Exception ex) + { + ex.Data["partitionKeyPath"] = _containerProperties.PartitionKeyPath; + ex.Data["entityType"] = typeof(TEntity); + throw; + } + } + + public async Task Test() + { + if (string.IsNullOrEmpty(this.CollectionName)) + { + throw new ArgumentException($"CosmosCollection cannot be null"); + } + + // open and test a new client / container + try + { + var containers = await GetContainerNames().ConfigureAwait(false); + var containerNames = string.Join(',', containers); + _logger.LogDebug($"Test {this.Id} -- '{containerNames}'"); + if (containers.Any(x => x == this.CollectionName) == false) + { + throw new ApplicationException(); // use same error path + } + return true; + } +#pragma warning disable CA1031 // Do not catch general exception types + catch (Exception ex) +#pragma warning restore CA1031 // Do not catch general exception types + { + _logger.LogError(ex, $"Failed to find collection in CosmosDB {_settings.DatabaseName}:{this.CollectionName}"); + return false; + } + + } + + public string Id => $"{DatabaseName}:{CollectionName}"; + + /// + /// Query the database for all the containers defined and return a list of the container names. + /// + /// A list of container names present in the configured database. + private async Task> GetContainerNames() + { + var containerNames = new List(); + var database = this.Client.GetDatabase(_settings.DatabaseName); + using var iter = database.GetContainerQueryIterator(); + while (iter.HasMoreResults) + { + var response = await iter.ReadNextAsync().ConfigureAwait(false); + + containerNames.AddRange(response.Select(c => c.Id)); + } + + return containerNames; + } + + /// + /// Get a proxy to the container. + /// + /// An instance of + /// An instance of . + Container GetContainer(CosmosClient client) + { + try + { + var container = client.GetContainer(_settings.DatabaseName, this.CollectionName); + + _containerProperties = GetContainerProperties(container).Result; + var partitionKeyName = _containerProperties.PartitionKeyPath.TrimStart('/'); + _partitionKeyPI = typeof(TEntity).GetProperty(partitionKeyName, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase); + if (_partitionKeyPI is null) + { + throw new ApplicationException($"Failed to find partition key property {partitionKeyName} on {typeof(TEntity).Name}. Collection definition does not match Entity definition"); + } + + _logger.LogDebug($"{CollectionName} partition key path {_containerProperties.PartitionKeyPath}"); + + return container; + } + catch (Exception ex) + { + var message = $"Failed to connect to CosmosDB {_settings.DatabaseName}:{this.CollectionName}"; + _logger.LogCritical(ex, message); + throw new ApplicationException(message, ex); + } + } + + /// + /// Get the properties for the container. + /// + /// Instance of a container or null. + /// An instance of or null. + protected async Task GetContainerProperties(Container container = null) + { + return (await (container ?? Container).ReadContainerAsync().ConfigureAwait(false)).Resource; + } + + /// + /// Generic function to be used by subclasses to execute arbitrary queries and return type T. + /// + /// POCO type to which results are serialized and returned. + /// Query to be executed. + /// Query options + /// Enumerable list of objects of type T. + protected async Task> InternalCosmosDBSqlQuery(string sql, QueryRequestOptions options = null) + { + // run query + var query = this.Container.GetItemQueryIterator(sql, requestOptions: options ?? _options.QueryRequestOptions); + + var results = new List(); + + while (query.HasMoreResults) + { + foreach (var doc in await query.ReadNextAsync().ConfigureAwait(false)) + { + results.Add(doc); + } + } + + return results; + } + + /// + /// Generic function to be used by subclasses to execute arbitrary queries and return type T. + /// + /// POCO type to which results are serialized and returned. + /// Query to be executed. + /// Enumerable list of objects of type T. + private async Task> InternalCosmosDBSqlQuery(QueryDefinition queryDefinition) + { + // run query + var query = this.Container.GetItemQueryIterator(queryDefinition, requestOptions: _options.QueryRequestOptions); + + var results = new List(); + + while (query.HasMoreResults) + { + foreach (var doc in await query.ReadNextAsync().ConfigureAwait(false)) + { + results.Add(doc); + } + } + + return results; + } + + /// + /// Given a document id and its partition value, retrieve the document, if it exists. + /// + /// Id of the document. + /// Value of the partitionkey for the document. + /// An instance of the document or null. + public async Task GetByIdAsync(string id, string partitionKey) + { + var result = await GetByIdWithMetaAsync(id, partitionKey).ConfigureAwait(false); + return result?.Resource; + } + + public async Task> GetByIdWithMetaAsync(string id, string partitionKey) + { + ItemResponse entityWithMeta = null; + try + { + var result = await this.Container.ReadItemAsync(id, new PartitionKey(partitionKey)).ConfigureAwait(false); + + entityWithMeta = result; + } +#pragma warning disable CA1031 // Do not catch general exception types + catch (Exception) +#pragma warning restore CA1031 // Do not catch general exception types + { + // swallow exception + } + + return entityWithMeta; + } + + public async Task ReplaceDocumentAsync(string id, TEntity newDocument, ItemRequestOptions reqOptions) + { + return await this.Container.ReplaceItemAsync(newDocument, id, ResolvePartitionKey(newDocument), reqOptions).ConfigureAwait(false); + } + + public async Task CreateDocumentAsync(TEntity newDocument) + { + return await this.Container.CreateItemAsync(newDocument, ResolvePartitionKey(newDocument)).ConfigureAwait(false); + } + + public async Task UpsertDocumentAsync(TEntity newDocument) + { + return await this.Container.UpsertItemAsync(newDocument, ResolvePartitionKey(newDocument)).ConfigureAwait(false); + } + + public async Task DeleteDocumentAsync(string id, string partitionKey) + { + return await this.Container.DeleteItemAsync(id, new PartitionKey(partitionKey)).ConfigureAwait(false); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Using lower case with cosmos queries as tested.")] + public async Task> GetPagedAsync(string q, int offset = 0, int limit = Constants.DefaultPageSize) + { + string sql = q; + + if (limit < 1) + { + limit = Constants.DefaultPageSize; + } + else if (limit > Constants.MaxPageSize) + { + limit = Constants.MaxPageSize; + } + + string offsetLimit = string.Format(CultureInfo.InvariantCulture, pagedOffsetString, offset, limit); + + if (!string.IsNullOrEmpty(q)) + { + // convert to lower and escape embedded ' + q = q.Trim().ToLowerInvariant().Replace("'", "''", System.StringComparison.OrdinalIgnoreCase); + + if (!string.IsNullOrEmpty(q)) + { + // get actors by a "like" search on name + sql += string.Format(CultureInfo.InvariantCulture, $" and contains(m.textSearch, @q) "); + } + } + + sql += offsetLimit; + + QueryDefinition queryDefinition = new QueryDefinition(sql); + + if (!string.IsNullOrEmpty(q)) + { + queryDefinition.WithParameter("@q", q); + } + + return await InternalCosmosDBSqlQuery(queryDefinition).ConfigureAwait(false); + } + + public async Task> GetAllAsync(TypeFilter filter = TypeFilter.any) + { + string sql = "select * from m"; + if (filter != TypeFilter.any) + { + sql += ($" m.objectType='{0}'", Enum.GetName(typeof(TypeFilter), filter)); + } + + return await InternalCosmosDBSqlQuery(sql).ConfigureAwait(false); + } + + #region IDisposable + public void Dispose() + { + //_client?.Dispose(); + } + #endregion + } +} diff --git a/src/Automation/CSE.Automation/DataAccess/CosmosDBSettings.cs b/src/Automation/CSE.Automation/DataAccess/CosmosDBSettings.cs new file mode 100644 index 00000000..124c6234 --- /dev/null +++ b/src/Automation/CSE.Automation/DataAccess/CosmosDBSettings.cs @@ -0,0 +1,58 @@ +using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using System.Configuration; +using SettingsBase = CSE.Automation.Model.SettingsBase; + +namespace CSE.Automation.DataAccess +{ + class CosmosDBSettings : SettingsBase, ICosmosDBSettings + { + private string uri; + private string key; + private string databaseName; + + public CosmosDBSettings(ISecretClient secretClient) + : base(secretClient) + { + } + + [Secret(Constants.CosmosDBURLName)] + public string Uri + { + get { return uri ?? GetSecret(); } + set { uri = value; } + } + + [Secret(Constants.CosmosDBKeyName)] + public string Key + { + get { return key ?? GetSecret(); } + set { key = value; } + } + + [Secret(Constants.CosmosDBDatabaseName)] + public string DatabaseName + { + get { return databaseName ?? GetSecret(); } + set { databaseName = value; } + } + + public override void Validate() + { + if (string.IsNullOrWhiteSpace(this.Uri)) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: Uri is invalid"); + } + + if (string.IsNullOrWhiteSpace(this.Key)) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: Key is invalid"); + } + + if (string.IsNullOrWhiteSpace(this.DatabaseName)) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: DatabaseName is invalid"); + } + } + } +} diff --git a/src/Automation/CSE.Automation/DataAccess/DAL.cs b/src/Automation/CSE.Automation/DataAccess/DAL.cs deleted file mode 100644 index f5caa833..00000000 --- a/src/Automation/CSE.Automation/DataAccess/DAL.cs +++ /dev/null @@ -1,293 +0,0 @@ -using Microsoft.Azure.Cosmos; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Threading.Tasks; -using CSE.Automation.Config; -using CSE.Automation.Interfaces; - -namespace CSE.Automation.DataAccess -{ - public partial class DAL : IDAL - { - const string pagedOffsetString = " offset {0} limit {1}"; - - public int DefaultPageSize { get; set; } = 100; - public int MaxPageSize { get; set; } = 1000; - public int CosmosTimeout { get; set; } = 60; - public int CosmosMaxRetries { get; set; } = 10; - - private CosmosConfig cosmosDetails; - - /// - /// Data Access Layer Constructor - /// - /// CosmosDB Url - /// CosmosDB connection key - /// CosmosDB Database - /// CosmosDB Collection - public DAL(Uri cosmosUrl, string cosmosKey, string cosmosDatabase, string cosmosCollection) - { - if (cosmosUrl == null) - { - throw new ArgumentNullException(nameof(cosmosUrl)); - } - - cosmosDetails = new CosmosConfig - { - MaxRows = MaxPageSize, - Timeout = CosmosTimeout, - CosmosCollection = cosmosCollection, - CosmosDatabase = cosmosDatabase, - CosmosKey = cosmosKey, - CosmosUrl = cosmosUrl.AbsoluteUri - }; - - // create the CosmosDB client and container - cosmosDetails.Client = OpenAndTestCosmosClient(cosmosUrl, cosmosKey, cosmosDatabase, cosmosCollection); - cosmosDetails.Container = cosmosDetails.Client.GetContainer(cosmosDatabase, cosmosCollection); - } - - /// - /// Recreate the Cosmos Client / Container (after a key rotation) - /// - /// Cosmos URL - /// Cosmos Key - /// Cosmos Database - /// Cosmos Collection - /// force reconnection even if no params changed - /// Task - public async Task Reconnect(Uri cosmosUrl, string cosmosKey, string cosmosDatabase, string cosmosCollection, bool force = false) - { - await Task.Run(() => - { - if (cosmosUrl == null) - { - throw new ArgumentNullException(nameof(cosmosUrl)); - } - - if (force || - cosmosDetails.CosmosCollection != cosmosCollection || - cosmosDetails.CosmosDatabase != cosmosDatabase || - cosmosDetails.CosmosKey != cosmosKey || - cosmosDetails.CosmosUrl != cosmosUrl.AbsoluteUri) - { - CosmosConfig d = new CosmosConfig - { - CosmosCollection = cosmosCollection, - CosmosDatabase = cosmosDatabase, - CosmosKey = cosmosKey, - CosmosUrl = cosmosUrl.AbsoluteUri - }; - - // open and test a new client / container - d.Client = OpenAndTestCosmosClient(cosmosUrl, cosmosKey, cosmosDatabase, cosmosCollection); - d.Container = d.Client.GetContainer(cosmosDatabase, cosmosCollection); - - // set the current CosmosDetail - cosmosDetails = d; - } - }).ConfigureAwait(false); - } - - /// - /// Open and test the Cosmos Client / Container / Query - /// - /// Cosmos URL - /// Cosmos Key - /// Cosmos Database - /// Cosmos Collection - /// An open and validated CosmosClient - private CosmosClient OpenAndTestCosmosClient(Uri cosmosUrl, string cosmosKey, string cosmosDatabase, string cosmosCollection) - { - // validate required parameters - if (cosmosUrl == null) - { - throw new ArgumentNullException(nameof(cosmosUrl)); - } - - if (string.IsNullOrEmpty(cosmosKey)) - { - throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, $"CosmosKey not set correctly {cosmosKey}")); - } - - if (string.IsNullOrEmpty(cosmosDatabase)) - { - throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, $"CosmosDatabase not set correctly {cosmosDatabase}")); - } - - if (string.IsNullOrEmpty(cosmosCollection)) - { - throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, $"CosmosCollection not set correctly {cosmosCollection}")); - } - - // open and test a new client / container -#pragma warning disable CA2000 // Dispose objects before losing scope. Disabling as the container connection remains in scope. - var c = new CosmosClient(cosmosUrl.AbsoluteUri, cosmosKey, cosmosDetails.CosmosClientOptions); -#pragma warning restore CA2000 // Dispose objects before losing scope - var con = c.GetContainer(cosmosDatabase, cosmosCollection); - - //TODO: commenting out for the moment. Need a good way to test that doesn't require a document - //await con.ReadItemAsync("action", new PartitionKey("0")).ConfigureAwait(false); - - return c; - } - - /// - /// Compute the partition key based on input id - /// - /// For this sample, the partitionkey is the id mod 10 - /// - /// In a full implementation, you would update the logic to determine the partition key - /// - /// document id - /// the partition key - public static string GetPartitionKey(string id) - { - // validate id - if (!string.IsNullOrEmpty(id) && - id.Length > 5 && - (id.StartsWith("tt", StringComparison.OrdinalIgnoreCase) || id.StartsWith("nm", StringComparison.OrdinalIgnoreCase)) && - int.TryParse(id.Substring(2), out int idInt)) - { - return (idInt % 10).ToString(CultureInfo.InvariantCulture); - } - - throw new ArgumentException("Invalid Partition Key"); - } - - /// - /// Generic function to be used by subclasses to execute arbitrary queries and return type T. - /// - /// POCO type to which results are serialized and returned. - /// Query to be executed. - /// Enumerable list of objects of type T. - private async Task> InternalCosmosDBSqlQuery(QueryDefinition queryDefinition) - { - // run query - var query = cosmosDetails.Container.GetItemQueryIterator(queryDefinition, requestOptions: cosmosDetails.QueryRequestOptions); - - List results = new List(); - - while (query.HasMoreResults) - { - foreach (var doc in await query.ReadNextAsync().ConfigureAwait(false)) - { - results.Add(doc); - } - } - - return results; - } - - /// - /// Generic function to be used by subclasses to execute arbitrary queries and return type T. - /// - /// POCO type to which results are serialized and returned. - /// Query to be executed. - /// Enumerable list of objects of type T. - private async Task> InternalCosmosDBSqlQuery(string sql) - { - // run query - var query = cosmosDetails.Container.GetItemQueryIterator(sql, requestOptions: cosmosDetails.QueryRequestOptions); - - List results = new List(); - - while (query.HasMoreResults) - { - foreach (var doc in await query.ReadNextAsync().ConfigureAwait(false)) - { - results.Add(doc); - } - } - return results; - } - - public async Task GetByIdAsync(string id, string partitionKey) - { - - var response = await cosmosDetails.Container.ReadItemAsync(id, new PartitionKey(partitionKey)).ConfigureAwait(false); - return response; - } - - public async Task ReplaceDocumentAsync(string id, T newDocument, string partitionKey = null) - { - var con = cosmosDetails.Client.GetContainer(cosmosDetails.CosmosDatabase, cosmosDetails.CosmosCollection); - - //PartitionKey pk = String.IsNullOrWhiteSpace(partitionKey) ? default : new PartitionKey(partitionKey); - - return await con.ReplaceItemAsync(newDocument, id, null).ConfigureAwait(false); - } - - public async Task CreateDocumentAsync(T newDocument, string partitionKey = null) - { - var con = cosmosDetails.Client.GetContainer(cosmosDetails.CosmosDatabase, cosmosDetails.CosmosCollection); - - return await con.CreateItemAsync(newDocument, new PartitionKey(partitionKey)).ConfigureAwait(false); - } - - public async Task DoesExistsAsync(string id, string partitionKey) - { - using (ResponseMessage response = await cosmosDetails.Container.ReadItemStreamAsync(id, new PartitionKey(partitionKey)).ConfigureAwait(false)) - { - return response.IsSuccessStatusCode; - } - - } - - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Using lower case with cosmos queries as tested.")] - public async Task> GetPagedAsync(string q, int offset = 0, int limit = Constants.DefaultPageSize) - { - string sql = q; - - - if (limit < 1) - { - limit = Constants.DefaultPageSize; - } - else if (limit > Constants.MaxPageSize) - { - limit = Constants.MaxPageSize; - } - - string offsetLimit = string.Format(CultureInfo.InvariantCulture, pagedOffsetString, offset, limit); - - if (!string.IsNullOrEmpty(q)) - { - // convert to lower and escape embedded ' - q = q.Trim().ToLowerInvariant().Replace("'", "''", System.StringComparison.OrdinalIgnoreCase); - - if (!string.IsNullOrEmpty(q)) - { - // get actors by a "like" search on name - sql += string.Format(CultureInfo.InvariantCulture, $" and contains(m.textSearch, @q) "); - - } - } - - sql += offsetLimit; - - QueryDefinition queryDefinition = new QueryDefinition(sql); - - if (!string.IsNullOrEmpty(q)) - { - queryDefinition.WithParameter("@q", q); - } - - return await InternalCosmosDBSqlQuery(queryDefinition).ConfigureAwait(false); - - } - - public async Task> GetAllAsync(TypeFilter filter = TypeFilter.any) - { - string sql = "select * from m"; - if (filter != TypeFilter.any) - { - sql += ($" m.objectType='{0}'", Enum.GetName(typeof(TypeFilter), filter)); - } - - return await InternalCosmosDBSqlQuery(sql).ConfigureAwait(false); - } - } -} diff --git a/src/Automation/CSE.Automation/DataAccess/DALResolver.cs b/src/Automation/CSE.Automation/DataAccess/DALResolver.cs deleted file mode 100644 index 27c717c2..00000000 --- a/src/Automation/CSE.Automation/DataAccess/DALResolver.cs +++ /dev/null @@ -1,66 +0,0 @@ -using CSE.Automation.Interfaces; -using CSE.Automation.Graph; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Security; -using System.Text; -using CSE.Automation.Model; - -namespace CSE.Automation.DataAccess -{ - - class DALResolver : IServiceResolver - { - private ConcurrentDictionary _registeredDALs = new System.Collections.Concurrent.ConcurrentDictionary(); - private readonly CosmosDBSettings _settings; - - public DALResolver(CosmosDBSettings settings) - { - _settings = settings; - } - - private IDAL CreateDAL(DALCollection collectionName) - { - string collectionNameKey = default; - string cosmosCollectionName = default; - - switch (collectionName) - { - case DALCollection.Audit: - collectionNameKey = Constants.CosmosDBAuditCollectionName; - break; - case DALCollection.ProcessorConfiguration: - collectionNameKey = Constants.CosmosDBConfigCollectionName; - break; - case DALCollection.ObjectTracking: - collectionNameKey = Constants.CosmosDBOjbectTrackingCollectionName; - break; - } - - return new DAL(new Uri(_settings.Uri), _settings.Key, _settings.DatabaseName, _settings.CollectionName); - - - } - - //public IDAL GetDAL(DALCollection collection) - //{ - // string collectionName = Enum.GetName(typeof(DALCollection), collection); - // return _registeredDALs.GetOrAdd(collectionName, CreateDAL(collection)); - - //} - - public T GetService(string keyName) - { - var targetInterface = typeof(IDAL); - if (typeof(T) != targetInterface) - throw new InvalidCastException($"For DAL resolver type T must be of type {targetInterface.Name}"); - - DALCollection collectionName = Enum.Parse(keyName); - - return (T)_registeredDALs.GetOrAdd(keyName, CreateDAL(collectionName)); - } - - - } -} diff --git a/src/Automation/CSE.Automation/GlobalSuppressions.cs b/src/Automation/CSE.Automation/GlobalSuppressions.cs index 7172b0fe..65c75b65 100644 --- a/src/Automation/CSE.Automation/GlobalSuppressions.cs +++ b/src/Automation/CSE.Automation/GlobalSuppressions.cs @@ -7,5 +7,5 @@ [assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "Fields needed to be accessed by derived classes.", Scope = "member", Target = "CSE.Automation.Processors.DeltaProcessorBase._uniqueId")] [assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "Fields needed to be accessed by derived classes.", Scope = "member", Target = "~F:CSE.Automation.Processors.DeltaProcessorBase._configDAL")] -[assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "Fields needed to be accessed by derived classes.", Scope = "member", Target = "~F:CSE.Automation.Processors.DeltaProcessorBase._config")] +[assembly: SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "Fields needed to be accessed by derived classes.", Scope = "member", Target = "~F:CSE.Automation.Processors.DeltaProcessorBase.config")] [assembly: SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "This property is part of DTO.", Scope = "member", Target = "~P:CSE.Automation.Model.ProcessorConfiguration.SelectFields")] diff --git a/src/Automation/CSE.Automation/Graph/GraphHelperBase.cs b/src/Automation/CSE.Automation/Graph/GraphHelperBase.cs index 69a563c2..e41791f7 100644 --- a/src/Automation/CSE.Automation/Graph/GraphHelperBase.cs +++ b/src/Automation/CSE.Automation/Graph/GraphHelperBase.cs @@ -39,19 +39,10 @@ public override void Validate() } } - public class GraphOperationMetrics - { - public string Name { get; set; } - public int Considered { get; set; } - public int Removed { get; set; } - public int Found { get; set; } - public string AdditionalData { get; set; } - } - - public interface IGraphHelper + internal interface IGraphHelper { Task<(GraphOperationMetrics metrics, IEnumerable data)> GetDeltaGraphObjects(ActivityContext context, ProcessorConfiguration config, string selectFields = null); - Task GetGraphObject(string id); + Task GetGraphObjectWithOwners(string id); Task PatchGraphObject(TEntity entity); } @@ -84,7 +75,7 @@ protected GraphHelperBase(GraphHelperSettings settings, IAuditService auditServi } public abstract Task<(GraphOperationMetrics metrics, IEnumerable data)> GetDeltaGraphObjects(ActivityContext context, ProcessorConfiguration config, string selectFields = null); - public abstract Task GetGraphObject(string id); + public abstract Task GetGraphObjectWithOwners(string id); public abstract Task PatchGraphObject(TEntity entity); } } diff --git a/src/Automation/CSE.Automation/Graph/GraphOperationMetrics.cs b/src/Automation/CSE.Automation/Graph/GraphOperationMetrics.cs new file mode 100644 index 00000000..fecf7bdd --- /dev/null +++ b/src/Automation/CSE.Automation/Graph/GraphOperationMetrics.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; + +namespace CSE.Automation.Graph +{ + public class GraphOperationMetrics + { + public string Name { get; set; } + public int Considered { get; set; } + public int Removed { get; set; } + public int Found { get; set; } + public string AdditionalData { get; set; } + + public IDictionary ToDictionary() + { + return new Dictionary() + { + { "Considered", Considered }, + { "Removed", Removed }, + { "Found", Found }, + { "AdditionalData", AdditionalData }, + }; + } + } +} diff --git a/src/Automation/CSE.Automation/Graph/ServicePrincipalGraphHelper.cs b/src/Automation/CSE.Automation/Graph/ServicePrincipalGraphHelper.cs index 01857611..6f2045f1 100644 --- a/src/Automation/CSE.Automation/Graph/ServicePrincipalGraphHelper.cs +++ b/src/Automation/CSE.Automation/Graph/ServicePrincipalGraphHelper.cs @@ -1,11 +1,11 @@ -using CSE.Automation.Model; -using Microsoft.Graph; -using System; +using System; using System.Collections.Generic; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; using System.Linq; +using System.Threading.Tasks; using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Graph; #pragma warning disable CA1031 // Do not catch general exception types @@ -13,9 +13,10 @@ namespace CSE.Automation.Graph { internal class ServicePrincipalGraphHelper : GraphHelperBase { - - - public ServicePrincipalGraphHelper(GraphHelperSettings settings, IAuditService auditService, ILogger logger) : base(settings, auditService, logger) { } + public ServicePrincipalGraphHelper(GraphHelperSettings settings, IAuditService auditService, ILogger logger) + : base(settings, auditService, logger) + { + } [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Console.WriteLine will be changed to logs")] public override async Task<(GraphOperationMetrics metrics, IEnumerable data)> GetDeltaGraphObjects(ActivityContext context, ProcessorConfiguration config, string selectFields = null) @@ -26,6 +27,7 @@ public ServicePrincipalGraphHelper(GraphHelperSettings settings, IAuditService a { throw new ArgumentNullException(nameof(config)); } + if (string.IsNullOrWhiteSpace(selectFields)) { selectFields = string.Join(',', config.SelectFields); @@ -42,8 +44,6 @@ public ServicePrincipalGraphHelper(GraphHelperSettings settings, IAuditService a servicePrincipalCollectionPage = await GraphClient.ServicePrincipals .Delta() .Request() - //.Select(selectFields) - .Top(500) .GetAsync() .ConfigureAwait(false); } @@ -65,7 +65,7 @@ public ServicePrincipalGraphHelper(GraphHelperSettings settings, IAuditService a metrics.Considered = servicePrincipalList.Count; while (servicePrincipalCollectionPage.NextPageRequest != null) { - servicePrincipalCollectionPage = await servicePrincipalCollectionPage.NextPageRequest.Top(500).GetAsync().ConfigureAwait(false); + servicePrincipalCollectionPage = await servicePrincipalCollectionPage.NextPageRequest.GetAsync().ConfigureAwait(false); var pageList = servicePrincipalCollectionPage.CurrentPage; var count = pageList.Count; @@ -103,7 +103,7 @@ public ServicePrincipalGraphHelper(GraphHelperSettings settings, IAuditService a return (metrics, servicePrincipalList); } - public async override Task GetGraphObject(string id) + public async override Task GetGraphObjectWithOwners(string id) { var entity = await GraphClient.ServicePrincipals[id] .Request() @@ -114,14 +114,6 @@ public async override Task GetGraphObject(string id) return entity; } - //public async override Task> GetGraphObjects(IEnumerable queryOptions) - //{ - // var entityList = await graphClient.ServicePrincipals - // .Request(queryOptions) - // .GetAsync() - // .ConfigureAwait(false); - // return entityList; - //} public async override Task PatchGraphObject(ServicePrincipal servicePrincipal) { // API call uses a PATCH so only include properties to change @@ -131,15 +123,12 @@ await GraphClient.ServicePrincipals[servicePrincipal.Id] .ConfigureAwait(false); } - #region HELPER - private static bool IsSeedRun(ProcessorConfiguration config) { return config.RunState == RunState.Seedonly || config.RunState == RunState.SeedAndRun || - String.IsNullOrEmpty(config.DeltaLink); + string.IsNullOrEmpty(config.DeltaLink); } - #endregion } } diff --git a/src/Automation/CSE.Automation/Graph/UserGraphHelper.cs b/src/Automation/CSE.Automation/Graph/UserGraphHelper.cs index 7d9e4d7d..4cc00bb3 100644 --- a/src/Automation/CSE.Automation/Graph/UserGraphHelper.cs +++ b/src/Automation/CSE.Automation/Graph/UserGraphHelper.cs @@ -21,7 +21,7 @@ public UserGraphHelper(GraphHelperSettings settings, IAuditService auditService, throw new NotImplementedException(); } - public async override Task GetGraphObject(string id) + public async override Task GetGraphObjectWithOwners(string id) { try { diff --git a/src/Automation/CSE.Automation/GraphDeltaProcessor.cs b/src/Automation/CSE.Automation/GraphDeltaProcessor.cs index 2f2cf7bb..235e2ddc 100644 --- a/src/Automation/CSE.Automation/GraphDeltaProcessor.cs +++ b/src/Automation/CSE.Automation/GraphDeltaProcessor.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.Threading.Tasks; +using CSE.Automation.Extensions; using CSE.Automation.Interfaces; using CSE.Automation.Model; using CSE.Automation.Processors; @@ -21,136 +22,282 @@ namespace CSE.Automation { internal class GraphDeltaProcessor { - private readonly IServicePrincipalProcessor _processor; - private readonly ILogger _logger; - private readonly IServiceProvider _serviceProvider; + private readonly IActivityService activityService; + private readonly IServicePrincipalProcessor processor; + private readonly ILogger logger; - public GraphDeltaProcessor(IServiceProvider serviceProvider, IServicePrincipalProcessor processor, ILogger logger) + public GraphDeltaProcessor(IServiceProvider serviceProvider, IActivityService activityService, IServicePrincipalProcessor processor, ILogger logger) { - _processor = processor; - _logger = logger; - _serviceProvider = serviceProvider; + this.activityService = activityService; + this.processor = processor; + this.logger = logger; + ValidateServices(serviceProvider); } + /// + /// Request an AAD Discovery Activity + /// + /// An instance of an . + /// Flag to request a full scan. Default is false. + /// An instance of an . + /// A JSON object containing information about the requested activity. + [FunctionName("RequestDiscovery")] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Ensure graceful return under all trappable error conditions.")] + public async Task RequestDiscovery([HttpTrigger(AuthorizationLevel.Function, "get")] HttpRequest req, ILogger log) + { + var discoveryMode = bool.TryParse(req.Query["full"], out var fullDiscovery) && fullDiscovery + ? DiscoveryMode.FullSeed + : DiscoveryMode.Deltas; + var hasRedirect = req.Query.ContainsKey("redirect"); - [FunctionName("DiscoverDeltas")] - [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Will add specific error in time.")] + try + { + if (req is null) + { + throw new ArgumentNullException(nameof(req)); + } + + var result = await CommandDiscovery(discoveryMode, "HTTP", log).ConfigureAwait(false); + + return hasRedirect + // TODO: construct this URI properly + ? new RedirectResult($"{req.Scheme}://{req.Host}/api/Activities?correlationId={result.CorrelationId}") + : (IActionResult)new JsonResult(result); + + } + catch (Exception ex) + { + var message = $"Failed to request Discovery {discoveryMode}"; + log.LogError(ex, message); + + return new BadRequestObjectResult(message); + } + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Ensure graceful return under all trappable error conditions.")] [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA1801:Review unused parameters", Justification = "Required as part of Trigger declaration.")] + [FunctionName("DiscoverDeltas")] public async Task Deltas([TimerTrigger(Constants.DeltaDiscoverySchedule, RunOnStartup = false)] TimerInfo myTimer, ILogger log) { + var discoveryMode = DiscoveryMode.Deltas; try { - using var context = new ActivityContext("Delta Detection").WithLock(_processor); - log.LogDebug("Executing SeedDeltaProcessorTimer Function"); - - var metrics = await _processor.DiscoverDeltas(context, false).ConfigureAwait(false); - context.End(); - log.LogTrace($"Deltas: {metrics.Found} ServicePrincipals discovered in {context.ElapsedTime}."); + await CommandDiscovery(discoveryMode, "TIMER", log).ConfigureAwait(false); } catch (Exception ex) { - log.LogError(ex, Resources.LockConflictMessage); + var message = $"Failed to request Discovery {discoveryMode}"; + log.LogError(ex, message); } } - [FunctionName("FullSeed")] - public async Task FullSeed([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req, ILogger log) + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Ensure graceful return under all trappable error conditions.")] + [FunctionName("Discover")] + [StorageAccount(Constants.SPStorageConnectionString)] + public async Task Discover([QueueTrigger(Constants.DiscoverQueueAppSetting)] CloudQueueMessage msg, ILogger log) { + RequestDiscoveryCommand command; + + log.LogInformation("Incoming message from Discover queue"); try { - using var context = new ActivityContext("Full Seed").WithLock(_processor); - log.LogDebug("Executing SeedDeltaProcessor HttpTrigger Function"); - - // TODO: If we end up with now request params needed for the seed function then remove the param and this check. - if (req is null) - { - throw new ArgumentNullException(nameof(req)); - } + command = JsonConvert.DeserializeObject>(msg.AsString).Document; + } + catch (Exception ex) + { + log.LogError(ex, $"Failed to deserialize queue message into RequestDicoveryCommand."); + return; + } - var metrics = await _processor.DiscoverDeltas(context, true).ConfigureAwait(false); - context.End(); + var operation = command.DiscoveryMode.Description(); + using var context = activityService.CreateContext(operation, withTracking: true, correlationId: command.CorrelationId); + try + { + log.LogDebug("Executing Discover QueueTrigger Function"); - var result = new - { - Operation = "Full Seed", - metrics.Considered, - Ignored = metrics.Removed, - metrics.Found, - context.ElapsedTime, - }; + context.Activity.CommandSource = command.Source; + context.WithProcessorLock(processor); - return new JsonResult(result); + var metrics = await processor.DiscoverDeltas(context, true).ConfigureAwait(false); + context.End(); } catch (Exception ex) { + context.Activity.Status = ActivityHistoryStatus.Failed; + + ex.Data["activityContext"] = context; log.LogError(ex, Resources.LockConflictMessage); - return new BadRequestObjectResult($"Cannot start processor: {Resources.LockConflictMessage}"); } } + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Ensure graceful return under all trappable error conditions.")] [FunctionName("Evaluate")] [StorageAccount(Constants.SPStorageConnectionString)] public async Task Evaluate([QueueTrigger(Constants.EvaluateQueueAppSetting)] CloudQueueMessage msg, ILogger log) { - using var context = new ActivityContext("Evaluate Service Principal"); + EvaluateServicePrincipalCommand command; + + log.LogInformation("Incoming message from Evaluate queue"); try { + command = JsonConvert.DeserializeObject>(msg.AsString).Document; + } + catch (Exception ex) + { + log.LogError(ex, $"Failed to deserialize queue message into EvaluateServicePrincipalCommand."); + return; + } + + using var context = activityService.CreateContext("Evaluate Service Principal", correlationId: command.CorrelationId); + try + { + context.Activity.CommandSource = "QUEUE"; if (msg == null) { throw new ArgumentNullException(nameof(msg)); } - log.LogTrace("Incoming message from Evaluate queue"); - var message = JsonConvert.DeserializeObject>(msg.AsString); - - await _processor.Evaluate(context, message.Document).ConfigureAwait(false); + await processor.Evaluate(context, command.Model).ConfigureAwait(false); context.End(); log.LogTrace($"Evaluate Queue trigger function processed: {msg.Id} in {context.ElapsedTime}"); } catch (Exception ex) { + context.Activity.Status = ActivityHistoryStatus.Failed; + ex.Data["activityContext"] = context; log.LogError(ex, $"Message {msg.Id} aborting: {ex.Message}"); throw; } - } + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Ensure graceful return under all trappable error conditions.")] [FunctionName("UpdateAAD")] [StorageAccount(Constants.SPStorageConnectionString)] public async Task UpdateAAD([QueueTrigger(Constants.UpdateQueueAppSetting)] CloudQueueMessage msg, ILogger log) { - using var context = new ActivityContext("Update Service Principal"); + ServicePrincipalUpdateCommand command; + + log.LogTrace("Incoming message from Update queue"); + try + { + command = JsonConvert.DeserializeObject>(msg.AsString).Document; + } + catch (Exception ex) + { + log.LogError(ex, $"Failed to deserialize queue message into ServicePrincipalUpdateCommand."); + return; + } + + using var context = activityService.CreateContext("Update Service Principal", correlationId: command.CorrelationId); try { + context.Activity.CommandSource = "QUEUE"; if (msg == null) { throw new ArgumentNullException(nameof(msg)); } - log.LogTrace("Incoming message from Update queue"); var message = JsonConvert.DeserializeObject>(msg.AsString); - await _processor.UpdateServicePrincipal(context, message.Document).ConfigureAwait(false); + await processor.UpdateServicePrincipal(context, message.Document).ConfigureAwait(false); context.End(); log.LogTrace($"Update Queue trigger function processed: {msg.Id} in {context.ElapsedTime}"); } catch (Exception ex) { + context.Activity.Status = ActivityHistoryStatus.Failed; + ex.Data["activityContext"] = context; log.LogError(ex, $"Message {msg.Id} aborting: {ex.Message}"); throw; } + } + + /// + /// Get the status of an activity. + /// + /// HttpRequest instance + /// An instance of an . + /// A JSON object containing information about the requested activity. + /// Querystring parameters: activityId, correlationId. One of the two must be provided. + [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "Ensure graceful return under all trappable error conditions.")] + [FunctionName("Activities")] + public async Task Activities([HttpTrigger(AuthorizationLevel.Function, "get", Route = null)] HttpRequest req, ILogger log) + { + var activityId = req.Query["activityId"]; + var correlationId = req.Query["correlationId"]; + + using var context = activityService.CreateContext("Activities", withTracking: false); + try + { + log.LogDebug("Executing ActivityStatus HttpTrigger Function"); + + if (string.IsNullOrWhiteSpace(correlationId) && string.IsNullOrWhiteSpace(activityId)) + { + throw new InvalidOperationException("Either activityId or correlationId must be specified."); + } + + var activityHistory = await processor.GetActivityStatus(context, activityId, correlationId).ConfigureAwait(false); + context.End(); + + var result = new + { + Operation = "Activity Status", + Activity = activityHistory, + context.ElapsedTime, + }; + + return new JsonResult(result); + } + catch (Exception ex) + { + context.Activity.Status = ActivityHistoryStatus.Failed; + ex.Data["activityContext"] = context; + log.LogError(ex, "Failed to retrieve activity status."); + + return new BadRequestObjectResult(ex.Message); + } } - #region RUNTIME VALIDATION - private static void ValidateServices(IServiceProvider serviceProvider) + private async Task CommandDiscovery(DiscoveryMode discoveryMode, string source, ILogger log) + { + using var context = activityService.CreateContext($"{discoveryMode.Description()} Request", withTracking: true); + try + { + context.Activity.CommandSource = source; + await processor.RequestDiscovery(context, discoveryMode, source).ConfigureAwait(false); + var result = new + { + Timestamp = DateTimeOffset.Now, + Operation = discoveryMode.ToString(), + DiscoveryMode = discoveryMode, + ActivityId = context.Activity.Id, + CorrelationId = context.CorrelationId, + }; + + return result; + } + catch (Exception ex) + { + context.Activity.Status = ActivityHistoryStatus.Failed; + + ex.Data["activityContext"] = context; + + var message = $"Failed to request Discovery {discoveryMode}"; + log.LogError(ex, message); + + throw; + } + } + + private void ValidateServices(IServiceProvider serviceProvider) { var repositories = serviceProvider.GetServices(); var hasFailingTest = false; @@ -166,17 +313,18 @@ private static void ValidateServices(IServiceProvider serviceProvider) var message = $"Repository test for {repository.Id} {result}"; if (testPassed) { - Trace.TraceInformation(message); + logger.LogInformation(message); } else { - Trace.TraceError(message); + logger.LogError(message); } } if (hasFailingTest) + { throw new ApplicationException($"One or more repositories failed test."); + } } - #endregion } } diff --git a/src/Automation/CSE.Automation/Interfaces/IActivityHistoryRepository.cs b/src/Automation/CSE.Automation/Interfaces/IActivityHistoryRepository.cs new file mode 100644 index 00000000..e19ae9b7 --- /dev/null +++ b/src/Automation/CSE.Automation/Interfaces/IActivityHistoryRepository.cs @@ -0,0 +1,12 @@ +using System.Collections; +using System.Collections.Generic; +using System.Threading.Tasks; +using CSE.Automation.Model; + +namespace CSE.Automation.Interfaces +{ + internal interface IActivityHistoryRepository : ICosmosDBRepository + { + Task> GetCorrelated(string correlationId); + } +} diff --git a/src/Automation/CSE.Automation/Interfaces/IActivityService.cs b/src/Automation/CSE.Automation/Interfaces/IActivityService.cs new file mode 100644 index 00000000..23e73207 --- /dev/null +++ b/src/Automation/CSE.Automation/Interfaces/IActivityService.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using CSE.Automation.Model; +using Microsoft.Azure.KeyVault; +using Microsoft.OData.UriParser; + +namespace CSE.Automation.Interfaces +{ + internal interface IActivityService + { + Task Put(ActivityHistory document); + Task Get(string id); + Task> GetCorrelated(string correlationId); + ActivityContext CreateContext(string name, string correlationId = null, bool withTracking = false); + } +} diff --git a/src/Automation/CSE.Automation/Interfaces/IAuditRepository.cs b/src/Automation/CSE.Automation/Interfaces/IAuditRepository.cs new file mode 100644 index 00000000..439a6b25 --- /dev/null +++ b/src/Automation/CSE.Automation/Interfaces/IAuditRepository.cs @@ -0,0 +1,6 @@ +using CSE.Automation.Model; + +namespace CSE.Automation.Interfaces +{ + internal interface IAuditRepository : ICosmosDBRepository { } +} diff --git a/src/Automation/CSE.Automation/Interfaces/IClassifier.cs b/src/Automation/CSE.Automation/Interfaces/IClassifier.cs new file mode 100644 index 00000000..48170aab --- /dev/null +++ b/src/Automation/CSE.Automation/Interfaces/IClassifier.cs @@ -0,0 +1,15 @@ +using System.Threading.Tasks; +using CSE.Automation.Model; + +namespace CSE.Automation.Interfaces +{ + internal interface IClassifier + where TEntity : class + { + Task Classify(TEntity entity); + } + + internal interface IServicePrincipalClassifier : IClassifier + { + } +} diff --git a/src/Automation/CSE.Automation/Interfaces/IConfigService.cs b/src/Automation/CSE.Automation/Interfaces/IConfigService.cs index 5bc0c09f..4b89a416 100644 --- a/src/Automation/CSE.Automation/Interfaces/IConfigService.cs +++ b/src/Automation/CSE.Automation/Interfaces/IConfigService.cs @@ -6,7 +6,8 @@ namespace CSE.Automation.Interfaces { - internal interface IConfigService where TConfig : class + internal interface IConfigService + where TConfig : class { Task Put(TConfig newDocument); TConfig Get(string id, ProcessorType processorType, string defaultConfigResourceName, bool createIfNotFound = false); diff --git a/src/Automation/CSE.Automation/Interfaces/ICosmosDBRepository.cs b/src/Automation/CSE.Automation/Interfaces/ICosmosDBRepository.cs index 60e9c183..b7da5e4c 100644 --- a/src/Automation/CSE.Automation/Interfaces/ICosmosDBRepository.cs +++ b/src/Automation/CSE.Automation/Interfaces/ICosmosDBRepository.cs @@ -7,7 +7,8 @@ namespace CSE.Automation.Interfaces { - public interface ICosmosDBRepository : IRepository where TEntity : class + public interface ICosmosDBRepository : IRepository + where TEntity : class { Task GetByIdAsync(string id, string partitionKey); Task> GetByIdWithMetaAsync(string id, string partitionKey); @@ -16,7 +17,6 @@ public interface ICosmosDBRepository : IRepository where TEntity : clas string GenerateId(TEntity entity); Task ReplaceDocumentAsync(string id, TEntity newDocument, ItemRequestOptions reqOptions = null); Task CreateDocumentAsync(TEntity newDocument); - //Task DoesExistsAsync(string id); Task UpsertDocumentAsync(TEntity newDocument); Task DeleteDocumentAsync(string id, string partitionKey); string DatabaseName { get; } diff --git a/src/Automation/CSE.Automation/Interfaces/IDAL.cs b/src/Automation/CSE.Automation/Interfaces/IDAL.cs deleted file mode 100644 index d442e4d3..00000000 --- a/src/Automation/CSE.Automation/Interfaces/IDAL.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - - -namespace CSE.Automation.Interfaces -{ - public interface IDAL - { - public Task GetByIdAsync(string Id, string partitionKey); - public Task> GetPagedAsync(string q, int offset = 0, int limit = 0); - public Task> GetAllAsync(TypeFilter filter = TypeFilter.any); - public Task Reconnect(Uri cosmosUrl, string cosmosKey, string cosmosDatabase, string cosmosCollection, bool force = false); - public Task ReplaceDocumentAsync(string id, T newDocument, string partitionKey = null); - public Task CreateDocumentAsync(T newDocument, string partitionKey = null); - public Task DoesExistsAsync(string id, string partitionKey); - } -} diff --git a/src/Automation/CSE.Automation/Interfaces/IDeltaProcessor.cs b/src/Automation/CSE.Automation/Interfaces/IDeltaProcessor.cs index 7519b04a..f67c43d2 100644 --- a/src/Automation/CSE.Automation/Interfaces/IDeltaProcessor.cs +++ b/src/Automation/CSE.Automation/Interfaces/IDeltaProcessor.cs @@ -1,15 +1,19 @@ using CSE.Automation.Graph; using CSE.Automation.Model; using System; +using System.Collections.Generic; using System.Threading.Tasks; namespace CSE.Automation.Interfaces { - public interface IDeltaProcessor + internal interface IDeltaProcessor { int VisibilityDelayGapSeconds { get; } int QueueRecordProcessThreshold { get; } + Task> GetActivityStatus(ActivityContext context, string activityId = null, string correlationId = null); + Task RequestDiscovery(ActivityContext context, DiscoveryMode discoveryMode, string source); + Task DiscoverDeltas(ActivityContext context, bool forceReseed = false); Task Lock(); Task Unlock(); diff --git a/src/Automation/CSE.Automation/Interfaces/IGraphModel.cs b/src/Automation/CSE.Automation/Interfaces/IGraphModel.cs index abd22c49..2ab89342 100644 --- a/src/Automation/CSE.Automation/Interfaces/IGraphModel.cs +++ b/src/Automation/CSE.Automation/Interfaces/IGraphModel.cs @@ -13,7 +13,5 @@ public interface IGraphModel public DateTimeOffset? LastUpdated { get; set; } public ObjectType ObjectType { get; set; } - - public Status Status { get; set; } } } diff --git a/src/Automation/CSE.Automation/Model/ActivityContext.cs b/src/Automation/CSE.Automation/Model/ActivityContext.cs index cff6a66b..6638d876 100644 --- a/src/Automation/CSE.Automation/Model/ActivityContext.cs +++ b/src/Automation/CSE.Automation/Model/ActivityContext.cs @@ -7,43 +7,54 @@ namespace CSE.Automation.Model { - public sealed class ActivityContext : IDisposable + internal sealed class ActivityContext : IDisposable { - public ActivityContext() + public ActivityContext(IActivityService activityService) + : this() { - Timer = new Stopwatch(); - Timer.Start(); + this.activityService = activityService; } - public ActivityContext(string activityName) - : this() + private ActivityContext() { - ActivityName = activityName; + Timer = new Stopwatch(); + Timer.Start(); } - public void End() + public void End(ActivityHistoryStatus status = ActivityHistoryStatus.Completed) { + if (status != ActivityHistoryStatus.Completed && status != ActivityHistoryStatus.Failed) + { + throw new ArgumentOutOfRangeException(nameof(status)); + } + + this.Activity.Status = ActivityHistoryStatus.Completed; + if (Timer is null) + { return; - _elapsed = Timer.Elapsed; + } + + elapsed = Timer.Elapsed; Timer = null; } - public string ActivityName { get; set; } - public Guid ActivityId => Guid.NewGuid(); - public DateTimeOffset StartTime => DateTimeOffset.Now; + public ActivityHistory Activity { get; set; } + public DateTimeOffset StartTime { get; } = DateTimeOffset.Now; + public string CorrelationId { get; private set; } = Guid.NewGuid().ToString(); + private IActivityService activityService; private IDeltaProcessor processor; - private TimeSpan? _elapsed; + private TimeSpan? elapsed; private bool disposedValue; - private bool isLocked = false; + private bool isLocked; - public TimeSpan ElapsedTime { get { return _elapsed ?? Timer.Elapsed; } } + public TimeSpan ElapsedTime { get { return elapsed ?? Timer.Elapsed; } } [JsonIgnore] public Stopwatch Timer { get; private set; } - public ActivityContext WithLock(IDeltaProcessor deltaProcessor) + public ActivityContext WithProcessorLock(IDeltaProcessor deltaProcessor) { if (deltaProcessor == null) { @@ -51,11 +62,28 @@ public ActivityContext WithLock(IDeltaProcessor deltaProcessor) } deltaProcessor.Lock().Wait(); + isLocked = true; processor = deltaProcessor; return this; } + /// + /// Set the correlation id of the activity in the context. + /// + /// A correlationId to use for this context. + /// The instance of the ActivityHistroy + public ActivityContext WithCorrelationId(string correlationId) + { + CorrelationId = correlationId; + if (Activity != null) + { + Activity.CorrelationId = CorrelationId; + } + + return this; + } + public void Dispose() { // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method @@ -69,6 +97,16 @@ private void Dispose(bool disposing) { if (disposing) { + if (activityService != null) + { + if (this.Activity.Status != ActivityHistoryStatus.Failed) + { + this.Activity.Status = ActivityHistoryStatus.Completed; + } + + activityService.Put(this.Activity).Wait(); + } + if (isLocked) { processor.Unlock().Wait(); diff --git a/src/Automation/CSE.Automation/Model/ActivityHistory.cs b/src/Automation/CSE.Automation/Model/ActivityHistory.cs new file mode 100644 index 00000000..a99e70d4 --- /dev/null +++ b/src/Automation/CSE.Automation/Model/ActivityHistory.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace CSE.Automation.Model +{ + internal class ActivityHistory + { + /// + /// Gets or sets unique Id of the document + /// + public string Id { get; set; } + + /// + /// Gets or sets the correlation Id of the activity + /// + public string CorrelationId { get; set; } + + public string Name { get; set; } + + public string CommandSource { get; set; } + /// + /// Gets or sets the status of the activity + /// + public ActivityHistoryStatus Status { get; set; } + + /// + /// Gets or sets the metrics for the run + /// + public IDictionary Metrics { get; set; } = new Dictionary(); + + /// + /// Gets or sets timestamp of when the document was created + /// + public DateTimeOffset Created { get; set; } + + /// + /// Gets or sets timestamp of when the document was last updated + /// + public DateTimeOffset LastUpdated { get; set; } + + public void MergeMetrics(IDictionary dict) + { + Metrics = Metrics.Concat(dict).ToDictionary(x => x.Key, x => x.Value); + } + } + + [JsonConverter(typeof(StringEnumConverter))] + internal enum ActivityHistoryStatus + { + /// + /// Activity is actively running + /// + Running, + + /// + /// Activity has completed successfully. + /// + Completed, + + /// + /// Activity has completed with errors + /// + Failed, + } +} diff --git a/src/Automation/CSE.Automation/Model/EvaluateServicePrincipalCommand.cs b/src/Automation/CSE.Automation/Model/EvaluateServicePrincipalCommand.cs new file mode 100644 index 00000000..dacc5d2d --- /dev/null +++ b/src/Automation/CSE.Automation/Model/EvaluateServicePrincipalCommand.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace CSE.Automation.Model +{ + internal class EvaluateServicePrincipalCommand + { + public string CorrelationId { get; set; } + public ServicePrincipalModel Model { get; set; } + } +} diff --git a/src/Automation/CSE.Automation/Model/GraphModel.cs b/src/Automation/CSE.Automation/Model/GraphModel.cs index 9dd312a0..86edcd61 100644 --- a/src/Automation/CSE.Automation/Model/GraphModel.cs +++ b/src/Automation/CSE.Automation/Model/GraphModel.cs @@ -14,16 +14,18 @@ public class GraphModel : IGraphModel public DateTimeOffset? LastUpdated { get; set; } public ObjectType ObjectType { get; set; } - - public Status Status { get; set; } } [JsonConverter(typeof(StringEnumConverter))] public enum ObjectType { - ServicePrincipal + /// + /// Graph model type of ServicePrincipal + /// + ServicePrincipal, } + /* [JsonConverter(typeof(StringEnumConverter))] public enum Status { @@ -32,4 +34,5 @@ public enum Status Deleted, Remediated } + */ } diff --git a/src/Automation/CSE.Automation/Model/ObjectClassification.cs b/src/Automation/CSE.Automation/Model/ObjectClassification.cs new file mode 100644 index 00000000..1ad47df6 --- /dev/null +++ b/src/Automation/CSE.Automation/Model/ObjectClassification.cs @@ -0,0 +1,13 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace CSE.Automation.Model +{ + [JsonConverter(typeof(StringEnumConverter))] + internal enum ObjectClassification + { + Microsoft, + External, + Company + } +} diff --git a/src/Automation/CSE.Automation/Model/RequestDiscoveryCommand.cs b/src/Automation/CSE.Automation/Model/RequestDiscoveryCommand.cs new file mode 100644 index 00000000..a91e94e4 --- /dev/null +++ b/src/Automation/CSE.Automation/Model/RequestDiscoveryCommand.cs @@ -0,0 +1,29 @@ +using System.ComponentModel; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace CSE.Automation.Model +{ + [JsonConverter(typeof(StringEnumConverter))] + internal enum DiscoveryMode + { + /// + /// Execute a full AAD scan + /// + [Description("Full Discovery")] + FullSeed, + + /// + /// Execute a delta query + /// + [Description("Delta Discovery")] + Deltas, + } + + internal class RequestDiscoveryCommand + { + public string CorrelationId { get; set; } + public DiscoveryMode DiscoveryMode { get; set; } + public string Source { get; set; } + } +} diff --git a/src/Automation/CSE.Automation/Model/ServicePrincipalClassification.cs b/src/Automation/CSE.Automation/Model/ServicePrincipalClassification.cs new file mode 100644 index 00000000..7d223eb3 --- /dev/null +++ b/src/Automation/CSE.Automation/Model/ServicePrincipalClassification.cs @@ -0,0 +1,14 @@ +using System; + +namespace CSE.Automation.Model +{ + internal class ServicePrincipalClassification + { + public Guid OwningTenant { get; set; } + public string Type { get; set; } + public ObjectClassification Classification { get; set; } + public string Category { get; set; } + public string OwningTenantDomain { get; set; } + public bool HasOwner { get; set; } + } +} diff --git a/src/Automation/CSE.Automation/Processors/DeltaProcessorBase.cs b/src/Automation/CSE.Automation/Processors/DeltaProcessorBase.cs index e00fc17c..80f4f2cd 100644 --- a/src/Automation/CSE.Automation/Processors/DeltaProcessorBase.cs +++ b/src/Automation/CSE.Automation/Processors/DeltaProcessorBase.cs @@ -1,75 +1,49 @@ -using CSE.Automation.Interfaces; -using CSE.Automation.Model; -using CSE.Automation.Properties; -using Newtonsoft.Json; -using System; -using System.Configuration; -using System.IO; +using System; +using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.Azure.Cosmos; -using SettingsBase = CSE.Automation.Model.SettingsBase; -using Microsoft.Identity.Client; -using System.Net.WebSockets; using CSE.Automation.Graph; +using CSE.Automation.Interfaces; +using CSE.Automation.Model; using Microsoft.Extensions.Logging; namespace CSE.Automation.Processors { - public class DeltaProcessorSettings : SettingsBase - { - public DeltaProcessorSettings(ISecretClient secretClient) - : base(secretClient) - { - } - - public Guid ConfigurationId { get; set; } - public int VisibilityDelayGapSeconds { get; set; } - public int QueueRecordProcessThreshold { get; set; } - - public override void Validate() - { - base.Validate(); - if (this.ConfigurationId == Guid.Empty) - { - throw new ConfigurationErrorsException($"{this.GetType().Name}: ConfigurationId is invalid"); - } - - if (this.VisibilityDelayGapSeconds <= 0 || this.VisibilityDelayGapSeconds > Constants.MaxVisibilityDelayGapSeconds) - { - throw new ConfigurationErrorsException($"{this.GetType().Name}: VisibilityDelayGapSeconds is invalid"); - } - - if (this.QueueRecordProcessThreshold <= 0 || this.QueueRecordProcessThreshold > Constants.MaxQueueRecordProcessThreshold) - { - throw new ConfigurationErrorsException($"{this.GetType().Name}: QueueRecordProcessThreshold is invalid"); - } - } - } - internal abstract class DeltaProcessorBase : IDeltaProcessor { + protected readonly ILogger logger; + protected readonly IConfigService configService; + protected ProcessorConfiguration config; + private bool initialized; + protected DeltaProcessorBase(IConfigService configService, ILogger logger) { - _configService = configService; - _logger = logger; + this.configService = configService; + this.logger = logger; } - protected readonly ILogger _logger; - protected readonly IConfigService _configService; - protected ProcessorConfiguration _config; - private bool _initialized; - public abstract int VisibilityDelayGapSeconds { get; } public abstract int QueueRecordProcessThreshold { get; } public abstract Guid ConfigurationId { get; } public abstract ProcessorType ProcessorType { get; } protected abstract string DefaultConfigurationResourceName { get; } + public abstract Task RequestDiscovery(ActivityContext context, DiscoveryMode discoveryMode, string source); + public abstract Task> GetActivityStatus(ActivityContext context, string activityId, string correlationId); public abstract Task DiscoverDeltas(ActivityContext context, bool forceReseed = false); + public async Task Lock() + { + await configService.Lock(this.ConfigurationId.ToString(), this.DefaultConfigurationResourceName).ConfigureAwait(false); + } + + public async Task Unlock() + { + await configService.Unlock().ConfigureAwait(false); + } + protected void EnsureInitialized() { - if (_initialized) + if (initialized) { return; } @@ -79,20 +53,11 @@ protected void EnsureInitialized() private void Initialize() { - _logger.LogInformation($"Initializing {this.GetType().Name}"); + logger.LogInformation($"Initializing {this.GetType().Name}"); - _config = _configService.Get(this.ConfigurationId.ToString(), ProcessorType, DefaultConfigurationResourceName, true); - _initialized = true; + config = configService.Get(this.ConfigurationId.ToString(), ProcessorType, DefaultConfigurationResourceName, true); + initialized = true; } - public async Task Lock() - { - await _configService.Lock(this.ConfigurationId.ToString(), this.DefaultConfigurationResourceName).ConfigureAwait(false); - } - - public async Task Unlock() - { - await _configService.Unlock().ConfigureAwait(false); - } } } diff --git a/src/Automation/CSE.Automation/Processors/DeltaProcessorSettings.cs b/src/Automation/CSE.Automation/Processors/DeltaProcessorSettings.cs new file mode 100644 index 00000000..d85524b8 --- /dev/null +++ b/src/Automation/CSE.Automation/Processors/DeltaProcessorSettings.cs @@ -0,0 +1,48 @@ +using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using CSE.Automation.Properties; +using Newtonsoft.Json; +using System; +using System.Configuration; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using SettingsBase = CSE.Automation.Model.SettingsBase; +using Microsoft.Identity.Client; +using System.Net.WebSockets; +using CSE.Automation.Graph; +using Microsoft.Extensions.Logging; + +namespace CSE.Automation.Processors +{ + public class DeltaProcessorSettings : SettingsBase + { + public DeltaProcessorSettings(ISecretClient secretClient) + : base(secretClient) + { + } + + public Guid ConfigurationId { get; set; } + public int VisibilityDelayGapSeconds { get; set; } + public int QueueRecordProcessThreshold { get; set; } + + public override void Validate() + { + base.Validate(); + if (this.ConfigurationId == Guid.Empty) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: ConfigurationId is invalid"); + } + + if (this.VisibilityDelayGapSeconds <= 0 || this.VisibilityDelayGapSeconds > Constants.MaxVisibilityDelayGapSeconds) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: VisibilityDelayGapSeconds is invalid"); + } + + if (this.QueueRecordProcessThreshold <= 0 || this.QueueRecordProcessThreshold > Constants.MaxQueueRecordProcessThreshold) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: QueueRecordProcessThreshold is invalid"); + } + } + } +} diff --git a/src/Automation/CSE.Automation/Processors/ServicePrincipalProcessor.cs b/src/Automation/CSE.Automation/Processors/ServicePrincipalProcessor.cs index ed15cb55..aebb52a0 100644 --- a/src/Automation/CSE.Automation/Processors/ServicePrincipalProcessor.cs +++ b/src/Automation/CSE.Automation/Processors/ServicePrincipalProcessor.cs @@ -42,6 +42,7 @@ internal class ServicePrincipalProcessorSettings : DeltaProcessorSettings private string _queueConnectionString; private string _evaluateQueueName; private string _updateQueueName; + private string _discoverQueueName; public ServicePrincipalProcessorSettings(ISecretClient secretClient) : base(secretClient) { } @@ -49,7 +50,7 @@ public ServicePrincipalProcessorSettings(ISecretClient secretClient) [Secret(Constants.SPStorageConnectionString)] public string QueueConnectionString { - get { return _queueConnectionString ?? base.GetSecret(); } + get { return _queueConnectionString ?? GetSecret(); } set { _queueConnectionString = value; } } @@ -67,6 +68,12 @@ public string UpdateQueueName set { _updateQueueName = value?.ToLower(); } } + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1304:Specify CultureInfo", Justification = "Not a localizable setting")] + public string DiscoverQueueName + { + get { return _discoverQueueName; } + set { _discoverQueueName = value?.ToLower(); } + } public UpdateMode AADUpdateMode { get; set; } public override void Validate() @@ -87,17 +94,22 @@ public override void Validate() throw new ConfigurationErrorsException($"{this.GetType().Name}: UpdateQueueName is invalid"); } + if (string.IsNullOrEmpty(this.DiscoverQueueName)) + { + throw new ConfigurationErrorsException($"{this.GetType().Name}: DiscoverQueueName is invalid"); + } } } internal class ServicePrincipalProcessor : DeltaProcessorBase, IServicePrincipalProcessor { - private readonly IGraphHelper _graphHelper; - private readonly ServicePrincipalProcessorSettings _settings; - private readonly IQueueServiceFactory _queueServiceFactory; - private readonly IObjectTrackingService _objectService; - private readonly IAuditService _auditService; - private readonly IEnumerable> _validators; + private readonly IGraphHelper graphHelper; + private readonly ServicePrincipalProcessorSettings settings; + private readonly IQueueServiceFactory queueServiceFactory; + private readonly IObjectTrackingService objectService; + private readonly IAuditService auditService; + private readonly IActivityService activityService; + private readonly IEnumerable> validators; public ServicePrincipalProcessor( ServicePrincipalProcessorSettings settings, @@ -106,66 +118,119 @@ public ServicePrincipalProcessor( IConfigService configService, IObjectTrackingService objectService, IAuditService auditService, + IActivityService activityService, IModelValidatorFactory modelValidatorFactory, ILogger logger) : base(configService, logger) { - _settings = settings; - _graphHelper = graphHelper; - _objectService = objectService; - _auditService = auditService; - - _queueServiceFactory = queueServiceFactory; - - _validators = modelValidatorFactory.Get(); + this.settings = settings; + this.graphHelper = graphHelper; + this.objectService = objectService; + this.auditService = auditService; + this.activityService = activityService; + this.queueServiceFactory = queueServiceFactory; + + validators = modelValidatorFactory.Get(); } - public override int VisibilityDelayGapSeconds => _settings.VisibilityDelayGapSeconds; - public override int QueueRecordProcessThreshold => _settings.QueueRecordProcessThreshold; - public override Guid ConfigurationId => _settings.ConfigurationId; + public override int VisibilityDelayGapSeconds => settings.VisibilityDelayGapSeconds; + public override int QueueRecordProcessThreshold => settings.QueueRecordProcessThreshold; + public override Guid ConfigurationId => settings.ConfigurationId; public override ProcessorType ProcessorType => ProcessorType.ServicePrincipal; protected override string DefaultConfigurationResourceName => "ServicePrincipalProcessorConfiguration"; + /// REQUESTDISCOVERY + /// + /// Submit a request to perform a Discovery + /// + /// An instance of an . + /// Type of discovery to perform. + /// A Task that may be awaited. + public override async Task RequestDiscovery(ActivityContext context, DiscoveryMode discoveryMode, string source) + { + var message = new QueueMessage() + { + QueueMessageType = QueueMessageType.Data, + Document = new RequestDiscoveryCommand + { + CorrelationId = context.CorrelationId, + DiscoveryMode = discoveryMode, + Source = source, + }, + Attempt = 0, + }; + + await queueServiceFactory + .Create(settings.QueueConnectionString, settings.DiscoverQueueName) + .Send(message, 0) + .ConfigureAwait(false); + } + + /// DISCOVERYSTATUS + /// + /// Return the status of an activity from activityhistory. + /// + /// An instance of an . + /// Id of the activity to report. + /// Correlation id of the activities to report. + /// A Task that may be awaited. + /// Either activityId or correlationId must be provided. + public override async Task> GetActivityStatus(ActivityContext context, string activityId, string correlationId) + { + if (string.IsNullOrWhiteSpace(correlationId)) + { + var activity = await activityService.Get(activityId).ConfigureAwait(false); + return new[] { activity }; + } + else + { + return await activityService.GetCorrelated(correlationId).ConfigureAwait(false); + } + } + /// DISCOVER /// /// Discover changes to ServicePrincipals in the Directory. Either perform an initial seed or a delta detection action. /// /// Context of the activity. /// Force a reseed regardless of config runstate or deltalink. - /// The number of items Found in the Directory for evaluation. - [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Console.WriteLine will be changed to logs")] + /// A Task that returns an instance of . public override async Task DiscoverDeltas(ActivityContext context, bool forceReseed = false) { EnsureInitialized(); if (forceReseed) { - _config.RunState = RunState.SeedAndRun; + config.RunState = RunState.SeedAndRun; } - IAzureQueueService queueService = _queueServiceFactory.Create(_settings.QueueConnectionString, _settings.EvaluateQueueName); + // Create the queue client for when we need to post the evaluate commands + IAzureQueueService queueService = queueServiceFactory.Create(settings.QueueConnectionString, settings.EvaluateQueueName); + // Perform the delta query against the Graph // var selectFields = new[] { "appId", "displayName", "notes", "additionalData" }; - var servicePrincipalResult = await _graphHelper.GetDeltaGraphObjects(context, _config, /*string.Join(',', selectFields)*/ null).ConfigureAwait(false); + var servicePrincipalResult = await graphHelper.GetDeltaGraphObjects(context, config, /*string.Join(',', selectFields)*/ null).ConfigureAwait(false); var metrics = servicePrincipalResult.metrics; string updatedDeltaLink = metrics.AdditionalData; var servicePrincipalList = servicePrincipalResult.data.ToList(); - _logger.LogInformation($"Resolving Owners for ServicePrincipal objects..."); + logger.LogInformation($"Resolving Owners for ServicePrincipal objects..."); int servicePrincipalCount = 0; var enrichedPrincipals = new List(); + + // TODO: Look at this filter, is it necessary? foreach (var sp in servicePrincipalList.Where(sp => string.IsNullOrWhiteSpace(sp.AppId) == false && string.IsNullOrWhiteSpace(sp.DisplayName) == false)) { - var fullSP = await _graphHelper.GetGraphObject(sp.Id).ConfigureAwait(false); + var fullSP = await graphHelper.GetGraphObjectWithOwners(sp.Id).ConfigureAwait(false); var owners = fullSP?.Owners.Select(x => (x as User)?.UserPrincipalName).ToList(); servicePrincipalCount++; if (servicePrincipalCount % QueueRecordProcessThreshold == 0) { - _logger.LogInformation($"\t{servicePrincipalCount}"); + logger.LogInformation($"\t{servicePrincipalCount}"); } enrichedPrincipals.Add(new ServicePrincipalModel() @@ -179,9 +244,10 @@ public override async Task DiscoverDeltas(ActivityContext Owners = owners, }); } - _logger.LogInformation($"{enrichedPrincipals.Count} ServicePrincipals resolved."); - _logger.LogInformation($"Sending Evaluate messages."); + logger.LogInformation($"{enrichedPrincipals.Count} ServicePrincipals resolved."); + + logger.LogInformation($"Sending Evaluate messages."); servicePrincipalCount = 0; enrichedPrincipals.ForEach(async sp => { @@ -197,27 +263,29 @@ public override async Task DiscoverDeltas(ActivityContext if (servicePrincipalCount % QueueRecordProcessThreshold == 0) { - _logger.LogInformation($"\t{servicePrincipalCount}"); + logger.LogInformation($"\t{servicePrincipalCount}"); } }); - _logger.LogInformation($"Evaluate messages complete."); + logger.LogInformation($"Evaluate messages complete."); - - if (_config.RunState == RunState.SeedAndRun || _config.RunState == RunState.Seedonly) + if (config.RunState == RunState.SeedAndRun || config.RunState == RunState.Seedonly) { - _config.LastSeedTime = DateTimeOffset.Now; + config.LastSeedTime = DateTimeOffset.Now; } else { - _config.LastDeltaRun = DateTimeOffset.Now; + config.LastDeltaRun = DateTimeOffset.Now; } - _config.DeltaLink = updatedDeltaLink; - _config.RunState = RunState.DeltaRun; + config.DeltaLink = updatedDeltaLink; + config.RunState = RunState.DeltaRun; + + await configService.Put(config).ConfigureAwait(false); - await _configService.Put(_config).ConfigureAwait(false); + logger.LogInformation($"Finished Processing {servicePrincipalCount} Service Principal Objects."); - _logger.LogInformation($"Finished Processing {servicePrincipalCount} Service Principal Objects."); + context.Activity.MergeMetrics(metrics.ToDictionary()); + await activityService.Put(context.Activity).ConfigureAwait(false); return metrics; } @@ -230,17 +298,17 @@ public override async Task DiscoverDeltas(ActivityContext /// Task to be awaited. public async Task Evaluate(ActivityContext context, ServicePrincipalModel entity) { - IAzureQueueService queueService = _queueServiceFactory.Create(_settings.QueueConnectionString, _settings.UpdateQueueName); + IAzureQueueService queueService = queueServiceFactory.Create(settings.QueueConnectionString, settings.UpdateQueueName); - var errors = _validators.SelectMany(v => v.Validate(entity).Errors).ToList(); + var errors = validators.SelectMany(v => v.Validate(entity).Errors).ToList(); if (errors.Count > 0) { // emit into Operations log var errorMsg = string.Join('\n', errors); - _logger.LogError($"ServicePrincipal {entity.Id} failed validation.\n{errorMsg}"); + logger.LogError($"ServicePrincipal {entity.Id} failed validation.\n{errorMsg}"); // emit into Audit log, all failures - errors.ForEach(async error => await _auditService.PutFail( + errors.ForEach(async error => await auditService.PutFail( context: context, code: AuditCode.Fail_AttributeValidation, objectId: entity.Id, @@ -266,7 +334,7 @@ public async Task Evaluate(ActivityContext context, ServicePrincipalModel entity { // remember this was the last time we saw the prinicpal as 'good' await UpdateLastKnownGood(context, entity).ConfigureAwait(true); - await _auditService.PutPass(context, AuditCode.Pass_ServicePrincipal, entity.Id, null, null).ConfigureAwait(false); + await auditService.PutPass(context, AuditCode.Pass_ServicePrincipal, entity.Id, null, null).ConfigureAwait(false); } } @@ -281,11 +349,11 @@ public async Task Evaluate(ActivityContext context, ServicePrincipalModel entity [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "All failure condition logging")] public async Task UpdateServicePrincipal(ActivityContext context, ServicePrincipalUpdateCommand command) { - if (_settings.AADUpdateMode == UpdateMode.Update) + if (settings.AADUpdateMode == UpdateMode.Update) { try { - await _graphHelper.PatchGraphObject(new ServicePrincipal + await graphHelper.PatchGraphObject(new ServicePrincipal { Id = command.Id, Notes = command.Notes.Changed, @@ -293,14 +361,14 @@ await _graphHelper.PatchGraphObject(new ServicePrincipal } catch (Microsoft.Graph.ServiceException exSvc) { - _logger.LogError(exSvc, $"Failed to update AAD Service Principal {command.Id}"); + logger.LogError(exSvc, $"Failed to update AAD Service Principal {command.Id}"); try { - await _auditService.PutFail(context, AuditCode.Fail_AADUpdate, command.Id, "Notes", command.Notes.Current, exSvc.Message).ConfigureAwait(false); + await auditService.PutFail(context, AuditCode.Fail_AADUpdate, command.Id, "Notes", command.Notes.Current, exSvc.Message).ConfigureAwait(false); } catch (Exception ex) { - _logger.LogError(ex, $"Failed to Audit update to AAD Service Principal {command.Id}"); + logger.LogError(ex, $"Failed to Audit update to AAD Service Principal {command.Id}"); // do not rethrow, it will hide the real failure } @@ -308,17 +376,17 @@ await _graphHelper.PatchGraphObject(new ServicePrincipal try { - await _auditService.PutChange(context, AuditCode.Change_ServicePrincipalUpdated, command.Id, "Notes", command.Notes.Current, command.Notes.Changed, command.Message).ConfigureAwait(false); + await auditService.PutChange(context, AuditCode.Change_ServicePrincipalUpdated, command.Id, "Notes", command.Notes.Current, command.Notes.Changed, command.Message).ConfigureAwait(false); } catch (Exception ex) { - _logger.LogError(ex, $"Failed to Audit update to AAD Service Principal {command.Id}"); + logger.LogError(ex, $"Failed to Audit update to AAD Service Principal {command.Id}"); throw; } } else { - _logger.LogInformation($"Update mode is {_settings.AADUpdateMode}, ServicePrincipal {command.Id} will not be updated."); + logger.LogInformation($"Update mode is {settings.AADUpdateMode}, ServicePrincipal {command.Id} will not be updated."); } } @@ -331,7 +399,7 @@ await _graphHelper.PatchGraphObject(new ServicePrincipal /// A Task that returns nothing. private async Task UpdateNotesFromOwners(ActivityContext context, ServicePrincipalModel entity, IAzureQueueService queueService) { - TrackingModel lastKnownGoodWrapper = await _objectService.Get(entity.Id).ConfigureAwait(true); + TrackingModel lastKnownGoodWrapper = await objectService.Get(entity.Id).ConfigureAwait(true); var lastKnownGood = TrackingModel.Unwrap(lastKnownGoodWrapper); // get new value for Notes (from the list of Owners) @@ -362,13 +430,13 @@ private async Task UpdateNotesFromOwners(ActivityContext context, ServicePrincip /// A Task that returns nothing. private async Task RevertToLastKnownGood(ActivityContext context, ServicePrincipalModel entity, IAzureQueueService queueService) { - TrackingModel lastKnownGoodWrapper = await _objectService.Get(entity.Id).ConfigureAwait(false); + TrackingModel lastKnownGoodWrapper = await objectService.Get(entity.Id).ConfigureAwait(false); var lastKnownGood = TrackingModel.Unwrap(lastKnownGoodWrapper); // bad SP Notes, bad SP Owners, last known good found if (lastKnownGood != null) { - _logger.LogInformation($"Reverting {entity.Id} to last known good state from {lastKnownGood.LastUpdated}"); + logger.LogInformation($"Reverting {entity.Id} to last known good state from {lastKnownGood.LastUpdated}"); // build the command here so we don't need to pass the delta values down the call tree var updateCommand = new ServicePrincipalUpdateCommand() @@ -402,18 +470,18 @@ private async Task UpdateLastKnownGood(ActivityContext context, TrackingModel tr trackingModel.Entity = model; // make sure to write the wrapper back to get the same document updated - await _objectService.Put(context, trackingModel).ConfigureAwait(false); + await objectService.Put(context, trackingModel).ConfigureAwait(false); } } private async Task UpdateLastKnownGood(ActivityContext context, ServicePrincipalModel entity) { - await _objectService.Put(context, entity).ConfigureAwait(false); + await objectService.Put(context, entity).ConfigureAwait(false); } private static async Task CommandAADUpdate(ActivityContext context, ServicePrincipalUpdateCommand command, IAzureQueueService queueService) { - command.CorrelationId = context.ActivityId.ToString(); + command.CorrelationId = context.Activity.Id.ToString(); var message = new QueueMessage() { QueueMessageType = QueueMessageType.Data, @@ -436,8 +504,8 @@ private async Task AlertInvalidPrincipal(ActivityContext context, ServicePrincip { // TODO: move reason text to resource var message = "Missing Owners on ServicePrincipal, cannot remediate."; - await _auditService.PutFail(context, AuditCode.Fail_MissingOwners, entity.Id, "Owners", null, message).ConfigureAwait(true); - _logger.LogWarning($"AUDIT FAIL: {entity.Id} {message}"); + await auditService.PutFail(context, AuditCode.Fail_MissingOwners, entity.Id, "Owners", null, message).ConfigureAwait(true); + logger.LogWarning($"AUDIT FAIL: {entity.Id} {message}"); } catch (Exception) { diff --git a/src/Automation/CSE.Automation/Services/ActivityService.cs b/src/Automation/CSE.Automation/Services/ActivityService.cs new file mode 100644 index 00000000..e459c73e --- /dev/null +++ b/src/Automation/CSE.Automation/Services/ActivityService.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using CSE.Automation.Interfaces; +using CSE.Automation.Model; +using Microsoft.Extensions.Logging; + +namespace CSE.Automation.Services +{ + internal class ActivityService : IActivityService + { + private readonly IActivityHistoryRepository repository; + private readonly ILogger logger; + + public ActivityService(IActivityHistoryRepository repository, ILogger logger) + { + this.repository = repository; + this.logger = logger; + } + + /// + /// Save the ActivityHistory document. + /// + /// The instance of to save. + /// The updated instance of . + public async Task Put(ActivityHistory document) + { + repository.GenerateId(document); + document.LastUpdated = DateTimeOffset.Now; + document = await repository.UpsertDocumentAsync(document).ConfigureAwait(false); + + logger.LogInformation($"Saved history for Run {document.Id}"); + return document; + } + + /// + /// Given a document Id, return an instance of the ActivityHistory document. + /// + /// Unique Id of the document. + /// An instance of document or null. + public async Task Get(string id) + { + return await repository.GetByIdAsync(id, id).ConfigureAwait(false); + } + + public async Task> GetCorrelated(string correlationId) + { + return await repository.GetCorrelated(correlationId).ConfigureAwait(false); + } + + /// + /// Create an instance of an ActivityHistory model + /// + /// Name of the activity + /// Correlation Id of the activity + /// True if the activity is tracked in ActivityHistory + /// A new instance of . + public ActivityContext CreateContext(string name, string correlationId = null, bool withTracking = false) + { + var now = DateTimeOffset.Now; + + correlationId ??= Guid.NewGuid().ToString(); + + var document = new ActivityHistory + { + CorrelationId = correlationId, + Created = now, + Name = name, + }; + + // we need the id of the run when we initiate + repository.GenerateId(document); + + if (withTracking) + { + document = this.Put(document).Result; + } + + return new ActivityContext(withTracking ? this : null) + { + Activity = document, + }.WithCorrelationId(correlationId); + } + } +} diff --git a/src/Automation/CSE.Automation/Services/AuditService.cs b/src/Automation/CSE.Automation/Services/AuditService.cs index edfb8704..7b1c56b4 100644 --- a/src/Automation/CSE.Automation/Services/AuditService.cs +++ b/src/Automation/CSE.Automation/Services/AuditService.cs @@ -2,7 +2,6 @@ using System.ComponentModel; using System.Globalization; using System.Threading.Tasks; -using CSE.Automation.DataAccess; using CSE.Automation.Extensions; using CSE.Automation.Interfaces; using CSE.Automation.Model; @@ -31,7 +30,7 @@ public async Task PutFail(ActivityContext context, AuditCode code, string object var entry = new AuditEntry { - CorrelationId = context.ActivityId.ToString(), + CorrelationId = context.Activity.Id.ToString(), ObjectId = objectId, Type = AuditActionType.Fail, Code = (int)code, @@ -56,7 +55,7 @@ public async Task PutPass(ActivityContext context, AuditCode code, string object var entry = new AuditEntry { - CorrelationId = context.ActivityId.ToString(), + CorrelationId = context.Activity.Id.ToString(), ObjectId = objectId, Type = AuditActionType.Pass, Code = (int)code, @@ -81,7 +80,7 @@ public async Task PutIgnore(ActivityContext context, AuditCode code, string obje var entry = new AuditEntry { - CorrelationId = context.ActivityId.ToString(), + CorrelationId = context.Activity.Id.ToString(), ObjectId = objectId, Type = AuditActionType.Ignore, Code = (int)code, @@ -106,7 +105,7 @@ public async Task PutChange(ActivityContext context, AuditCode code, string obje var entry = new AuditEntry { - CorrelationId = context.ActivityId.ToString(), + CorrelationId = context.Activity.Id.ToString(), ObjectId = objectId, Type = AuditActionType.Change, Code = (int)code, diff --git a/src/Automation/CSE.Automation/Services/ObjectTrackingService.cs b/src/Automation/CSE.Automation/Services/ObjectTrackingService.cs index 046ed337..9d0916f0 100644 --- a/src/Automation/CSE.Automation/Services/ObjectTrackingService.cs +++ b/src/Automation/CSE.Automation/Services/ObjectTrackingService.cs @@ -11,29 +11,31 @@ namespace CSE.Automation.Services { internal class ObjectTrackingService : IObjectTrackingService { - private readonly IObjectTrackingRepository _objectRepository; - private readonly IAuditService _auditService; - private readonly ILogger _logger; + private readonly IObjectTrackingRepository objectRepository; + private readonly IAuditService auditService; + private readonly ILogger logger; public ObjectTrackingService(IObjectTrackingRepository objectRepository, IAuditService auditService, ILogger logger) { - _objectRepository = objectRepository; - _auditService = auditService; - _logger = logger; + this.objectRepository = objectRepository; + this.auditService = auditService; + this.logger = logger; } - public async Task Get(string id) where TEntity : GraphModel + public async Task Get(string id) + where TEntity : GraphModel { - var entity = await _objectRepository + var entity = await objectRepository .GetByIdAsync(id, EntityToObjectType(typeof(TEntity)).ToString().ToCamelCase()) .ConfigureAwait(false); return entity; } - public async Task GetAndUnwrap(string id) where TEntity : GraphModel + public async Task GetAndUnwrap(string id) + where TEntity : GraphModel { - var entity = await _objectRepository + var entity = await objectRepository .GetByIdAsync(id, EntityToObjectType(typeof(TEntity)).ToString().ToCamelCase()) .ConfigureAwait(false); @@ -42,37 +44,39 @@ public async Task GetAndUnwrap(string id) where TEntity : Grap public async Task Put(ActivityContext context, TrackingModel entity) { - _objectRepository.GenerateId(entity); - entity.CorrelationId = context.ActivityId.ToString(); + objectRepository.GenerateId(entity); + entity.CorrelationId = context.Activity.Id.ToString(); entity.LastUpdated = DateTimeOffset.Now; - return await _objectRepository.UpsertDocumentAsync(entity).ConfigureAwait(false); + return await objectRepository.UpsertDocumentAsync(entity).ConfigureAwait(false); } - public async Task Put(ActivityContext context, TEntity entity) where TEntity : GraphModel + public async Task Put(ActivityContext context, TEntity entity) + where TEntity : GraphModel { var now = DateTimeOffset.Now; var model = new TrackingModel { - CorrelationId = context.ActivityId.ToString(), + CorrelationId = context.Activity.Id.ToString(), Created = now, LastUpdated = now, TypedEntity = entity, }; - _objectRepository.GenerateId(model); - return await _objectRepository.UpsertDocumentAsync(model).ConfigureAwait(false); + objectRepository.GenerateId(model); + return await objectRepository.UpsertDocumentAsync(model).ConfigureAwait(false); } /// /// Type map between enumeration value and Type /// - /// - /// - static ObjectType EntityToObjectType(Type type) + /// Type of the model corresponding to an . + /// An enumeration value of type . + public static ObjectType EntityToObjectType(Type type) { if (type == typeof(ServicePrincipalModel)) { return ObjectType.ServicePrincipal; } + throw new ApplicationException($"Unexpected tracking object type {type.Name}"); } } diff --git a/src/Automation/CSE.Automation/Services/ServicePrincipalClassifier.cs b/src/Automation/CSE.Automation/Services/ServicePrincipalClassifier.cs new file mode 100644 index 00000000..9449a529 --- /dev/null +++ b/src/Automation/CSE.Automation/Services/ServicePrincipalClassifier.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using CSE.Automation.Interfaces; +using CSE.Automation.Model; + +namespace CSE.Automation.Services +{ + internal class ServicePrincipalClassifier : IServicePrincipalClassifier + { + public async Task Classify(ServicePrincipalClassification entity) + { + return await Task.FromResult((ServicePrincipalClassification)null).ConfigureAwait(false); + } + } +} diff --git a/src/Automation/CSE.Automation/Startup.cs b/src/Automation/CSE.Automation/Startup.cs index ed61be14..54043f80 100644 --- a/src/Automation/CSE.Automation/Startup.cs +++ b/src/Automation/CSE.Automation/Startup.cs @@ -25,12 +25,12 @@ namespace CSE.Automation { public class Startup : FunctionsStartup { - private ILogger _logger; + private ILogger logger; /// /// Configure the host runtime /// - /// + /// An instance of . /// /// 1. Load configuration for host context /// 2. Register runtime services in container @@ -44,9 +44,9 @@ public override void Configure(IFunctionsHostBuilder builder) throw new ArgumentNullException(nameof(builder)); } - _logger = CreateBootstrapLogger(); - _logger.LogInformation($"Bootstrap logger initialized."); - _logger.LogDebug($"AUTH_TYPE: {Environment.GetEnvironmentVariable("AUTH_TYPE")}"); + logger = CreateBootstrapLogger(); + logger.LogInformation($"Bootstrap logger initialized."); + logger.LogDebug($"AUTH_TYPE: {Environment.GetEnvironmentVariable("AUTH_TYPE")}"); // CONFIGURATION BuildConfiguration(builder); @@ -57,15 +57,12 @@ public override void Configure(IFunctionsHostBuilder builder) RegisterServices(builder); ValidateSettings(builder); - - //ValidateServices(builder); - } /// /// Create a basic logger (low dependency) so that we can get some logs out of bootstrap /// - /// + /// An instance of . private static ILogger CreateBootstrapLogger() { var serviceProvider = new ServiceCollection() @@ -134,14 +131,14 @@ private static void RegisterSettings(IFunctionsHostBuilder builder) }) .AddSingleton(provider => provider.GetRequiredService()) - .AddSingleton(x => new AuditRespositorySettings(x.GetRequiredService()) + .AddSingleton(x => new AuditRepositorySettings(x.GetRequiredService()) { Uri = config[Constants.CosmosDBURLName], Key = config[Constants.CosmosDBKeyName], DatabaseName = config[Constants.CosmosDBDatabaseName], CollectionName = config[Constants.CosmosDBAuditCollectionName], }) - .AddSingleton(provider => provider.GetRequiredService()) + .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton(x => new ObjectTrackingRepositorySettings(x.GetRequiredService()) { @@ -152,11 +149,21 @@ private static void RegisterSettings(IFunctionsHostBuilder builder) }) .AddSingleton(provider => provider.GetRequiredService()) + .AddSingleton(x => new ActivityHistoryRepositorySettings(x.GetRequiredService()) + { + Uri = config[Constants.CosmosDBURLName], + Key = config[Constants.CosmosDBKeyName], + DatabaseName = config[Constants.CosmosDBDatabaseName], + CollectionName = config[Constants.CosmosDBActivityHistoryCollectionName], + }) + .AddSingleton(provider => provider.GetRequiredService()) + .AddSingleton(x => new ServicePrincipalProcessorSettings(x.GetRequiredService()) { QueueConnectionString = config[Constants.SPStorageConnectionString], EvaluateQueueName = config[Constants.EvaluateQueueAppSetting.Trim('%')], UpdateQueueName = config[Constants.UpdateQueueAppSetting.Trim('%')], + DiscoverQueueName = config[Constants.DiscoverQueueAppSetting.Trim('%')], ConfigurationId = config["configId"].ToGuid(Guid.Parse("02a54ac9-441e-43f1-88ee-fde420db2559")), VisibilityDelayGapSeconds = config["visibilityDelayGapSeconds"].ToInt(8), QueueRecordProcessThreshold = config["queueRecordProcessThreshold"].ToInt(10), @@ -179,17 +186,17 @@ private void ValidateSettings(IFunctionsHostBuilder builder) } catch (Azure.Identity.CredentialUnavailableException credEx) { - _logger.LogCritical(credEx, $"Failed to validate application configuration: Azure Identity is incorrect."); + logger.LogCritical(credEx, $"Failed to validate application configuration: Azure Identity is incorrect."); throw; } catch (Exception ex) { - _logger.LogCritical(ex, $"Failed to validate application configuration"); + logger.LogCritical(ex, $"Failed to validate application configuration"); throw; } } - _logger.LogInformation($"All settings classes validated."); + logger.LogInformation($"All settings classes validated."); } private static void RegisterServices(IFunctionsHostBuilder builder) @@ -202,6 +209,7 @@ private static void RegisterServices(IFunctionsHostBuilder builder) .AddSingleton() .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton, ConfigRepository>(provider => provider.GetRequiredService()) + .AddSingleton() .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton, AuditRepository>(provider => provider.GetRequiredService()) @@ -210,6 +218,10 @@ private static void RegisterServices(IFunctionsHostBuilder builder) .AddSingleton(provider => provider.GetRequiredService()) .AddSingleton, ObjectTrackingRepository>(provider => provider.GetRequiredService()) + .AddSingleton() + .AddSingleton(provider => provider.GetRequiredService()) + .AddSingleton, ActivityHistoryRepository>(provider => provider.GetRequiredService()) + .AddScoped() .AddScoped, ConfigService>() @@ -219,6 +231,7 @@ private static void RegisterServices(IFunctionsHostBuilder builder) .AddScoped() .AddScoped() + .AddScoped() .AddScoped, GraphModelValidator>() .AddScoped, ServicePrincipalModelValidator>() @@ -227,6 +240,8 @@ private static void RegisterServices(IFunctionsHostBuilder builder) .AddScoped() + .AddScoped() + .AddTransient(); } diff --git a/src/Automation/CSE.Automation/Validators/GraphModelValidator.cs b/src/Automation/CSE.Automation/Validators/GraphModelValidator.cs index 55675249..e2a320ef 100644 --- a/src/Automation/CSE.Automation/Validators/GraphModelValidator.cs +++ b/src/Automation/CSE.Automation/Validators/GraphModelValidator.cs @@ -29,8 +29,6 @@ public GraphModelValidator() .WithMessage("'Created', 'Deleted', 'LastUpdated'"); RuleFor(m => m.ObjectType) .IsInEnum(); - RuleFor(m => m.Status) - .IsInEnum(); } protected static bool BeValidModelDateSequence(GraphModel model) diff --git a/src/Automation/CSE.Automation/Validators/ServicePrincipalModelValidator.cs b/src/Automation/CSE.Automation/Validators/ServicePrincipalModelValidator.cs index 0433cc7f..b2368c3d 100644 --- a/src/Automation/CSE.Automation/Validators/ServicePrincipalModelValidator.cs +++ b/src/Automation/CSE.Automation/Validators/ServicePrincipalModelValidator.cs @@ -9,20 +9,12 @@ namespace CSE.Automation.Validators { - public class ServicePrincipalModelValidator : AbstractValidator, IModelValidator + internal class ServicePrincipalModelValidator : AbstractValidator, IModelValidator { public ServicePrincipalModelValidator(IGraphHelper graphHelper) { Include(new GraphModelValidator()); - RuleFor(m => m.AppId) - .NotEmpty() - .MaximumLength(Constants.MaxStringLength); - //RuleFor(m => m.AppDisplayName) - // .NotEmpty() - // .MaximumLength(Constants.MaxStringLength); - //RuleFor(m => m.DisplayName) - // .NotEmpty() - // .MaximumLength(Constants.MaxStringLength); + RuleFor(m => m.Notes) .NotEmpty() .HasOnlyEmailAddresses() @@ -30,7 +22,7 @@ public ServicePrincipalModelValidator(IGraphHelper graphHelper) { field?.Split(',', ';').ToList().ForEach(token => { - if (graphHelper.GetGraphObject(token).Result is null) + if (graphHelper.GetGraphObjectWithOwners(token).Result is null) { context.AddFailure($"'{token}' is not a valid UserPrincipalName in this directory"); } diff --git a/src/Automation/CSE.Automation/appsettings.Development.json b/src/Automation/CSE.Automation/appsettings.Development.json index d6e25e55..d8aca96f 100644 --- a/src/Automation/CSE.Automation/appsettings.Development.json +++ b/src/Automation/CSE.Automation/appsettings.Development.json @@ -8,13 +8,10 @@ "SPObjectTrackingCollection": "ObjectTracking", "SPAuditCollection": "Audit", - "SPEvaluateQueue": "evaluate", - "SPUpdateQueue": "update", - "configId": "02a54ac9-441e-43f1-88ee-fde420db2559", "visibilityDelayGapSeconds": "8", "queueRecordProcessThreshold": "300", - "SPDeltaDiscoverySchedule": "0 */30 * * * *", - "aadUpdateMode": "ReportOnly" + "SPDeltaDiscoverySchedule": "0 */30 * * * *" + } diff --git a/src/Automation/CSE.Automation/host.json b/src/Automation/CSE.Automation/host.json index 0ca79db8..2df65f0c 100644 --- a/src/Automation/CSE.Automation/host.json +++ b/src/Automation/CSE.Automation/host.json @@ -1,6 +1,6 @@ { "version": "2.0", - "functionTimeout": "01:00:00", + "functionTimeout": "01:30:00", "extensions": { "queues": { "maxPollingInterval": "00:00:15", diff --git a/src/Automation/SolutionScripts/SolutionCommands.ps1 b/src/Automation/SolutionScripts/SolutionCommands.ps1 index 13c87f14..b4b2180f 100644 --- a/src/Automation/SolutionScripts/SolutionCommands.ps1 +++ b/src/Automation/SolutionScripts/SolutionCommands.ps1 @@ -54,7 +54,7 @@ function global:ProvisionLocalResources() } function global:ProvisionStorageResources(){ - $queues = "evaluate"#, "update" + $queues = "discover", "evaluate", "update" # Creates queues $queues | % { @@ -82,6 +82,7 @@ function global:ProvisionCosmosResources(){ "Configuration"= "/configType"; "Audit" = "/auditYearMonth"; "ObjectTracking" = "/objectType"; + "ActivityHistory" = "/correlationId"; } $databases = GetCosmosDatabases