Merge pull request #240 from microsoft/dev

many minor changes +resetdemo

src/TeamCloud.Audit/CommandAuditWriter.cs

@@ -3,14 +3,14 @@
  *  Licensed under the MIT License.
  */
 
+using Microsoft.Azure.Cosmos.Table;
+using Microsoft.Azure.Storage.Blob;
+using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
-using Microsoft.Azure.Storage.Blob;
-using Microsoft.Azure.Cosmos.Table;
-using Newtonsoft.Json;
 using TeamCloud.Audit.Model;
 using TeamCloud.Model.Commands.Core;
 using TeamCloud.Orchestration;
@@ -69,10 +69,10 @@ public CommandAuditWriter(ICommandAuditOptions options = null)
                 .CreateCloudTableClient().GetTableReference(GetAuditTableName(options)));
         }
 
-        public Task AuditAsync(ICommand command, ICommandResult commandResult = default, string providerId = default) => Task.WhenAll
+        public Task AuditAsync(ICommand command, ICommandResult commandResult = default) => Task.WhenAll
         (
-            WriteContainerAsync(command, commandResult, providerId),
-            WriteTableAsync(command, commandResult, providerId)
+            WriteContainerAsync(command, commandResult),
+            WriteTableAsync(command, commandResult)
         );
 
         private async Task<CloudBlobContainer> GetAuditContainerAsync()
@@ -87,7 +87,7 @@ private async Task<CloudBlobContainer> GetAuditContainerAsync()
             return auditContainer.Value;
         }
 
-        private Task WriteContainerAsync(ICommand command, ICommandResult commandResult, string providerId)
+        private Task WriteContainerAsync(ICommand command, ICommandResult commandResult)
         {
             return Task.WhenAll
             (
@@ -97,7 +97,7 @@ private Task WriteContainerAsync(ICommand command, ICommandResult commandResult,
 
             async Task WriteBlobAsync(object data)
             {
-                var auditPath = $"{command.ProjectId}/{command.CommandId}/{providerId}/{data.GetType().Name}.json";
+                var auditPath = $"{command.OrganizationId}/{command.ProjectId}/{command.CommandId}/{data.GetType().Name}.json";
 
                 var auditContainer = await GetAuditContainerAsync().ConfigureAwait(false);
                 var auditBlob = auditContainer.GetBlockBlobReference(auditPath.Replace("//", "/", StringComparison.OrdinalIgnoreCase));
@@ -120,9 +120,9 @@ private async Task<CloudTable> GetAuditTableAsync()
             return auditTable.Value;
         }
 
-        private async Task WriteTableAsync(ICommand command, ICommandResult commandResult, string providerId)
+        private async Task WriteTableAsync(ICommand command, ICommandResult commandResult)
         {
-            var entity = new CommandAuditEntity(command, providerId);
+            var entity = new CommandAuditEntity(command);
 
             var auditTable = await GetAuditTableAsync()
                 .ConfigureAwait(false);
@@ -144,9 +144,6 @@ private async Task WriteTableAsync(ICommand command, ICommandResult commandResul
 
                 if (entity.RuntimeStatus.IsFinal())
                     entity.Errors = string.Join(Environment.NewLine, commandResult.Errors.Select(error => $"[{error.Severity}] {error.Message}"));
-
-                if (!string.IsNullOrEmpty(entity.Provider))
-                    entity.Timeout ??= entity.Created?.Add(commandResult.Timeout);
             }
 
             await auditTable

src/TeamCloud.Audit/ICommandAuditWriter.cs

@@ -10,6 +10,6 @@ namespace TeamCloud.Audit
 {
     public interface ICommandAuditWriter
     {
-        Task AuditAsync(ICommand command, ICommandResult commandResult = default, string providerId = default);
+        Task AuditAsync(ICommand command, ICommandResult commandResult = default);
     }
 }

src/TeamCloud.Audit/Model/CommandAuditEntity.cs

@@ -3,33 +3,33 @@
  *  Licensed under the MIT License.
  */
 
+using Microsoft.Azure.Cosmos.Table;
 using System;
 using System.ComponentModel.DataAnnotations.Schema;
-using Microsoft.Azure.Cosmos.Table;
 using TeamCloud.Model.Commands;
 using TeamCloud.Model.Commands.Core;
 
 namespace TeamCloud.Audit.Model
 {
     public sealed class CommandAuditEntity : TableEntityBase
     {
-        private static readonly string NoneProjectPartitionKey = Guid.Empty.ToString();
+        private static readonly string EmptyKey = Guid.Empty.ToString();
 
         public CommandAuditEntity()
         { }
 
-        public CommandAuditEntity(ICommand command, string providerId = null) : this()
+        public CommandAuditEntity(ICommand command) : this()
         {
             if (command is null)
                 throw new ArgumentNullException(nameof(command));
 
-            PartitionKey = command.ProjectId ?? NoneProjectPartitionKey;
-            RowKey = $"{command.CommandId}@{providerId}".TrimEnd('@');
+            PartitionKey = $"{command.OrganizationId ?? EmptyKey}|{command.ProjectId ?? EmptyKey}";
+            RowKey = command.CommandId.ToString();
 
             CommandId = command.CommandId.ToString();
             Command = command.GetType().Name;
-            // Event = (command as ProviderEventCommand)?.Payload?.EventType ?? string.Empty;
-            Provider = providerId ?? string.Empty;
+
+            ComponentTask = (command as ComponentTaskRunCommand)?.Payload?.TypeName ?? (command as ComponentTaskRunCommand)?.Payload?.Type.ToString() ?? string.Empty;
         }
 
         [IgnoreProperty]
@@ -42,23 +42,20 @@ public CommandAuditEntity(ICommand command, string providerId = null) : this()
         public string CommandId { get; private set; }
         [Column(Order = 102)]
         public string Command { get; private set; }
-        [Column(Order = 103)]
-        // public string Event { get; private set; }
-        // [Column(Order = 104)]
-        public string Provider { get; private set; }
 
         [Column(Order = 201)]
+        public string ComponentTask { get; private set; }
+
+        [Column(Order = 301)]
         public CommandRuntimeStatus RuntimeStatus { get; set; } = CommandRuntimeStatus.Unknown;
-        [Column(Order = 202)]
+        [Column(Order = 302)]
         public string CustomStatus { get; set; }
-        [Column(Order = 203)]
+        [Column(Order = 303)]
         public string Errors { get; set; }
 
-        [Column(Order = 301)]
+        [Column(Order = 901)]
         public DateTime? Created { get; set; }
-        [Column(Order = 302)]
+        [Column(Order = 902)]
         public DateTime? Updated { get; set; }
-        [Column(Order = 303)]
-        public DateTime? Timeout { get; set; }
     }
 }

src/TeamCloud.Model/Commands/ComponentCreateCommand.cs

@@ -11,6 +11,6 @@ namespace TeamCloud.Model.Commands
     public sealed class ComponentCreateCommand : CreateCommand<Component, ComponentCreateCommandResult>
     {
         public ComponentCreateCommand(User user, Component payload) : base(user, payload)
-            => ProjectId = payload?.ProjectId ?? throw new System.ArgumentNullException(nameof(payload));
+        { }
     }
 }

src/TeamCloud.Model/Commands/ComponentDeleteCommand.cs

@@ -11,6 +11,6 @@ namespace TeamCloud.Model.Commands
     public sealed class ComponentDeleteCommand : DeleteCommand<Component, ComponentDeleteCommandResult>
     {
         public ComponentDeleteCommand(User user, Component payload) : base(user, payload)
-            => ProjectId = payload?.ProjectId ?? throw new System.ArgumentNullException(nameof(payload));
+        { }
     }
 }

src/TeamCloud.Model/Commands/ComponentTaskCommand.cs

@@ -11,6 +11,6 @@ namespace TeamCloud.Model.Commands
     public sealed class ComponentTaskCommand : CustomCommand<ComponentTask, ComponentTaskCommandResult>
     {
         public ComponentTaskCommand(User user, ComponentTask payload) : base(user, payload)
-            => ProjectId = payload?.ProjectId ?? throw new System.ArgumentNullException(nameof(payload));
+        { }
     }
 }

src/TeamCloud.Model/Commands/ComponentTaskRunCommand.cs

@@ -11,6 +11,7 @@ namespace TeamCloud.Model.Commands
     public sealed class ComponentTaskRunCommand : CustomCommand<ComponentTask, ComponentTaskRunCommandResult>
     {
         public ComponentTaskRunCommand(User user, ComponentTask payload) : base(user, payload)
-            => ProjectId = payload?.ProjectId ?? throw new System.ArgumentNullException(nameof(payload));
+        { }
+
     }
 }

src/TeamCloud.Model/Commands/ComponentUpdateCommand.cs

@@ -11,6 +11,7 @@ namespace TeamCloud.Model.Commands
     public sealed class ComponentUpdateCommand : UpdateCommand<Component, ComponentUpdateCommandResult>
     {
         public ComponentUpdateCommand(User user, Component payload) : base(user, payload)
-            => ProjectId = payload?.ProjectId ?? throw new System.ArgumentNullException(nameof(payload));
+        { }
+
     }
 }

src/TeamCloud.Model/Commands/Core/Command.cs

@@ -15,28 +15,30 @@ public abstract class Command<TPayload, TCommandResult> : ICommand<TPayload, TCo
     {
         protected Command(CommandAction action, User user, TPayload payload = default, Guid? commandId = default)
         {
+            CommandId = commandId.GetValueOrDefault(Guid.NewGuid());
             CommandAction = action;
             User = user ?? throw new ArgumentNullException(nameof(user));
             Payload = payload;
-            CommandId = commandId.GetValueOrDefault(Guid.NewGuid());
-
-            if (payload is IProjectContext child)
-                OrganizationId = child.Organization;
         }
 
         public Guid CommandId { get; private set; }
 
+        public CommandAction CommandAction { get; private set; }
 
-        public string OrganizationId { get; private set; }
+        private string organizationId;
 
-        public CommandAction CommandAction { get; private set; }
+        public string OrganizationId
+        {
+            get => organizationId ?? (Payload as IOrganizationContext)?.Organization;
+            set => organizationId = value;
+        }
 
         private string projectId;
 
         public virtual string ProjectId
         {
-            get => Payload is Project project && !string.IsNullOrEmpty(project.Id) ? project.Id : projectId;
-            protected set => projectId = value;
+            get => projectId ?? (Payload as IProjectContext)?.ProjectId;
+            set => projectId = value;
         }
 
         public User User { get; set; }

src/TeamCloud.Orchestrator/API/CommandTrigger.cs

@@ -3,10 +3,6 @@
  *  Licensed under the MIT License.
  */
 
-using System;
-using System.Linq;
-using System.Net.Http;
-using System.Threading.Tasks;
 using FluentValidation;
 using Flurl;
 using Microsoft.AspNetCore.Http;
@@ -19,12 +15,17 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
+using System;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
 using TeamCloud.Audit;
 using TeamCloud.Http;
 using TeamCloud.Model.Commands.Core;
 using TeamCloud.Model.Validation;
 using TeamCloud.Orchestrator.Handlers;
 using TeamCloud.Orchestrator.Operations.Orchestrations.Utilities;
+using TeamCloud.Serialization;
 
 namespace TeamCloud.Orchestrator.API
 {
@@ -114,12 +115,21 @@ public async Task Dequeue(
             if (log is null)
                 throw new ArgumentNullException(nameof(log));
 
-            var command = JsonConvert.DeserializeObject<ICommand>(commandMessage.AsString);
+            try
+            {
+                var command = JsonConvert.DeserializeObject<ICommand>(commandMessage.AsString, TeamCloudSerializerSettings.Default);
 
-            command.Validate(throwOnValidationError: true);
+                command.Validate(throwOnValidationError: true);
 
-            _ = await ProcessCommandAsync(durableClient, command, commandProcessor, commandMonitor)
-                .ConfigureAwait(false);
+                _ = await ProcessCommandAsync(durableClient, command, commandProcessor, commandMonitor)
+                    .ConfigureAwait(false);
+            }
+            catch (Exception exc)
+            {
+                log.LogError(exc, $"Failed to process queued command: {exc.Message}");
+
+                throw;
+            }
         }
 
         [FunctionName(nameof(CommandTrigger) + nameof(Monitor))]
@@ -138,42 +148,47 @@ public async Task Monitor(
             if (durableClient is null)
                 throw new ArgumentNullException(nameof(durableClient));
 
-            // there is no error handler on purpose - this way we can leverage
-            // the function runtime capabilities for poisened message queues
-            // and don't need to handle this on our own.
-
             if (Guid.TryParse(commandMessage.AsString, out var commandId))
             {
-                var command = await durableClient
-                    .GetCommandAsync(commandId)
-                    .ConfigureAwait(false);
-
-                if (command is null)
-                {
-                    // we could find a command based on the enqueued command id - warn and forget
-
-                    log.LogWarning($"Monitoring command failed: Could not find command {commandId}");
-                }
-                else
+                try
                 {
-                    var commandResult = await durableClient
-                        .GetCommandResultAsync(commandId)
+                    var command = await durableClient
+                        .GetCommandAsync(commandId)
                         .ConfigureAwait(false);
 
-                    await commandAuditWriter
-                        .AuditAsync(command, commandResult)
-                        .ConfigureAwait(false);
+                    if (command is null)
+                    {
+                        // we could find a command based on the enqueued command id - warn and forget
 
-                    if (!(commandResult?.RuntimeStatus.IsFinal() ?? false))
+                        log.LogWarning($"Monitoring command failed: Could not find command {commandId}");
+                    }
+                    else
                     {
-                        // the command result is still not in a final state - as we want to monitor the command until it is done,
-                        // we are going to re-enqueue the command ID with a visibility offset to delay the next result lookup.
+                        var commandResult = await durableClient
+                            .GetCommandResultAsync(commandId)
+                            .ConfigureAwait(false);
 
-                        await commandMonitor
-                            .AddMessageAsync(new CloudQueueMessage(commandId.ToString()), null, TimeSpan.FromSeconds(10), null, null)
+                        await commandAuditWriter
+                            .AuditAsync(command, commandResult)
                             .ConfigureAwait(false);
+
+                        if (!(commandResult?.RuntimeStatus.IsFinal() ?? false))
+                        {
+                            // the command result is still not in a final state - as we want to monitor the command until it is done,
+                            // we are going to re-enqueue the command ID with a visibility offset to delay the next result lookup.
+
+                            await commandMonitor
+                                .AddMessageAsync(new CloudQueueMessage(commandId.ToString()), null, TimeSpan.FromSeconds(10), null, null)
+                                .ConfigureAwait(false);
+                        }
                     }
                 }
+                catch (Exception exc)
+                {
+                    log.LogError(exc, $"Monitoring command failed: {exc.Message}");
+
+                    throw;
+                }
             }
             else
             {

src/TeamCloud.Orchestrator/Operations/Activities/CommandAuditActivity.cs

@@ -3,14 +3,13 @@
  *  Licensed under the MIT License.
  */
 
-using System;
-using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs;
 using Microsoft.Azure.WebJobs.Extensions.DurableTask;
 using Microsoft.Extensions.Logging;
+using System;
+using System.Threading.Tasks;
 using TeamCloud.Audit;
 using TeamCloud.Model.Commands.Core;
-using TeamCloud.Model.Data;
 using TeamCloud.Orchestration;
 
 namespace TeamCloud.Orchestrator.Operations.Activities
@@ -40,7 +39,7 @@ public async Task RunActivity(
                 var functionInput = activityContext.GetInput<Input>();
 
                 await commandAuditWriter
-                    .AuditAsync(functionInput.Command, functionInput.CommandResult, functionInput.ProviderId)
+                    .AuditAsync(functionInput.Command, functionInput.CommandResult)
                     .ConfigureAwait(false);
             }
             catch (Exception exc)
@@ -54,8 +53,6 @@ internal struct Input
             public ICommand Command { get; set; }
 
             public ICommandResult CommandResult { get; set; }
-
-            public string ProviderId { get; set; }
         }
     }