Updates to feature/egressExtension Part 1

documentation/schema.json

@@ -965,17 +965,6 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
-        "AzureBlobStorage": {
-          "type": [
-            "null",
-            "object"
-          ],
-          "description": "Mapping of Azure blob storage egress provider names to their options.",
-          "default": {},
-          "additionalProperties": {
-            "$ref": "#/definitions/AzureBlobEgressProviderOptions"
-          }
-        },
         "FileSystem": {
           "type": [
             "null",
@@ -1000,116 +989,6 @@
         }
       }
     },
-    "AzureBlobEgressProviderOptions": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "AccountUri",
-        "ContainerName"
-      ],
-      "properties": {
-        "AccountUri": {
-          "type": "string",
-          "description": "The URI of the Azure blob storage account.",
-          "format": "uri",
-          "minLength": 1
-        },
-        "AccountKey": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The account key used to access the Azure blob storage account."
-        },
-        "AccountKeyName": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The name of the account key used to look up the value from the Egress options Properties map."
-        },
-        "SharedAccessSignature": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The shared access signature (SAS) used to access the Azure blob and optionally queue storage accounts."
-        },
-        "SharedAccessSignatureName": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The name of the shared access signature (SAS) used to look up the value from the Egress options Properties map."
-        },
-        "ManagedIdentityClientId": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "Client id of the Managed Identity used for authentication. The identity must have permissions to create containers and write to blob storage."
-        },
-        "ContainerName": {
-          "type": "string",
-          "description": "The name of the container to which the blob will be egressed. If egressing to the root container, use the \"$root\" sentinel value.",
-          "minLength": 1
-        },
-        "BlobPrefix": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The prefix to prepend to the blob name."
-        },
-        "CopyBufferSize": {
-          "type": [
-            "integer",
-            "null"
-          ],
-          "description": "Buffer size used when copying data from an egress callback returning a stream to the egress callback that is provided a stream to which data is written.",
-          "format": "int32"
-        },
-        "QueueName": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The name of the queue to which a message will be dispatched upon writing to a blob."
-        },
-        "QueueAccountUri": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The URI of the Azure queue storage account.",
-          "format": "uri"
-        },
-        "QueueSharedAccessSignature": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The shared access signature (SAS) used to access the Azure queue storage account."
-        },
-        "QueueSharedAccessSignatureName": {
-          "type": [
-            "null",
-            "string"
-          ],
-          "description": "The name of the queue shared access signature (SAS) used to look up the value from the Egress options Properties map."
-        },
-        "Metadata": {
-          "type": [
-            "null",
-            "object"
-          ],
-          "description": "A mapping of metadata keys to environment variable names. The values of the environment variables will be added as metadata for egressed artifacts.",
-          "additionalProperties": {
-            "type": "string"
-          }
-        }
-      }
-    },
     "FileSystemEgressProviderOptions": {
       "type": "object",
       "additionalProperties": false,

src/Extensions/AzureBlobStorage.sln

@@ -0,0 +1,25 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.3.32819.101
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureBlobStorage", "AzureBlobStorage\AzureBlobStorage.csproj", "{5ED61A7B-F0AA-45F2-9E9A-8972FF7F7278}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{5ED61A7B-F0AA-45F2-9E9A-8972FF7F7278}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5ED61A7B-F0AA-45F2-9E9A-8972FF7F7278}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5ED61A7B-F0AA-45F2-9E9A-8972FF7F7278}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5ED61A7B-F0AA-45F2-9E9A-8972FF7F7278}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {6679B669-CE81-4761-8452-D5218551CE0F}
+	EndGlobalSection
+EndGlobal

src/Tools/dotnet-monitor/Egress/AzureBlob/AzureBlobEgressProvider.cs → src/Extensions/AzureBlobStorage/AzureBlob/AzureBlobEgressProvider.cs

@@ -9,36 +9,28 @@
 using Azure.Storage.Blobs.Models;
 using Azure.Storage.Blobs.Specialized;
 using Azure.Storage.Queues;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
 using System.Globalization;
-using System.IO;
 using System.Net;
-using System.Threading;
-using System.Threading.Tasks;
 
-namespace Microsoft.Diagnostics.Tools.Monitor.Egress.AzureBlob
+namespace Microsoft.Diagnostics.Monitoring.AzureStorage.AzureBlob
 {
     /// <summary>
     /// Egress provider for egressing stream data to an Azure blob storage account.
     /// </summary>
     /// <remarks>
     /// Blobs created through this provider will overwrite existing blobs if they have the same blob name.
     /// </remarks>
-    internal partial class AzureBlobEgressProvider :
-        EgressProvider<AzureBlobEgressProviderOptions>
+    internal partial class AzureBlobEgressProvider
     {
-        private int BlobStorageBufferSize = 4 * 1024 * 1024;
+        private readonly string AzureBlobStorage = "AzureBlobStorage";
+        protected ILogger Logger { get; }
 
-        public AzureBlobEgressProvider(ILogger<AzureBlobEgressProvider> logger)
-            : base(logger)
+        public AzureBlobEgressProvider(ILogger logger)
         {
+            Logger = logger;
         }
 
-        public override async Task<string> EgressAsync(
-            string providerType,
-            string providerName,
+        public async Task<string> EgressAsync(
             AzureBlobEgressProviderOptions options,
             Func<CancellationToken, Task<Stream>> action,
             EgressArtifactSettings artifactSettings,
@@ -54,7 +46,7 @@ public override async Task<string> EgressAsync(
 
                 BlobClient blobClient = containerClient.GetBlobClient(blobName);
 
-                Logger?.EgressProviderInvokeStreamAction(EgressProviderTypes.AzureBlobStorage);
+                Logger.EgressProviderInvokeStreamAction(AzureBlobStorage);
                 using var stream = await action(token);
 
                 // Write blob content, headers, and metadata
@@ -63,75 +55,7 @@ public override async Task<string> EgressAsync(
                 await SetBlobClientMetadata(blobClient, artifactSettings, token);
 
                 string blobUriString = GetBlobUri(blobClient);
-                Logger?.EgressProviderSavedStream(EgressProviderTypes.AzureBlobStorage, blobUriString);
-
-                if (CheckQueueEgressOptions(options))
-                {
-                    await EgressMessageToQueue(blobName, options, token);
-                }
-
-                return blobUriString;
-            }
-            catch (AggregateException ex) when (ex.InnerException is RequestFailedException innerException)
-            {
-                throw CreateException(innerException);
-            }
-            catch (RequestFailedException ex)
-            {
-                throw CreateException(ex);
-            }
-            catch (CredentialUnavailableException ex)
-            {
-                throw CreateException(ex);
-            }
-        }
-
-        public override async Task<string> EgressAsync(
-            string providerType,
-            string providerName,
-            AzureBlobEgressProviderOptions options,
-            Func<Stream, CancellationToken, Task> action,
-            EgressArtifactSettings artifactSettings,
-            CancellationToken token)
-        {
-            try
-            {
-                AddConfiguredMetadataAsync(options, artifactSettings);
-
-                var containerClient = await GetBlobContainerClientAsync(options, token);
-
-                string blobName = GetBlobName(options, artifactSettings);
-
-                BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName);
-
-                // Write blob content
-
-                var bloboptions = new BlockBlobOpenWriteOptions
-                {
-                    BufferSize = BlobStorageBufferSize,
-                };
-                using (Stream blobStream = await blobClient.OpenWriteAsync(overwrite: true, options: bloboptions, cancellationToken: token))
-                using (AutoFlushStream flushStream = new AutoFlushStream(blobStream, BlobStorageBufferSize))
-                {
-                    //Azure's stream from OpenWriteAsync will do the following
-                    //1. Write the data to a local buffer
-                    //2. Once that buffer is full, stage the data remotely (this data is not considered valid yet)
-                    //3. After 4Gi of data has been staged, the data will be commited. This can be forced earlier by flushing
-                    //the stream.
-                    // Since we want the data to be readily available, we automatically flush (and therefore commit) every time we fill up the buffer.
-                    Logger?.EgressProviderInvokeStreamAction(EgressProviderTypes.AzureBlobStorage);
-                    await action(flushStream, token);
-
-                    await flushStream.FlushAsync(token);
-                }
-
-                // Write blob headers
-                await blobClient.SetHttpHeadersAsync(CreateHttpHeaders(artifactSettings), cancellationToken: token);
-
-                await SetBlobClientMetadata(blobClient, artifactSettings, token);
-
-                string blobUriString = GetBlobUri(blobClient);
-                Logger?.EgressProviderSavedStream(EgressProviderTypes.AzureBlobStorage, blobUriString);
+                Logger.EgressProviderSavedStream(AzureBlobStorage, blobUriString);
 
                 if (CheckQueueEgressOptions(options))
                 {
@@ -395,7 +319,6 @@ private string GetBlobName(AzureBlobEgressProviderOptions options, EgressArtifac
         private BlobHttpHeaders CreateHttpHeaders(EgressArtifactSettings artifactSettings)
         {
             BlobHttpHeaders headers = new BlobHttpHeaders();
-            headers.ContentEncoding = artifactSettings.ContentEncoding;
             headers.ContentType = artifactSettings.ContentType;
             return headers;
         }

src/Microsoft.Diagnostics.Monitoring.Options/AzureBlobEgressProviderOptions.Validate.cs → src/Extensions/AzureBlobStorage/AzureBlobEgressProviderOptions.Validate.cs

@@ -2,11 +2,9 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using Microsoft.Diagnostics.Monitoring.WebApi;
-using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
 
-namespace Microsoft.Diagnostics.Tools.Monitor.Egress.AzureBlob
+namespace Microsoft.Diagnostics.Monitoring.AzureStorage
 {
     internal sealed partial class AzureBlobEgressProviderOptions :
         IValidatableObject
@@ -25,7 +23,7 @@ IEnumerable<ValidationResult> IValidatableObject.Validate(ValidationContext vali
                 results.Add(
                     new ValidationResult(
                         string.Format(
-                            OptionsDisplayStrings.ErrorMessage_CredentialsMissing,
+                            Strings.ErrorMessage_CredentialsMissing,
                             nameof(AccountKey),
                             nameof(SharedAccessSignature),
                             nameof(ManagedIdentityClientId))));

src/Extensions/AzureBlobStorage/AzureBlobEgressProviderOptions.cs

@@ -0,0 +1,47 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+
+namespace Microsoft.Diagnostics.Monitoring.AzureStorage
+{
+    /// <summary>
+    /// Egress provider options for Azure blob storage.
+    /// </summary>
+    internal sealed partial class AzureBlobEgressProviderOptions
+    {
+        [Required]
+        public Uri AccountUri { get; set; }
+
+        public string AccountKey { get; set; }
+
+        public string AccountKeyName { get; set; }
+
+        public string SharedAccessSignature { get; set; }
+
+        public string SharedAccessSignatureName { get; set; }
+
+        public string ManagedIdentityClientId { get; set; }
+
+        [Required]
+        public string ContainerName { get; set; }
+
+        public string BlobPrefix { get; set; }
+
+        public int? CopyBufferSize { get; set; }
+
+        public string QueueName { get; set; }
+
+        public Uri QueueAccountUri { get; set; }
+
+        public string QueueSharedAccessSignature { get; set; }
+
+        public string QueueSharedAccessSignatureName { get; set; }
+
+        public IDictionary<string, string> Metadata { get; set; }
+            = new Dictionary<string, string>(0);
+    }
+}