[ASCN-375] Setup Serilog and Kubernetes console logging

cnab/Invoke-CNAB.ps1

@@ -7,7 +7,7 @@ param (
     $RunAsContainer = $false,
     [ValidateSet("GCP", "DockerDesktop")]
     [string]
-    $Target = "GCP",
+    $Target = "DockerDesktop",
     [bool]
     $DownloadLocalScriptsForLocalDebugging = $true
 )

src/Opserver.Web/Opserver.Web.csproj

@@ -10,6 +10,14 @@
     <PackageReference Include="BuildWebCompiler" Condition="'$(OS)' == 'Windows_NT'" Version="1.12.405" PrivateAssets="all" />
     <PackageReference Include="MiniProfiler.AspNetCore.Mvc" Version="4.2.22" />
     <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="6.15.0" />
+    <PackageReference Include="Serilog" Version="4.1.0" />
+    <PackageReference Include="Serilog.AspNetCore" Version="8.0.3" />
+    <PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.1" />
+    <PackageReference Include="Serilog.Enrichers.Sensitive" Version="1.7.3" />
+    <PackageReference Include="Serilog.Enrichers.Thread" Version="4.0.0" />
+    <PackageReference Include="Serilog.Exceptions" Version="8.4.0" />
+    <PackageReference Include="Serilog.Expressions" Version="5.0.0" />
+    <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
     <PackageReference Include="StackExchange.Exceptional.AspNetCore" Version="2.2.32" />
     <PackageReference Include="System.DirectoryServices.AccountManagement" Version="6.0.0" />
     <Reference Include="System.Management" />

src/Opserver.Web/Program.cs

@@ -7,6 +7,9 @@
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Opserver.Helpers;
+using Serilog;
+using Serilog.Enrichers.Sensitive;
+using Serilog.Exceptions;
 namespace Opserver
 {
     public static class Program
@@ -40,14 +43,54 @@ private static async Task<int> Main(string[] args)
                                 .AddEnvironmentVariables();
                         }
                     )
-                    .ConfigureLogging(
-                        (hostingContext, config) =>
+                   .ConfigureLogging(
+                    (hostingContext, config) =>
+                    {
+                        config.ClearProviders();
+
+                        if (Debugger.IsAttached)
                         {
-                            var loggingConfig = hostingContext.Configuration.GetSection("Logging");
-                            config.AddConfiguration(loggingConfig)
-                                  .AddConsole();
+                            // To increase the visibility of internal Serilog errors we will log
+                            // them to the debug output when the application is run via a debugger.
+                            Serilog.Debugging.SelfLog.Enable(message => Debug.WriteLine(message));
                         }
-                    )
+                        else
+                        {
+                            // If not running via a debugger, we'll default to the console error log (stderr)
+                            // because we don't really have a better place and for Kubernetes style deployments
+                            // this actually _is_ the right place to go.
+                            Serilog.Debugging.SelfLog.Enable(Console.Error);
+                        }
+
+                        var dataDogServiceName = Environment.GetEnvironmentVariable("DD_SERVICE") ?? "opserver";
+                        var product = hostingContext.Configuration.GetValue<string>("Product");
+                        var tier = hostingContext.Configuration.GetValue<string>("Tier");
+
+                        var loggerconfig = new LoggerConfiguration()
+                            .ReadFrom.Configuration(hostingContext.Configuration);
+
+                        loggerconfig
+                                .Enrich.FromLogContext()
+                                .Enrich.WithEnvironmentUserName()
+                                .Enrich.WithEnvironmentName()
+                                .Enrich.WithProperty("dd_service", dataDogServiceName)
+                                .Enrich.WithMachineName()
+                                .Enrich.WithThreadId()
+                                .Enrich.WithExceptionDetails()
+                                .Enrich.WithSensitiveDataMasking(options =>
+                                {
+                                    options.MaskingOperators = new List<IMaskingOperator>
+                                    {
+                                        new EmailAddressMaskingOperator()
+                                    };
+                                });
+
+                        loggerconfig.Filter.ByExcluding("StartsWith(SourceContext, 'StackExchange.Exceptional.')");
+                        loggerconfig.Filter.ByExcluding("SourceContext='Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware' and @mt='An unhandled exception has occurred while executing the request.'");
+
+                        config.AddSerilog();
+                    }
+                )
                     .UseStartup<Startup>()
                     .Build();
 

src/Opserver.Web/Startup.cs

@@ -1,6 +1,7 @@
-using System;
+using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Net;
 using System.Text.RegularExpressions;
@@ -20,6 +21,8 @@
 using Opserver.Data;
 using Opserver.Helpers;
 using Opserver.Security;
+using Serilog;
+using StackExchange.Exceptional;
 using StackExchange.Profiling;
 
 namespace Opserver
@@ -65,6 +68,7 @@ public void ConfigureServices(IServiceCollection services)
                     {
                         settings.UseExceptionalPageOnThrow = true;
                         settings.DataIncludeRegex = new Regex("^(Redis|Elastic|ErrorLog|Jil)", RegexOptions.Singleline | RegexOptions.Compiled);
+                        settings.OnAfterLog += ExceptionalAfterLog;
                         settings.GetCustomData = (ex, data) =>
                         {
                             // everything below needs a context
@@ -204,6 +208,17 @@ IEnumerable<StatusModule> modules
                               }
                           }
                       })
+                      .UseExceptionHandler(errorApp =>
+                      {
+                          errorApp.Run(ctx =>
+                          {
+                              var logger = ctx.RequestServices.GetRequiredService<ILogger<Startup>>();
+                              var exception = ctx.Features.Get<IExceptionHandlerFeature>().Error;
+                              logger.LogError(exception, exception.Message);
+
+                              return Task.CompletedTask;
+                          });
+                      })
                       .UseExceptional()
                       .UseRouting()
                       .UseMiniProfiler()
@@ -240,9 +255,17 @@ IEnumerable<StatusModule> modules
                                 AllowCachingResponses = false,
                                 Predicate =
                                     _ => false // We don't use any healthchecks for liveliness, just that the app responds
-                     });
+                            });
             NavTab.ConfigureAll(modules); // TODO: UseNavTabs() or something
             Cache.Configure(settings);
         }
+
+        private void ExceptionalAfterLog(object sender, ErrorAfterLogEventArgs args)
+        {
+            if (args != null)
+            {
+                Serilog.Log.Logger.Error(args.Error.Exception, "{ErrorMessage}", args.Error.Message);
+            }
+        }
     }
 }

src/Opserver.Web/appsettings.json

@@ -0,0 +1,30 @@
+{
+  "Serilog": {
+    "Using": [ "Serilog.Sinks.Console" ],
+    "Properties": {
+      "Application": "OpServer"
+    },
+    "MinimumLevel": {
+      "Default": "Information",
+      "Override": {
+        "Microsoft": "Warning",
+        "System": "Warning"
+      }
+    },
+    "WriteTo": [
+      {
+        "Name": "Async",
+        "Args": {
+          "configure": [
+            {
+              "Name": "Console",
+              "Args": {
+                "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact"
+              }
+            }
+          ]
+        }
+      }
+    ]
+  }
+}

src/Opserver.Web/opserverSettings.json

@@ -1,17 +1,22 @@
 {
   "ForwardedHeaders": {
-    "KnownNetworks": ["10.0.0.0/16"],
+    "KnownNetworks": [ "10.0.0.0/16" ],
     "KnownProxies": [],
     "ForwardedHeaders": "All"
   },
+  "security": {
+    "adminGroups": "",
+    "viewGroups": "",
+    "provider": "EveryonesAnAdmin"
+  },
   "Modules": {
     /* Configuration for the SQL Server dashboard */
     "Sql": {
-      //"defaultConnectionString": "Data Source=$ServerName$;Initial Catalog=master;Integrated Security=SSPI;",
-      //"refreshIntervalSeconds": 30,
-      //"instances": [
-      //  { "name": "localhost" }
-      //]
+      "defaultConnectionString": "Data Source=$ServerName$;Initial Catalog=master;Integrated Security=SSPI;",
+      "refreshIntervalSeconds": 30,
+      "instances": [
+        { "name": "localhost" }
+      ]
     },
     /* Configuration for the Redis dashboard */
     "Redis": {
@@ -38,13 +43,13 @@
     },
     /* Configuration for the Exceptions dashboard */
     "Exceptions": {
-      //"stores": [
-      //  {
-      //    "name": "Local",
-      //    "queryTimeoutMs": 2000,
-      //    "connectionString": "Server=.;Database=Local.Exceptions;Integrated Security=SSPI;"
-      //  }
-      //],
+      "stores": [
+        {
+          "name": "Local",
+          "queryTimeoutMs": 2000,
+          "connectionString": "Server=.;Database=Local.Exceptions;Integrated Security=SSPI;"
+        }
+      ],
       "stackTraceReplacements": [
         {
           "name": "github",