From ce9180c37e32ccc7e9103a74d5ae285f7b6b4422 Mon Sep 17 00:00:00 2001 From: Wouter de Kort Date: Thu, 31 Oct 2024 14:32:03 +0100 Subject: [PATCH 1/5] Try to setup Serilog --- cnab/Invoke-CNAB.ps1 | 2 +- src/Opserver.Web/Opserver.Web.csproj | 8 ++++ src/Opserver.Web/Program.cs | 55 +++++++++++++++++++++++--- src/Opserver.Web/Startup.cs | 27 ++++++++++++- src/Opserver.Web/appsettings.json | 30 ++++++++++++++ src/Opserver.Web/opserverSettings.json | 31 +++++++++------ 6 files changed, 131 insertions(+), 22 deletions(-) create mode 100644 src/Opserver.Web/appsettings.json diff --git a/cnab/Invoke-CNAB.ps1 b/cnab/Invoke-CNAB.ps1 index 8ace25af5..15418d01c 100644 --- a/cnab/Invoke-CNAB.ps1 +++ b/cnab/Invoke-CNAB.ps1 @@ -7,7 +7,7 @@ param ( $RunAsContainer = $false, [ValidateSet("GCP", "DockerDesktop")] [string] - $Target = "GCP", + $Target = "DockerDesktop", [bool] $DownloadLocalScriptsForLocalDebugging = $true ) diff --git a/src/Opserver.Web/Opserver.Web.csproj b/src/Opserver.Web/Opserver.Web.csproj index 6b7318c78..4da2aa3a1 100644 --- a/src/Opserver.Web/Opserver.Web.csproj +++ b/src/Opserver.Web/Opserver.Web.csproj @@ -10,6 +10,14 @@ + + + + + + + + diff --git a/src/Opserver.Web/Program.cs b/src/Opserver.Web/Program.cs index 75fbeaa47..3be5afaf9 100644 --- a/src/Opserver.Web/Program.cs +++ b/src/Opserver.Web/Program.cs @@ -7,6 +7,9 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Opserver.Helpers; +using Serilog; +using Serilog.Enrichers.Sensitive; +using Serilog.Exceptions; namespace Opserver { public static class Program @@ -40,14 +43,54 @@ private static async Task Main(string[] args) .AddEnvironmentVariables(); } ) - .ConfigureLogging( - (hostingContext, config) => + .ConfigureLogging( + (hostingContext, config) => + { + config.ClearProviders(); + + if (Debugger.IsAttached) { - var loggingConfig = hostingContext.Configuration.GetSection("Logging"); - config.AddConfiguration(loggingConfig) - .AddConsole(); + // To increase the visibility of internal Serilog errors we will log + // them to the debug output when the application is run via a debugger. + Serilog.Debugging.SelfLog.Enable(message => Debug.WriteLine(message)); } - ) + else + { + // If not running via a debugger, we'll default to the console error log (stderr) + // because we don't really have a better place and for Kubernetes style deployments + // this actually _is_ the right place to go. + Serilog.Debugging.SelfLog.Enable(Console.Error); + } + + var dataDogServiceName = Environment.GetEnvironmentVariable("DD_SERVICE") ?? "opserver"; + var product = hostingContext.Configuration.GetValue("Product"); + var tier = hostingContext.Configuration.GetValue("Tier"); + + var loggerconfig = new LoggerConfiguration() + .ReadFrom.Configuration(hostingContext.Configuration); + + loggerconfig + .Enrich.FromLogContext() + .Enrich.WithEnvironmentUserName() + .Enrich.WithEnvironmentName() + .Enrich.WithProperty("dd_service", dataDogServiceName) + .Enrich.WithMachineName() + .Enrich.WithThreadId() + .Enrich.WithExceptionDetails() + .Enrich.WithSensitiveDataMasking(options => + { + options.MaskingOperators = new List + { + new EmailAddressMaskingOperator() + }; + }); + + loggerconfig.Filter.ByExcluding("StartsWith(SourceContext, 'StackExchange.Exceptional.')"); + loggerconfig.Filter.ByExcluding("SourceContext='Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware' and @mt='An unhandled exception has occurred while executing the request.'"); + + config.AddSerilog(); + } + ) .UseStartup() .Build(); diff --git a/src/Opserver.Web/Startup.cs b/src/Opserver.Web/Startup.cs index d943554b2..93dcf8fd2 100644 --- a/src/Opserver.Web/Startup.cs +++ b/src/Opserver.Web/Startup.cs @@ -1,6 +1,7 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net; using System.Text.RegularExpressions; @@ -20,6 +21,8 @@ using Opserver.Data; using Opserver.Helpers; using Opserver.Security; +using Serilog; +using StackExchange.Exceptional; using StackExchange.Profiling; namespace Opserver @@ -65,6 +68,7 @@ public void ConfigureServices(IServiceCollection services) { settings.UseExceptionalPageOnThrow = true; settings.DataIncludeRegex = new Regex("^(Redis|Elastic|ErrorLog|Jil)", RegexOptions.Singleline | RegexOptions.Compiled); + settings.OnAfterLog += ExceptionalAfterLog; settings.GetCustomData = (ex, data) => { // everything below needs a context @@ -204,6 +208,17 @@ IEnumerable modules } } }) + .UseExceptionHandler(errorApp => + { + errorApp.Run(ctx => + { + var logger = ctx.RequestServices.GetRequiredService>(); + var exception = ctx.Features.Get().Error; + logger.LogError(exception, exception.Message); + + return Task.CompletedTask; + }); + }) .UseExceptional() .UseRouting() .UseMiniProfiler() @@ -240,9 +255,17 @@ IEnumerable modules AllowCachingResponses = false, Predicate = _ => false // We don't use any healthchecks for liveliness, just that the app responds - }); + }); NavTab.ConfigureAll(modules); // TODO: UseNavTabs() or something Cache.Configure(settings); } + + private void ExceptionalAfterLog(object sender, ErrorAfterLogEventArgs args) + { + if (args != null) + { + Serilog.Log.Logger.Error(args.Error.Exception, "{ErrorMessage}", args.Error.Message); + } + } } } diff --git a/src/Opserver.Web/appsettings.json b/src/Opserver.Web/appsettings.json new file mode 100644 index 000000000..81a2f1a6d --- /dev/null +++ b/src/Opserver.Web/appsettings.json @@ -0,0 +1,30 @@ +{ + "Serilog": { + "Using": [ "Serilog.Sinks.Console" ], + "Properties": { + "Application": "OpServer" + }, + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Warning", + "System": "Warning" + } + }, + "WriteTo": [ + { + "Name": "Async", + "Args": { + "configure": [ + { + "Name": "Console", + "Args": { + "formatter": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact" + } + } + ] + } + } + ] + } +} diff --git a/src/Opserver.Web/opserverSettings.json b/src/Opserver.Web/opserverSettings.json index 3efc89baf..54fa32459 100644 --- a/src/Opserver.Web/opserverSettings.json +++ b/src/Opserver.Web/opserverSettings.json @@ -1,17 +1,22 @@ { "ForwardedHeaders": { - "KnownNetworks": ["10.0.0.0/16"], + "KnownNetworks": [ "10.0.0.0/16" ], "KnownProxies": [], "ForwardedHeaders": "All" }, + "security": { + "adminGroups": "", + "viewGroups": "", + "provider": "EveryonesAnAdmin" + }, "Modules": { /* Configuration for the SQL Server dashboard */ "Sql": { - //"defaultConnectionString": "Data Source=$ServerName$;Initial Catalog=master;Integrated Security=SSPI;", - //"refreshIntervalSeconds": 30, - //"instances": [ - // { "name": "localhost" } - //] + "defaultConnectionString": "Data Source=$ServerName$;Initial Catalog=master;Integrated Security=SSPI;", + "refreshIntervalSeconds": 30, + "instances": [ + { "name": "localhost" } + ] }, /* Configuration for the Redis dashboard */ "Redis": { @@ -38,13 +43,13 @@ }, /* Configuration for the Exceptions dashboard */ "Exceptions": { - //"stores": [ - // { - // "name": "Local", - // "queryTimeoutMs": 2000, - // "connectionString": "Server=.;Database=Local.Exceptions;Integrated Security=SSPI;" - // } - //], + "stores": [ + { + "name": "Local", + "queryTimeoutMs": 2000, + "connectionString": "Server=.;Database=Local.Exceptions;Integrated Security=SSPI;" + } + ], "stackTraceReplacements": [ { "name": "github", From 0e0c0bb3a3951bbe6788dfe36425c13e54595650 Mon Sep 17 00:00:00 2001 From: Wouter de Kort Date: Thu, 31 Oct 2024 15:23:48 +0100 Subject: [PATCH 2/5] Don't use exceptional page --- src/Opserver.Web/Startup.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Opserver.Web/Startup.cs b/src/Opserver.Web/Startup.cs index 93dcf8fd2..7ca88a78f 100644 --- a/src/Opserver.Web/Startup.cs +++ b/src/Opserver.Web/Startup.cs @@ -66,7 +66,7 @@ public void ConfigureServices(IServiceCollection services) _configuration.GetSection("Exceptional"), settings => { - settings.UseExceptionalPageOnThrow = true; + settings.UseExceptionalPageOnThrow = false; settings.DataIncludeRegex = new Regex("^(Redis|Elastic|ErrorLog|Jil)", RegexOptions.Singleline | RegexOptions.Compiled); settings.OnAfterLog += ExceptionalAfterLog; settings.GetCustomData = (ex, data) => From 660f83db0b1f380a31a79357932a70593887b284 Mon Sep 17 00:00:00 2001 From: Wouter de Kort Date: Thu, 31 Oct 2024 15:23:52 +0100 Subject: [PATCH 3/5] Upgrade package versions --- src/Opserver.Web/Opserver.Web.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Opserver.Web/Opserver.Web.csproj b/src/Opserver.Web/Opserver.Web.csproj index 4da2aa3a1..89ab09c33 100644 --- a/src/Opserver.Web/Opserver.Web.csproj +++ b/src/Opserver.Web/Opserver.Web.csproj @@ -17,7 +17,7 @@ - + From 0659243ba8c7d57b8d5fb85886c81d0e9ebcced0 Mon Sep 17 00:00:00 2001 From: Wouter de Kort Date: Thu, 31 Oct 2024 16:32:34 +0100 Subject: [PATCH 4/5] Make it work! --- src/Opserver.Web/Opserver.Web.csproj | 5 +++ src/Opserver.Web/Program.cs | 50 +++++++++++----------------- src/Opserver.Web/Startup.cs | 14 ++------ src/Opserver.Web/appsettings.json | 2 ++ 4 files changed, 28 insertions(+), 43 deletions(-) diff --git a/src/Opserver.Web/Opserver.Web.csproj b/src/Opserver.Web/Opserver.Web.csproj index 89ab09c33..6774711ff 100644 --- a/src/Opserver.Web/Opserver.Web.csproj +++ b/src/Opserver.Web/Opserver.Web.csproj @@ -5,6 +5,10 @@ net6.0 + + + + @@ -18,6 +22,7 @@ + diff --git a/src/Opserver.Web/Program.cs b/src/Opserver.Web/Program.cs index 3be5afaf9..854a748ed 100644 --- a/src/Opserver.Web/Program.cs +++ b/src/Opserver.Web/Program.cs @@ -3,9 +3,11 @@ using System.Diagnostics; using System.Threading.Tasks; using Microsoft.AspNetCore; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; + using Opserver.Helpers; using Serilog; using Serilog.Enrichers.Sensitive; @@ -20,6 +22,7 @@ private static async Task Main(string[] args) { try { + var host = WebHost.CreateDefaultBuilder(args) .ConfigureAppConfiguration( (_, config) => @@ -48,47 +51,32 @@ private static async Task Main(string[] args) { config.ClearProviders(); - if (Debugger.IsAttached) - { - // To increase the visibility of internal Serilog errors we will log - // them to the debug output when the application is run via a debugger. - Serilog.Debugging.SelfLog.Enable(message => Debug.WriteLine(message)); - } - else - { - // If not running via a debugger, we'll default to the console error log (stderr) - // because we don't really have a better place and for Kubernetes style deployments - // this actually _is_ the right place to go. - Serilog.Debugging.SelfLog.Enable(Console.Error); - } - var dataDogServiceName = Environment.GetEnvironmentVariable("DD_SERVICE") ?? "opserver"; var product = hostingContext.Configuration.GetValue("Product"); var tier = hostingContext.Configuration.GetValue("Tier"); var loggerconfig = new LoggerConfiguration() - .ReadFrom.Configuration(hostingContext.Configuration); - - loggerconfig - .Enrich.FromLogContext() - .Enrich.WithEnvironmentUserName() - .Enrich.WithEnvironmentName() - .Enrich.WithProperty("dd_service", dataDogServiceName) - .Enrich.WithMachineName() - .Enrich.WithThreadId() - .Enrich.WithExceptionDetails() - .Enrich.WithSensitiveDataMasking(options => + .ReadFrom.Configuration(hostingContext.Configuration) + .Enrich.FromLogContext() + .Enrich.WithEnvironmentUserName() + .Enrich.WithEnvironmentName() + .Enrich.WithProperty("dd_service", dataDogServiceName) + .Enrich.WithMachineName() + .Enrich.WithThreadId() + .Enrich.WithExceptionDetails() + .Enrich.WithSensitiveDataMasking(options => + { + options.MaskingOperators = new List { - options.MaskingOperators = new List - { - new EmailAddressMaskingOperator() - }; - }); + new EmailAddressMaskingOperator() + }; + }); loggerconfig.Filter.ByExcluding("StartsWith(SourceContext, 'StackExchange.Exceptional.')"); loggerconfig.Filter.ByExcluding("SourceContext='Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware' and @mt='An unhandled exception has occurred while executing the request.'"); - config.AddSerilog(); + Log.Logger = loggerconfig.CreateLogger(); + config.AddSerilog(Log.Logger); } ) .UseStartup() diff --git a/src/Opserver.Web/Startup.cs b/src/Opserver.Web/Startup.cs index 7ca88a78f..d55d80aea 100644 --- a/src/Opserver.Web/Startup.cs +++ b/src/Opserver.Web/Startup.cs @@ -208,17 +208,6 @@ IEnumerable modules } } }) - .UseExceptionHandler(errorApp => - { - errorApp.Run(ctx => - { - var logger = ctx.RequestServices.GetRequiredService>(); - var exception = ctx.Features.Get().Error; - logger.LogError(exception, exception.Message); - - return Task.CompletedTask; - }); - }) .UseExceptional() .UseRouting() .UseMiniProfiler() @@ -264,7 +253,8 @@ private void ExceptionalAfterLog(object sender, ErrorAfterLogEventArgs args) { if (args != null) { - Serilog.Log.Logger.Error(args.Error.Exception, "{ErrorMessage}", args.Error.Message); + string message = args.Error.Message; + Serilog.Log.Error(args.Error.Exception, "{Message}", message); } } } diff --git a/src/Opserver.Web/appsettings.json b/src/Opserver.Web/appsettings.json index 81a2f1a6d..4d8890c11 100644 --- a/src/Opserver.Web/appsettings.json +++ b/src/Opserver.Web/appsettings.json @@ -1,4 +1,6 @@ { + "Product": "pubplat" + "Tier": "Local", "Serilog": { "Using": [ "Serilog.Sinks.Console" ], "Properties": { From 9962379937d60f6acb61a3923fee02fa1daf24d4 Mon Sep 17 00:00:00 2001 From: Wouter de Kort Date: Thu, 31 Oct 2024 16:36:26 +0100 Subject: [PATCH 5/5] Revert some testing changes --- cnab/Invoke-CNAB.ps1 | 2 +- src/Opserver.Web/Opserver.Web.csproj | 6 ++--- src/Opserver.Web/appsettings.json | 2 -- src/Opserver.Web/opserverSettings.json | 31 +++++++++++--------------- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/cnab/Invoke-CNAB.ps1 b/cnab/Invoke-CNAB.ps1 index 15418d01c..8ace25af5 100644 --- a/cnab/Invoke-CNAB.ps1 +++ b/cnab/Invoke-CNAB.ps1 @@ -7,7 +7,7 @@ param ( $RunAsContainer = $false, [ValidateSet("GCP", "DockerDesktop")] [string] - $Target = "DockerDesktop", + $Target = "GCP", [bool] $DownloadLocalScriptsForLocalDebugging = $true ) diff --git a/src/Opserver.Web/Opserver.Web.csproj b/src/Opserver.Web/Opserver.Web.csproj index 6774711ff..a4c506322 100644 --- a/src/Opserver.Web/Opserver.Web.csproj +++ b/src/Opserver.Web/Opserver.Web.csproj @@ -6,21 +6,21 @@ - - - + + + diff --git a/src/Opserver.Web/appsettings.json b/src/Opserver.Web/appsettings.json index 4d8890c11..81a2f1a6d 100644 --- a/src/Opserver.Web/appsettings.json +++ b/src/Opserver.Web/appsettings.json @@ -1,6 +1,4 @@ { - "Product": "pubplat" - "Tier": "Local", "Serilog": { "Using": [ "Serilog.Sinks.Console" ], "Properties": { diff --git a/src/Opserver.Web/opserverSettings.json b/src/Opserver.Web/opserverSettings.json index 54fa32459..3efc89baf 100644 --- a/src/Opserver.Web/opserverSettings.json +++ b/src/Opserver.Web/opserverSettings.json @@ -1,22 +1,17 @@ { "ForwardedHeaders": { - "KnownNetworks": [ "10.0.0.0/16" ], + "KnownNetworks": ["10.0.0.0/16"], "KnownProxies": [], "ForwardedHeaders": "All" }, - "security": { - "adminGroups": "", - "viewGroups": "", - "provider": "EveryonesAnAdmin" - }, "Modules": { /* Configuration for the SQL Server dashboard */ "Sql": { - "defaultConnectionString": "Data Source=$ServerName$;Initial Catalog=master;Integrated Security=SSPI;", - "refreshIntervalSeconds": 30, - "instances": [ - { "name": "localhost" } - ] + //"defaultConnectionString": "Data Source=$ServerName$;Initial Catalog=master;Integrated Security=SSPI;", + //"refreshIntervalSeconds": 30, + //"instances": [ + // { "name": "localhost" } + //] }, /* Configuration for the Redis dashboard */ "Redis": { @@ -43,13 +38,13 @@ }, /* Configuration for the Exceptions dashboard */ "Exceptions": { - "stores": [ - { - "name": "Local", - "queryTimeoutMs": 2000, - "connectionString": "Server=.;Database=Local.Exceptions;Integrated Security=SSPI;" - } - ], + //"stores": [ + // { + // "name": "Local", + // "queryTimeoutMs": 2000, + // "connectionString": "Server=.;Database=Local.Exceptions;Integrated Security=SSPI;" + // } + //], "stackTraceReplacements": [ { "name": "github",