From eaffe8f57fc8158bc85bf944829f56d38393b309 Mon Sep 17 00:00:00 2001 From: Nazul Grimaldo Date: Mon, 6 Nov 2023 01:10:57 -0600 Subject: [PATCH 1/2] Adds a custom exporter option on command line args --- .../ConsoleArguments/CommandLineOptions.cs | 3 +++ .../ConsoleArguments/ConfigParser.cs | 18 ++++++++++++++++++ .../BenchmarkDotNet.Tests/ConfigParserTests.cs | 16 ++++++++++++++++ 3 files changed, 37 insertions(+) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index e3aed1fedd..5550095482 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -31,6 +31,9 @@ public class CommandLineOptions [Option('e', "exporters", Required = false, HelpText = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML")] public IEnumerable Exporters { get; set; } + [Option( "customExporter", Required = false, HelpText = "The assembly-qualified name of the Custom Exporter type")] + public string CustomExporter { get; set; } + [Option('m', "memory", Required = false, Default = false, HelpText = "Prints memory statistics")] public bool UseMemoryDiagnoser { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 0a125900f4..a246fa2758 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -49,6 +49,9 @@ public static class ConfigParser { "verylong", Job.VeryLongRun } }; + public static void RegisterCustomExporter(string commandLineName, IExporter exporter) => CustomExporters.Add(commandLineName, exporter); + private static readonly IDictionary CustomExporters = new Dictionary(); + [SuppressMessage("ReSharper", "StringLiteralTypo")] [SuppressMessage("ReSharper", "CoVariantArrayConversion")] private static readonly IReadOnlyDictionary AvailableExporters = @@ -252,6 +255,20 @@ private static bool Validate(CommandLineOptions options, ILogger logger) } } + if (!string.IsNullOrWhiteSpace(options.CustomExporter)) + { + try + { + var customExporter = Activator.CreateInstance(Type.GetType(options.CustomExporter)); + CustomExporters["customExporter"] = (IExporter)customExporter; + } + catch (Exception ex) + { + logger.WriteLineError(ex.ToString()); + } + + } + foreach (string exporter in options.Exporters) if (!AvailableExporters.ContainsKey(exporter)) { @@ -339,6 +356,7 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.AddJob(baseJob); config.AddExporter(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray()); + config.AddExporter(CustomExporters.Select(c => c.Value).ToArray()); config.AddHardwareCounters(options.HardwareCounters .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true)) diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index ec9b85021f..3634dbb871 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -25,15 +25,31 @@ using Perfolizer.Horology; using Perfolizer.Mathematics.SignificanceTesting; using Perfolizer.Mathematics.Thresholds; +using BenchmarkDotNet.Exporters.Json; namespace BenchmarkDotNet.Tests { + public class CustomExporterTestClass : JsonExporterBase { } public class ConfigParserTests { public ITestOutputHelper Output { get; } public ConfigParserTests(ITestOutputHelper output) => Output = output; + [Theory] + [InlineData("--customExporter", "BenchmarkDotNet.Tests.CustomExporterTestClass, BenchmarkDotNet.Tests")] + public void CustomExporterConfigParsedCorrectly(params string[] args) + { + var config = ConfigParser.Parse(args, new OutputLogger(Output)).config; + + var customExporter = config.GetExporters().ToList(); + Assert.Equal("CustomExporterTestClass", customExporter[0].Name); + Assert.Empty(config.GetColumnProviders()); + Assert.Empty(config.GetDiagnosers()); + Assert.Empty(config.GetAnalysers()); + Assert.Empty(config.GetLoggers()); + } + [Theory] [InlineData("--job=dry", "--exporters", "html", "rplot")] [InlineData("--JOB=dry", "--EXPORTERS", "html", "rplot")] // case insensitive From b3406e5588823e796d9aa93bac05d4bbd926dafb Mon Sep 17 00:00:00 2001 From: Nazul Grimaldo Date: Thu, 9 Nov 2023 07:37:11 -0600 Subject: [PATCH 2/2] Extends exporters to accept a type definition for a custom exporter --- .../ConsoleArguments/CommandLineOptions.cs | 3 -- .../ConsoleArguments/ConfigParser.cs | 41 ++++++++++--------- .../ConfigParserTests.cs | 12 ++++-- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 5550095482..e3aed1fedd 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -31,9 +31,6 @@ public class CommandLineOptions [Option('e', "exporters", Required = false, HelpText = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML")] public IEnumerable Exporters { get; set; } - [Option( "customExporter", Required = false, HelpText = "The assembly-qualified name of the Custom Exporter type")] - public string CustomExporter { get; set; } - [Option('m', "memory", Required = false, Default = false, HelpText = "Prints memory statistics")] public bool UseMemoryDiagnoser { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index a246fa2758..7acfc89392 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -49,12 +49,9 @@ public static class ConfigParser { "verylong", Job.VeryLongRun } }; - public static void RegisterCustomExporter(string commandLineName, IExporter exporter) => CustomExporters.Add(commandLineName, exporter); - private static readonly IDictionary CustomExporters = new Dictionary(); - [SuppressMessage("ReSharper", "StringLiteralTypo")] [SuppressMessage("ReSharper", "CoVariantArrayConversion")] - private static readonly IReadOnlyDictionary AvailableExporters = + private static readonly IDictionary AvailableExporters = new Dictionary(StringComparer.InvariantCultureIgnoreCase) { { "csv", new[] { CsvExporter.Default } }, @@ -75,6 +72,21 @@ public static class ConfigParser { "fullxml", new[] { XmlExporter.Full } } }; + + private static bool TryCreateCustomExporter(string customExporterName) + { + try + { + var customExporter = Activator.CreateInstance(Type.GetType(customExporterName)); + AvailableExporters.Add(customExporterName, new IExporter[] { (IExporter)customExporter }); + return true; + } + catch (Exception) + { + return false; + } + } + public static (bool isSuccess, IConfig config, CommandLineOptions options) Parse(string[] args, ILogger logger, IConfig? globalConfig = null) { (bool isSuccess, IConfig config, CommandLineOptions options) result = default; @@ -255,26 +267,18 @@ private static bool Validate(CommandLineOptions options, ILogger logger) } } - if (!string.IsNullOrWhiteSpace(options.CustomExporter)) + foreach (string exporter in options.Exporters) { - try - { - var customExporter = Activator.CreateInstance(Type.GetType(options.CustomExporter)); - CustomExporters["customExporter"] = (IExporter)customExporter; - } - catch (Exception ex) + if (AvailableExporters.ContainsKey(exporter) || TryCreateCustomExporter(exporter)) { - logger.WriteLineError(ex.ToString()); + continue; } - - } - - foreach (string exporter in options.Exporters) - if (!AvailableExporters.ContainsKey(exporter)) + else { - logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)}."); + logger.WriteLineError($"The provided exporter \"{exporter}\" is invalid. Available options are: {string.Join(", ", AvailableExporters.Keys)} or custom exporter by assembly-qualified name."); return false; } + } if (options.CliPath.IsNotNullButDoesNotExist()) { @@ -356,7 +360,6 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo config.AddJob(baseJob); config.AddExporter(options.Exporters.SelectMany(exporter => AvailableExporters[exporter]).ToArray()); - config.AddExporter(CustomExporters.Select(c => c.Value).ToArray()); config.AddHardwareCounters(options.HardwareCounters .Select(counterName => (HardwareCounter)Enum.Parse(typeof(HardwareCounter), counterName, ignoreCase: true)) diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index 3634dbb871..283135c661 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -26,6 +26,7 @@ using Perfolizer.Mathematics.SignificanceTesting; using Perfolizer.Mathematics.Thresholds; using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Exporters.Xml; namespace BenchmarkDotNet.Tests { @@ -37,13 +38,18 @@ public class ConfigParserTests public ConfigParserTests(ITestOutputHelper output) => Output = output; [Theory] - [InlineData("--customExporter", "BenchmarkDotNet.Tests.CustomExporterTestClass, BenchmarkDotNet.Tests")] + [InlineData("--exporters", "BenchmarkDotNet.Tests.CustomExporterTestClass, BenchmarkDotNet.Tests", "html", "xml")] + [InlineData("--exporters", "html", "BenchmarkDotNet.Tests.CustomExporterTestClass, BenchmarkDotNet.Tests", "xml")] + [InlineData("--exporters", "html", "xml", "BenchmarkDotNet.Tests.CustomExporterTestClass, BenchmarkDotNet.Tests")] public void CustomExporterConfigParsedCorrectly(params string[] args) { var config = ConfigParser.Parse(args, new OutputLogger(Output)).config; - var customExporter = config.GetExporters().ToList(); - Assert.Equal("CustomExporterTestClass", customExporter[0].Name); + Assert.Equal(3, config.GetExporters().Count()); + Assert.Contains(typeof(CustomExporterTestClass).Name, config.GetExporters().Select(e => e.Name)); + Assert.Contains(HtmlExporter.Default, config.GetExporters()); + Assert.Contains(XmlExporter.Default, config.GetExporters()); + Assert.Empty(config.GetColumnProviders()); Assert.Empty(config.GetDiagnosers()); Assert.Empty(config.GetAnalysers());